From 569716fc5c2c232adcd5ff840637be596c1de9b9 Mon Sep 17 00:00:00 2001
From: Tom Eccles <tom.eccles@arm.com>
Date: Wed, 14 Jun 2023 13:23:00 +0000
Subject: [PATCH 001/130] [flang][hlfir] Fix multiple return declaration type

When the ENTRY statement is used, the same source can return different
types depending on the entry point. These different return values are
storage associated (share the same storage). Previously, this led to the
declaration of the results to all have the largest type. This patch adds
a convert between the stack allocation and the declaration so that the
hlfir.decl gets the right type.

I haven't managed to generate code where this convert converted a
reference to an allocation for a smaller type into an allocation for a
larger one, but I have added an assert just in case.

This is a different solution to https://reviews.llvm.org/D152725, see
discussion there.

Differential Revision: https://reviews.llvm.org/D152931
---
 flang/lib/Lower/Bridge.cpp              | 32 ++++++++-
 flang/test/Lower/HLFIR/entry_return.f90 | 86 +++++++++++++++++++++++++
 2 files changed, 117 insertions(+), 1 deletion(-)
 create mode 100644 flang/test/Lower/HLFIR/entry_return.f90

diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index ee21d63c02333..f3efbfaa2dc21 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -3993,8 +3993,38 @@ class FirConverter : public Fortran::lower::AbstractConverter {
         Fortran::lower::mapSymbolAttributes(*this, altResult, localSymbols,
                                             stmtCtx);
       } else {
+        // catch cases where the allocation for the function result storage type
+        // doesn't match the type of this symbol
+        mlir::Value preAlloc = primaryFuncResultStorage;
+        mlir::Type resTy = primaryFuncResultStorage.getType();
+        mlir::Type symTy = genType(altResult);
+        mlir::Type wrappedSymTy = fir::ReferenceType::get(symTy);
+        if (resTy != wrappedSymTy) {
+          // check size of the pointed to type so we can't overflow by writing
+          // double precision to a single precision allocation, etc
+          LLVM_ATTRIBUTE_UNUSED auto getBitWidth = [this](mlir::Type ty) {
+            // 15.6.2.6.3: differering result types should be integer, real,
+            // complex or logical
+            if (auto cmplx = mlir::dyn_cast_or_null<fir::ComplexType>(ty)) {
+              fir::KindTy kind = cmplx.getFKind();
+              return 2 * builder->getKindMap().getRealBitsize(kind);
+            }
+            if (auto logical = mlir::dyn_cast_or_null<fir::LogicalType>(ty)) {
+              fir::KindTy kind = logical.getFKind();
+              return builder->getKindMap().getLogicalBitsize(kind);
+            }
+            return ty.getIntOrFloatBitWidth();
+          };
+          assert(getBitWidth(fir::unwrapRefType(resTy)) >= getBitWidth(symTy));
+
+          // convert the storage to the symbol type so that the hlfir.declare
+          // gets the correct type for this symbol
+          preAlloc = builder->create<fir::ConvertOp>(getCurrentLocation(),
+                                                     wrappedSymTy, preAlloc);
+        }
+
         Fortran::lower::mapSymbolAttributes(*this, altResult, localSymbols,
-                                            stmtCtx, primaryFuncResultStorage);
+                                            stmtCtx, preAlloc);
       }
     }
 
diff --git a/flang/test/Lower/HLFIR/entry_return.f90 b/flang/test/Lower/HLFIR/entry_return.f90
new file mode 100644
index 0000000000000..d2fb80c8b97b7
--- /dev/null
+++ b/flang/test/Lower/HLFIR/entry_return.f90
@@ -0,0 +1,86 @@
+! RUN: bbc -emit-hlfir -o - %s | FileCheck %s
+! test multiple return values with different types coming from ENTRY statements
+
+complex function f1()
+  logical e1
+  entry e1()
+  e1 = .false.
+end function
+! CHECK-LABEL:   func.func @_QPf1() -> !fir.complex<4> {
+! CHECK:           %[[VAL_0:.*]] = fir.alloca !fir.complex<4> {bindc_name = "f1", uniq_name = "_QFf1Ef1"}
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFf1Ef1"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#1 : (!fir.ref<!fir.complex<4>>) -> !fir.ref<!fir.logical<4>>
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFf1Ee1"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:           cf.br ^bb1
+! CHECK:         ^bb1:
+! CHECK:           %[[VAL_4:.*]] = arith.constant false
+! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (i1) -> !fir.logical<4>
+! CHECK:           hlfir.assign %[[VAL_5]] to %[[VAL_3]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
+! CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_1]]#1 : !fir.ref<!fir.complex<4>>
+! CHECK:           return %[[VAL_6]] : !fir.complex<4>
+! CHECK:         }
+
+! // CHECK-LABEL:   func.func @_QPe1() -> !fir.logical<4> {
+! CHECK:           %[[VAL_0:.*]] = fir.alloca !fir.complex<4> {bindc_name = "f1", uniq_name = "_QFf1Ef1"}
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFf1Ef1"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#1 : (!fir.ref<!fir.complex<4>>) -> !fir.ref<!fir.logical<4>>
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFf1Ee1"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:           cf.br ^bb1
+! CHECK:         ^bb1:
+! CHECK:           %[[VAL_4:.*]] = arith.constant false
+! CHECK:           %[[VAL_5:.*]] = fir.convert %[[VAL_4]] : (i1) -> !fir.logical<4>
+! CHECK:           hlfir.assign %[[VAL_5]] to %[[VAL_3]]#0 : !fir.logical<4>, !fir.ref<!fir.logical<4>>
+! CHECK:           %[[VAL_6:.*]] = fir.load %[[VAL_3]]#1 : !fir.ref<!fir.logical<4>>
+! CHECK:           return %[[VAL_6]] : !fir.logical<4>
+! CHECK:         }
+
+logical function f2()
+  complex e2
+  entry e2()
+  e2 = complex(1.0, 2.0)
+end function
+! CHECK-LABEL:   func.func @_QPf2() -> !fir.logical<4> {
+! CHECK:           %[[VAL_0:.*]] = fir.alloca !fir.complex<4> {bindc_name = "e2", uniq_name = "_QFf2Ee2"}
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFf2Ee2"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#1 : (!fir.ref<!fir.complex<4>>) -> !fir.ref<!fir.logical<4>>
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFf2Ef2"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:           cf.br ^bb1
+! CHECK:         ^bb1:
+! CHECK:           %[[VAL_4:.*]] = arith.constant 1.000000e+00 : f32
+! CHECK:           %[[VAL_5:.*]] = arith.constant 2.000000e+00 : f32
+! CHECK:           %[[VAL_6:.*]]:3 = hlfir.associate %[[VAL_4]] {uniq_name = "adapt.valuebyref"} : (f32) -> (!fir.ref<f32>, !fir.ref<f32>, i1)
+! CHECK:           %[[VAL_7:.*]]:3 = hlfir.associate %[[VAL_5]] {uniq_name = "adapt.valuebyref"} : (f32) -> (!fir.ref<f32>, !fir.ref<f32>, i1)
+! CHECK:           %[[VAL_8:.*]] = fir.call @_QPcomplex(%[[VAL_6]]#1, %[[VAL_7]]#1) fastmath<contract> : (!fir.ref<f32>, !fir.ref<f32>) -> f32
+! CHECK:           hlfir.end_associate %[[VAL_6]]#1, %[[VAL_6]]#2 : !fir.ref<f32>, i1
+! CHECK:           hlfir.end_associate %[[VAL_7]]#1, %[[VAL_7]]#2 : !fir.ref<f32>, i1
+! CHECK:           %[[VAL_9:.*]] = arith.constant 0.000000e+00 : f32
+! CHECK:           %[[VAL_10:.*]] = fir.undefined !fir.complex<4>
+! CHECK:           %[[VAL_11:.*]] = fir.insert_value %[[VAL_10]], %[[VAL_8]], [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
+! CHECK:           %[[VAL_12:.*]] = fir.insert_value %[[VAL_11]], %[[VAL_9]], [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
+! CHECK:           hlfir.assign %[[VAL_12]] to %[[VAL_1]]#0 : !fir.complex<4>, !fir.ref<!fir.complex<4>>
+! CHECK:           %[[VAL_13:.*]] = fir.load %[[VAL_3]]#1 : !fir.ref<!fir.logical<4>>
+! CHECK:           return %[[VAL_13]] : !fir.logical<4>
+! CHECK:         }
+
+! CHECK-LABEL:   func.func @_QPe2() -> !fir.complex<4> {
+! CHECK:           %[[VAL_0:.*]] = fir.alloca !fir.complex<4> {bindc_name = "e2", uniq_name = "_QFf2Ee2"}
+! CHECK:           %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] {uniq_name = "_QFf2Ee2"} : (!fir.ref<!fir.complex<4>>) -> (!fir.ref<!fir.complex<4>>, !fir.ref<!fir.complex<4>>)
+! CHECK:           %[[VAL_2:.*]] = fir.convert %[[VAL_1]]#1 : (!fir.ref<!fir.complex<4>>) -> !fir.ref<!fir.logical<4>>
+! CHECK:           %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_2]] {uniq_name = "_QFf2Ef2"} : (!fir.ref<!fir.logical<4>>) -> (!fir.ref<!fir.logical<4>>, !fir.ref<!fir.logical<4>>)
+! CHECK:           cf.br ^bb1
+! CHECK:         ^bb1:
+! CHECK:           %[[VAL_4:.*]] = arith.constant 1.000000e+00 : f32
+! CHECK:           %[[VAL_5:.*]] = arith.constant 2.000000e+00 : f32
+! CHECK:           %[[VAL_6:.*]]:3 = hlfir.associate %[[VAL_4]] {uniq_name = "adapt.valuebyref"} : (f32) -> (!fir.ref<f32>, !fir.ref<f32>, i1)
+! CHECK:           %[[VAL_7:.*]]:3 = hlfir.associate %[[VAL_5]] {uniq_name = "adapt.valuebyref"} : (f32) -> (!fir.ref<f32>, !fir.ref<f32>, i1)
+! CHECK:           %[[VAL_8:.*]] = fir.call @_QPcomplex(%[[VAL_6]]#1, %[[VAL_7]]#1) fastmath<contract> : (!fir.ref<f32>, !fir.ref<f32>) -> f32
+! CHECK:           hlfir.end_associate %[[VAL_6]]#1, %[[VAL_6]]#2 : !fir.ref<f32>, i1
+! CHECK:           hlfir.end_associate %[[VAL_7]]#1, %[[VAL_7]]#2 : !fir.ref<f32>, i1
+! CHECK:           %[[VAL_9:.*]] = arith.constant 0.000000e+00 : f32
+! CHECK:           %[[VAL_10:.*]] = fir.undefined !fir.complex<4>
+! CHECK:           %[[VAL_11:.*]] = fir.insert_value %[[VAL_10]], %[[VAL_8]], [0 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
+! CHECK:           %[[VAL_12:.*]] = fir.insert_value %[[VAL_11]], %[[VAL_9]], [1 : index] : (!fir.complex<4>, f32) -> !fir.complex<4>
+! CHECK:           hlfir.assign %[[VAL_12]] to %[[VAL_1]]#0 : !fir.complex<4>, !fir.ref<!fir.complex<4>>
+! CHECK:           %[[VAL_13:.*]] = fir.load %[[VAL_1]]#1 : !fir.ref<!fir.complex<4>>
+! CHECK:           return %[[VAL_13]] : !fir.complex<4>
+! CHECK:         }

From f523b9a55a3adecf1a8373ca7525630bdd7fb5ef Mon Sep 17 00:00:00 2001
From: Tom Eccles <tom.eccles@arm.com>
Date: Wed, 14 Jun 2023 16:23:40 +0000
Subject: [PATCH 002/130] [flang] don't allow conversions between logical and
 floating point

Codegen only supports conversions between logicals and integers. The
verifier should reflect this.

Differential Revision: https://reviews.llvm.org/D152935
---
 flang/include/flang/Optimizer/Dialect/FIROps.td |  1 +
 flang/lib/Optimizer/Dialect/FIROps.cpp          | 11 +++++++----
 flang/test/Fir/invalid.fir                      | 16 ++++++++++++++++
 3 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index c9d68f1e1cf90..8b05c97360607 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -2606,6 +2606,7 @@ def fir_ConvertOp : fir_OneResultOp<"convert", [NoMemoryEffect]> {
   let hasVerifier = 1;
 
   let extraClassDeclaration = [{
+    static bool isInteger(mlir::Type ty);
     static bool isIntegerCompatible(mlir::Type ty);
     static bool isFloatCompatible(mlir::Type ty);
     static bool isPointerCompatible(mlir::Type ty);
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index e796f2c385d95..7f899a2937987 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -928,9 +928,12 @@ mlir::OpFoldResult fir::ConvertOp::fold(FoldAdaptor adaptor) {
   return {};
 }
 
+bool fir::ConvertOp::isInteger(mlir::Type ty) {
+  return ty.isa<mlir::IntegerType, mlir::IndexType, fir::IntegerType>();
+}
+
 bool fir::ConvertOp::isIntegerCompatible(mlir::Type ty) {
-  return ty.isa<mlir::IntegerType, mlir::IndexType, fir::IntegerType,
-                fir::LogicalType>();
+  return isInteger(ty) || mlir::isa<fir::LogicalType>(ty);
 }
 
 bool fir::ConvertOp::isFloatCompatible(mlir::Type ty) {
@@ -1001,8 +1004,8 @@ bool fir::ConvertOp::canBeConverted(mlir::Type inType, mlir::Type outType) {
     return true;
   return (isPointerCompatible(inType) && isPointerCompatible(outType)) ||
          (isIntegerCompatible(inType) && isIntegerCompatible(outType)) ||
-         (isIntegerCompatible(inType) && isFloatCompatible(outType)) ||
-         (isFloatCompatible(inType) && isIntegerCompatible(outType)) ||
+         (isInteger(inType) && isFloatCompatible(outType)) ||
+         (isFloatCompatible(inType) && isInteger(outType)) ||
          (isFloatCompatible(inType) && isFloatCompatible(outType)) ||
          (isIntegerCompatible(inType) && isPointerCompatible(outType)) ||
          (isPointerCompatible(inType) && isIntegerCompatible(outType)) ||
diff --git a/flang/test/Fir/invalid.fir b/flang/test/Fir/invalid.fir
index c01bcc809d341..c3bfb6922deda 100644
--- a/flang/test/Fir/invalid.fir
+++ b/flang/test/Fir/invalid.fir
@@ -946,3 +946,19 @@ func.func @invalid_selector(%arg : !fir.box<!fir.ref<i32>>) -> i32 {
   %zero = arith.constant 0 : i32
   return %zero : i32
 }
+
+// -----
+
+func.func @logical_to_fp(%arg0: !fir.logical<4>) -> f32 {
+  // expected-error@+1{{'fir.convert' op invalid type conversion}}
+  %0 = fir.convert %arg0 : (!fir.logical<4>) -> f32
+  return %0 : f32
+}
+
+// -----
+
+func.func @fp_to_logical(%arg0: f32) -> !fir.logical<4> {
+  // expected-error@+1{{'fir.convert' op invalid type conversion}}
+  %0 = fir.convert %arg0 : (f32) -> !fir.logical<4>
+  return %0 : !fir.logical<4>
+}

From 3f8d8c1aac3086f603ad73f18fe2bd4fb91fa10a Mon Sep 17 00:00:00 2001
From: Jacob Crawley <jacob.crawley@arm.com>
Date: Mon, 12 Jun 2023 12:52:47 +0000
Subject: [PATCH 003/130] [flang][hlfir] Add hlfir.count intrinsic

Adds a new HLFIR operation for the COUNT intrinsic according to
the design set out in flang/docs/HighLevel.md. This patch includes all
the necessary changes to create a new HLFIR operation and lower it into
the fir runtime call.

Author was @jacob-crawley. Minor adjustments by @tblah

Differential Revision: https://reviews.llvm.org/D152521
---
 .../include/flang/Optimizer/HLFIR/HLFIROps.td |  23 ++-
 flang/lib/Lower/ConvertCall.cpp               |  12 ++
 flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp     |  37 ++++
 .../HLFIR/Transforms/LowerHLFIRIntrinsics.cpp |  37 +++-
 flang/test/HLFIR/count-lowering.fir           | 164 ++++++++++++++++++
 flang/test/HLFIR/count.fir                    |  83 +++++++++
 flang/test/HLFIR/invalid.fir                  |  24 +++
 flang/test/Lower/HLFIR/count.f90              |  82 +++++++++
 8 files changed, 459 insertions(+), 3 deletions(-)
 create mode 100644 flang/test/HLFIR/count-lowering.fir
 create mode 100644 flang/test/HLFIR/count.fir
 create mode 100644 flang/test/Lower/HLFIR/count.f90

diff --git a/flang/include/flang/Optimizer/HLFIR/HLFIROps.td b/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
index 2dd85a2c5c181..beb50a48d86df 100644
--- a/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
+++ b/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
@@ -359,6 +359,27 @@ def hlfir_AnyOp : hlfir_Op<"any", []> {
   let hasVerifier = 1;
 }
 
+def hlfir_CountOp : hlfir_Op<"count", [AttrSizedOperandSegments]> {
+  let summary = "COUNT transformational intrinsic";
+  let description = [{
+    Takes a logical and counts the number of true values.
+  }];
+
+  let arguments = (ins
+    AnyFortranLogicalArrayObject:$mask,
+    Optional<AnyIntegerType>:$dim,
+    Optional<AnyIntegerType>:$kind
+  );
+
+  let results = (outs AnyFortranValue);
+
+  let assemblyFormat = [{
+    $mask  (`dim` $dim^)? (`kind` $kind^)?  attr-dict `:` functional-type(operands, results)
+  }];
+
+  let hasVerifier = 1;
+}
+
 
 def hlfir_ProductOp : hlfir_Op<"product", [AttrSizedOperandSegments,
     DeclareOpInterfaceMethods<ArithFastMathInterface>]> {
@@ -430,7 +451,7 @@ def hlfir_SumOp : hlfir_Op<"sum", [AttrSizedOperandSegments,
   let hasVerifier = 1;
 }
 
-def hlifr_DotProductOp : hlfir_Op<"dot_product",
+def hlfir_DotProductOp : hlfir_Op<"dot_product",
     [DeclareOpInterfaceMethods<ArithFastMathInterface>]> {
   let summary = "DOT_PRODUCT transformational intrinsic";
   let description = [{
diff --git a/flang/lib/Lower/ConvertCall.cpp b/flang/lib/Lower/ConvertCall.cpp
index 604291cdbec6d..4f4505c8b0664 100644
--- a/flang/lib/Lower/ConvertCall.cpp
+++ b/flang/lib/Lower/ConvertCall.cpp
@@ -1497,6 +1497,18 @@ genHLFIRIntrinsicRefCore(PreparedActualArguments &loweredActuals,
 
     return {hlfir::EntityWithAttributes{dotProductOp.getResult()}};
   }
+  if (intrinsicName == "count") {
+    llvm::SmallVector<mlir::Value> operands = getOperandVector(loweredActuals);
+    mlir::Value array = operands[0];
+    mlir::Value dim = operands[1];
+    if (dim)
+      dim = hlfir::loadTrivialScalar(loc, builder, hlfir::Entity{dim});
+    mlir::Value kind = operands[2];
+    mlir::Type resultTy = computeResultType(array, *callContext.resultType);
+    hlfir::CountOp countOp =
+        builder.create<hlfir::CountOp>(loc, resultTy, array, dim, kind);
+    return {hlfir::EntityWithAttributes{countOp.getResult()}};
+  }
 
   // TODO add hlfir operations for other transformational intrinsics here
 
diff --git a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
index 7a00bfae75ed1..21a44c07953b6 100644
--- a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
+++ b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
@@ -509,6 +509,43 @@ mlir::LogicalResult hlfir::AnyOp::verify() {
   return verifyLogicalReductionOp<hlfir::AnyOp *>(this);
 }
 
+//===----------------------------------------------------------------------===//
+// CountOp
+//===----------------------------------------------------------------------===//
+
+mlir::LogicalResult hlfir::CountOp::verify() {
+  mlir::Operation *op = getOperation();
+
+  auto results = op->getResultTypes();
+  assert(results.size() == 1);
+  mlir::Value mask = getMask();
+  mlir::Value dim = getDim();
+
+  fir::SequenceType maskTy =
+      hlfir::getFortranElementOrSequenceType(mask.getType())
+          .cast<fir::SequenceType>();
+  llvm::ArrayRef<int64_t> maskShape = maskTy.getShape();
+
+  mlir::Type resultType = results[0];
+  if (auto resultExpr = mlir::dyn_cast_or_null<hlfir::ExprType>(resultType)) {
+    if (maskShape.size() > 1 && dim != nullptr) {
+      if (!resultExpr.isArray())
+        return emitOpError("result must be an array");
+
+      llvm::ArrayRef<int64_t> resultShape = resultExpr.getShape();
+      // Result has rank n-1
+      if (resultShape.size() != (maskShape.size() - 1))
+        return emitOpError("result rank must be one less than MASK");
+    } else {
+      return emitOpError("result must be of numerical scalar type");
+    }
+  } else if (!hlfir::isFortranScalarNumericalType(resultType)) {
+    return emitOpError("result must be of numerical scalar type");
+  }
+
+  return mlir::success();
+}
+
 //===----------------------------------------------------------------------===//
 // ConcatOp
 //===----------------------------------------------------------------------===//
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp
index b88b30e235a15..d3c604e249a9c 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/LowerHLFIRIntrinsics.cpp
@@ -262,6 +262,39 @@ using AnyOpConversion = HlfirReductionIntrinsicConversion<hlfir::AnyOp>;
 
 using AllOpConversion = HlfirReductionIntrinsicConversion<hlfir::AllOp>;
 
+struct CountOpConversion : public HlfirIntrinsicConversion<hlfir::CountOp> {
+  using HlfirIntrinsicConversion<hlfir::CountOp>::HlfirIntrinsicConversion;
+
+  mlir::LogicalResult
+  matchAndRewrite(hlfir::CountOp count,
+                  mlir::PatternRewriter &rewriter) const override {
+    fir::KindMapping kindMapping{rewriter.getContext()};
+    fir::FirOpBuilder builder{rewriter, kindMapping};
+    const mlir::Location &loc = count->getLoc();
+
+    mlir::Type i32 = builder.getI32Type();
+    mlir::Type logicalType = fir::LogicalType::get(
+        builder.getContext(), builder.getKindMap().defaultLogicalKind());
+
+    llvm::SmallVector<IntrinsicArgument, 3> inArgs;
+    inArgs.push_back({count.getMask(), logicalType});
+    inArgs.push_back({count.getDim(), i32});
+    inArgs.push_back({count.getKind(), i32});
+
+    auto *argLowering = fir::getIntrinsicArgumentLowering("count");
+    llvm::SmallVector<fir::ExtendedValue, 3> args =
+        lowerArguments(count, inArgs, rewriter, argLowering);
+
+    mlir::Type scalarResultType = hlfir::getFortranElementType(count.getType());
+
+    auto [resultExv, mustBeFreed] =
+        fir::genIntrinsicCall(builder, loc, "count", scalarResultType, args);
+
+    processReturnValue(count, resultExv, mustBeFreed, builder, rewriter);
+    return mlir::success();
+  }
+};
+
 struct MatmulOpConversion : public HlfirIntrinsicConversion<hlfir::MatmulOp> {
   using HlfirIntrinsicConversion<hlfir::MatmulOp>::HlfirIntrinsicConversion;
 
@@ -405,14 +438,14 @@ class LowerHLFIRIntrinsics
     patterns.insert<MatmulOpConversion, MatmulTransposeOpConversion,
                     AllOpConversion, AnyOpConversion, SumOpConversion,
                     ProductOpConversion, TransposeOpConversion,
-                    DotProductOpConversion>(context);
+                    CountOpConversion, DotProductOpConversion>(context);
     mlir::ConversionTarget target(*context);
     target.addLegalDialect<mlir::BuiltinDialect, mlir::arith::ArithDialect,
                            mlir::func::FuncDialect, fir::FIROpsDialect,
                            hlfir::hlfirDialect>();
     target.addIllegalOp<hlfir::MatmulOp, hlfir::MatmulTransposeOp, hlfir::SumOp,
                         hlfir::ProductOp, hlfir::TransposeOp, hlfir::AnyOp,
-                        hlfir::AllOp, hlfir::DotProductOp>();
+                        hlfir::AllOp, hlfir::DotProductOp, hlfir::CountOp>();
     target.markUnknownOpDynamicallyLegal(
         [](mlir::Operation *) { return true; });
     if (mlir::failed(
diff --git a/flang/test/HLFIR/count-lowering.fir b/flang/test/HLFIR/count-lowering.fir
new file mode 100644
index 0000000000000..0d9cc34a316eb
--- /dev/null
+++ b/flang/test/HLFIR/count-lowering.fir
@@ -0,0 +1,164 @@
+// Test hlfir.count operation lowering to fir runtime call
+// RUN: fir-opt %s -lower-hlfir-intrinsics | FileCheck %s
+
+func.func @_QPcount1(%arg0: !fir.box<!fir.array<?x!fir.logical<4>>> {fir.bindc_name = "a"}, %arg1: !fir.ref<i32> {fir.bindc_name = "s"}) {
+  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFcount1Ea"} : (!fir.box<!fir.array<?x!fir.logical<4>>>) -> (!fir.box<!fir.array<?x!fir.logical<4>>>, !fir.box<!fir.array<?x!fir.logical<4>>>)
+  %1:2 = hlfir.declare %arg1 {uniq_name = "_QFcount1Es"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2 = hlfir.count %0#0 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?x!fir.logical<4>>>) -> i32
+  hlfir.assign %2 to %1#0 : i32, !fir.ref<i32>
+  return
+}
+// CHECK-LABEL: func.func @_QPcount1(
+// CHECK:           %[[ARG0:.*]]: !fir.box<!fir.array<?x!fir.logical<4>>>
+// CHECK:           %[[ARG1:.*]]: !fir.ref<i32>
+// CHECK-DAG:     %[[MASK:.*]]:2 = hlfir.declare %[[ARG0]]
+// CHECK-DAG:     %[[RES:.*]]:2 = hlfir.declare %[[ARG1]]
+// CHECK-DAG:     %[[MASK_ARG:.*]] = fir.convert %[[MASK]]#1 : (!fir.box<!fir.array<?x!fir.logical<4>>>) -> !fir.box<none>
+// CHECK:         %[[RET_ARG:.*]] = fir.call @_FortranACount(%[[MASK_ARG]], %[[LOC_STR:.*]], %[[LOC_N:.*]], %[[C1:.*]]) : (!fir.box<none>, !fir.ref<i8>, i32, i32) -> i64
+// CHECK-NEXT:    %[[RET:.*]] = fir.convert %[[RET_ARG]] : (i64) -> i32
+// CHECK-NEXT:    hlfir.assign %[[RET]] to %[[RES]]#0 : i32, !fir.ref<i32>
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
+
+func.func @_QPcount2(%arg0: !fir.box<!fir.array<?x?x!fir.logical<4>>> {fir.bindc_name = "a"}, %arg1: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "s"}, %arg2: !fir.ref<i32> {fir.bindc_name = "d"}) {
+  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFcount2Ea"} : (!fir.box<!fir.array<?x?x!fir.logical<4>>>) -> (!fir.box<!fir.array<?x?x!fir.logical<4>>>, !fir.box<!fir.array<?x?x!fir.logical<4>>>)
+  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFcount2Ed"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFcount2Es"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %3 = fir.load %1#0 : !fir.ref<i32>
+  %4 = hlfir.count %0#0 dim %3 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, i32) -> !hlfir.expr<?xi32>
+  hlfir.assign %4 to %2#0 : !hlfir.expr<?xi32>, !fir.box<!fir.array<?xi32>>
+  hlfir.destroy %4 : !hlfir.expr<?xi32>
+  return
+}
+// CHECK-LABEL: func.func @_QPcount2(
+// CHECK:           %[[ARG0:.*]]: !fir.box<!fir.array<?x?x!fir.logical<4>>>
+// CHECK:           %[[ARG1:.*]]: !fir.box<!fir.array<?xi32>
+// CHECK:           %[[ARG2:.*]]: !fir.ref<i32>
+// CHECK-DAG:     %[[MASK:.*]]:2 = hlfir.declare %[[ARG0]]
+// CHECK-DAG:     %[[DIM_VAR:.*]]:2 = hlfir.declare %[[ARG2]]
+// CHECK-DAG:     %[[RES:.*]]:2 = hlfir.declare %[[ARG1]]
+
+// CHECK-DAG:     %[[RET_BOX:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xi32>>>
+// CHECK-DAG:     %[[RET_ADDR:.*]] = fir.zero_bits !fir.heap<!fir.array<?xi32>>
+// CHECK-DAG:     %[[C0:.*]] = arith.constant 0 : index
+// CHECK-DAG:     %[[RET_SHAPE:.*]] = fir.shape %[[C0]] : (index) -> !fir.shape<1>
+// CHECK-DAG:     %[[RET_EMBOX:.*]] = fir.embox %[[RET_ADDR]](%[[RET_SHAPE]])
+// CHECK-DAG:     fir.store %[[RET_EMBOX]] to %[[RET_BOX]]
+
+// CHECK-DAG:     %[[DIM:.*]] = fir.load %[[DIM_VAR]]#0 : !fir.ref<i32>
+// CHECK-DAG:     %[[RET_ARG:.*]] = fir.convert %[[RET_BOX]]
+// CHECK-DAG:     %[[MASK_ARG:.*]] = fir.convert %[[MASK]]#1
+
+// CHECK:         %[[NONE:.*]] = fir.call @_FortranACountDim(%[[RET_ARG]], %[[MASK_ARG]], %[[DIM]], %[[LOC_STR:.*]], %[[LOC_N:.*]]) : (!fir.ref<!fir.box<none>>, !fir.box<none>, i32, i32, !fir.ref<i8>, i32) -> none
+// CHECK:         %[[RET:.*]] = fir.load %[[RET_BOX]]
+// CHECK:         %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[RET]]
+// CHECK-NEXT:    %[[ADDR:.*]] = fir.box_addr %[[RET]]
+// CHECK-NEXT:    %[[SHIFT:.*]] = fir.shape_shift %[[BOX_DIMS]]#0, %[[BOX_DIMS]]#1
+// CHECK-NEXT:    %[[TMP:.*]]:2 = hlfir.declare %[[ADDR]](%[[SHIFT]]) {uniq_name = ".tmp.intrinsic_result"}
+// CHECK:         %[[TRUE:.*]] = arith.constant true
+// CHECK:         %[[EXPR:.*]] = hlfir.as_expr %[[TMP]]#0 move %[[TRUE]] : (!fir.box<!fir.array<?xi32>>, i1) -> !hlfir.expr<?xi32>
+// CHECK:         hlfir.assign %[[EXPR]] to %[[RES]]#0
+// CHECK:         hlfir.destroy %[[EXPR]]
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
+
+func.func @_QPcount3(%arg0: !fir.ref<!fir.array<2xi32>> {fir.bindc_name = "s"}) {
+  %0 = fir.address_of(@_QFcount3Ea) : !fir.ref<!fir.array<2x2x!fir.logical<4>>>
+  %c2 = arith.constant 2 : index
+  %c2_0 = arith.constant 2 : index
+  %1 = fir.shape %c2, %c2_0 : (index, index) -> !fir.shape<2>
+  %2:2 = hlfir.declare %0(%1) {uniq_name = "_QFcount3Ea"} : (!fir.ref<!fir.array<2x2x!fir.logical<4>>>, !fir.shape<2>) -> (!fir.ref<!fir.array<2x2x!fir.logical<4>>>, !fir.ref<!fir.array<2x2x!fir.logical<4>>>)
+  %c2_1 = arith.constant 2 : index
+  %3 = fir.shape %c2_1 : (index) -> !fir.shape<1>
+  %4:2 = hlfir.declare %arg0(%3) {uniq_name = "_QFcount3Es"} : (!fir.ref<!fir.array<2xi32>>, !fir.shape<1>) -> (!fir.ref<!fir.array<2xi32>>, !fir.ref<!fir.array<2xi32>>)
+  %c1_i32 = arith.constant 1 : i32
+  %5 = hlfir.count %2#0 dim %c1_i32 {fastmath = #arith.fastmath<contract>} : (!fir.ref<!fir.array<2x2x!fir.logical<4>>>, i32) -> !hlfir.expr<2xi32>
+  hlfir.assign %5 to %4#0 : !hlfir.expr<2xi32>, !fir.ref<!fir.array<2xi32>>
+  hlfir.destroy %5 : !hlfir.expr<2xi32>
+  return
+}
+// CHECK-LABEL:  func.func @_QPcount3(
+// CHECK:           %[[ARG0:.*]]: !fir.ref<!fir.array<2xi32>>
+// CHECK-DAG:     %[[RET_BOX:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xi32>>>
+// CHECK-DAG:     %[[RET_ADDR:.*]] = fir.zero_bits !fir.heap<!fir.array<?xi32>>
+// CHECK-DAG:     %[[C0:.*]] = arith.constant 0 : index
+// CHECK-DAG:     %[[RET_SHAPE:.*]] = fir.shape %[[C0]] : (index) -> !fir.shape<1>
+// CHECK-DAG:     %[[RET_EMBOX:.*]] = fir.embox %[[RET_ADDR]](%[[RET_SHAPE]])
+// CHECK-DAG:     fir.store %[[RET_EMBOX]] to %[[RET_BOX]]
+// CHECK-DAG:     %[[RES:.*]]:2 = hlfir.declare %[[ARG0]](%[[RES_SHAPE:.*]])
+
+// CHECK-DAG:     %[[MASK_ADDR:.*]] = fir.address_of
+// CHECK-DAG:     %[[MASK_VAR:.*]]:2 = hlfir.declare %[[MASK_ADDR]](%[[MASK_SHAPE:.*]])
+// CHECK-DAG:     %[[MASK_BOX:.*]] = fir.embox %[[MASK_VAR]]#1(%[[MASK_SHAPE:.*]])
+
+// CHECK-DAG:     %[[DIM:.*]] = arith.constant 1 : i32
+
+// CHECK-DAG:     %[[RET_ARG:.*]] = fir.convert %[[RET_BOX]]
+// CHECK-DAG:     %[[MASK_ARG:.*]] = fir.convert %[[MASK_BOX]] : (!fir.box<!fir.array<2x2x!fir.logical<4>>>) -> !fir.box<none>
+// CHECK:         %[[NONE:.*]] = fir.call @_FortranACountDim(%[[RET_ARG]], %[[MASK_ARG]], %[[DIM]], %[[LOC_STR:.*]], %[[LOC_N:.*]])
+// CHECK:         %[[RET:.*]] = fir.load %[[RET_BOX]]
+// CHECK:         %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[RET]]
+// CHECK-NEXT:    %[[ADDR:.*]] = fir.box_addr %[[RET]]
+// CHECK-NEXT:    %[[SHIFT:.*]] = fir.shape_shift %[[BOX_DIMS]]#0, %[[BOX_DIMS]]#1
+// CHECK-NEXT:    %[[TMP:.*]]:2 = hlfir.declare %[[ADDR]](%[[SHIFT]]) {uniq_name = ".tmp.intrinsic_result"}
+// CHECK:         %[[TRUE:.*]] = arith.constant true
+// CHECK:         %[[EXPR:.*]] = hlfir.as_expr %[[TMP]]#0 move %[[TRUE]] : (!fir.box<!fir.array<?xi32>>, i1) -> !hlfir.expr<?xi32>
+// CHECK:         hlfir.assign %[[EXPR]] to %[[RES]]
+// CHECK:         hlfir.destroy %[[EXPR]]
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
+
+func.func @_QPcount4(%arg0: !fir.box<!fir.array<?x?x!fir.logical<4>>> {fir.bindc_name = "a"}, %arg1: !fir.box<!fir.array<?xi32>> {fir.bindc_name = "s"}, %arg2: !fir.ref<i32> {fir.bindc_name = "d"}) {
+  %0:2 = hlfir.declare %arg0 {uniq_name = "_QFcount4Ea"} : (!fir.box<!fir.array<?x?x!fir.logical<4>>>) -> (!fir.box<!fir.array<?x?x!fir.logical<4>>>, !fir.box<!fir.array<?x?x!fir.logical<4>>>)
+  %1:2 = hlfir.declare %arg2 {uniq_name = "_QFcount4Ed"} : (!fir.ref<i32>) -> (!fir.ref<i32>, !fir.ref<i32>)
+  %2:2 = hlfir.declare %arg1 {uniq_name = "_QFcount4Es"} : (!fir.box<!fir.array<?xi32>>) -> (!fir.box<!fir.array<?xi32>>, !fir.box<!fir.array<?xi32>>)
+  %c8_i32 = arith.constant 8 : i32
+  %3 = fir.load %1#0 : !fir.ref<i32>
+  %4 = hlfir.count %0#0 dim %3 kind %c8_i32 {fastmath = #arith.fastmath<contract>} : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, i32, i32) -> !hlfir.expr<?xi64>
+  %5 = hlfir.shape_of %4 : (!hlfir.expr<?xi64>) -> !fir.shape<1>
+  %6 = hlfir.elemental %5 : (!fir.shape<1>) -> !hlfir.expr<?xi32> {
+  ^bb0(%arg3: index):
+    %7 = hlfir.apply %4, %arg3 : (!hlfir.expr<?xi64>, index) -> i64
+    %8 = fir.convert %7 : (i64) -> i32
+    hlfir.yield_element %8 : i32
+  }
+  hlfir.assign %6 to %2#0 : !hlfir.expr<?xi32>, !fir.box<!fir.array<?xi32>>
+  hlfir.destroy %6 : !hlfir.expr<?xi32>
+  hlfir.destroy %4 : !hlfir.expr<?xi64>
+  return
+}
+// CHECK-LABEL: func.func @_QPcount4(
+// CHECK:           %[[ARG0:.*]]: !fir.box<!fir.array<?x?x!fir.logical<4>>>
+// CHECK:           %[[ARG1:.*]]: !fir.box<!fir.array<?xi32>
+// CHECK:           %[[ARG2:.*]]: !fir.ref<i32>
+// CHECK-DAG:     %[[MASK:.*]]:2 = hlfir.declare %[[ARG0]]
+// CHECK-DAG:     %[[DIM_VAR:.*]]:2 = hlfir.declare %[[ARG2]]
+// CHECK-DAG:     %[[RES:.*]]:2 = hlfir.declare %[[ARG1]]
+
+// CHECK-DAG:     %[[RET_BOX:.*]] = fir.alloca !fir.box<!fir.heap<!fir.array<?xi64>>>
+// CHECK-DAG:     %[[RET_ADDR:.*]] = fir.zero_bits !fir.heap<!fir.array<?xi64>>
+// CHECK-DAG:     %[[C0:.*]] = arith.constant 0 : index
+// CHECK-DAG:     %[[RET_SHAPE:.*]] = fir.shape %[[C0]] : (index) -> !fir.shape<1>
+// CHECK-DAG:     %[[RET_EMBOX:.*]] = fir.embox %[[RET_ADDR]](%[[RET_SHAPE]])
+// CHECK-DAG:     fir.store %[[RET_EMBOX]] to %[[RET_BOX]]
+
+// CHECK-DAG:     %[[DIM:.*]] = fir.load %[[DIM_VAR]]#0 : !fir.ref<i32>
+// CHECK-DAG:     %[[KIND:.*]] = arith.constant 8 : i32
+// CHECK-DAG:     %[[RET_ARG:.*]] = fir.convert %[[RET_BOX]]
+// CHECK-DAG:     %[[MASK_ARG:.*]] = fir.convert %[[MASK]]#1
+
+// CHECK:         %[[NONE:.*]] = fir.call @_FortranACountDim(%[[RET_ARG]], %[[MASK_ARG]], %[[DIM]], %[[KIND]], %[[LOC_STR:.*]], %[[LOC_N:.*]]) : (!fir.ref<!fir.box<none>>, !fir.box<none>, i32, i32, !fir.ref<i8>, i32) -> none
+// CHECK:         %[[RET:.*]] = fir.load %[[RET_BOX]]
+// CHECK:         %[[BOX_DIMS:.*]]:3 = fir.box_dims %[[RET]]
+// CHECK-NEXT:    %[[ADDR:.*]] = fir.box_addr %[[RET]]
+// CHECK-NEXT:    %[[SHIFT:.*]] = fir.shape_shift %[[BOX_DIMS]]#0, %[[BOX_DIMS]]#1
+// CHECK-NEXT:    %[[TMP:.*]]:2 = hlfir.declare %[[ADDR]](%[[SHIFT]]) {uniq_name = ".tmp.intrinsic_result"}
+// CHECK:         %[[TRUE:.*]] = arith.constant true
+// CHECK:         %[[EXPR:.*]] = hlfir.as_expr %[[TMP]]#0 move %[[TRUE]] : (!fir.box<!fir.array<?xi64>>, i1) -> !hlfir.expr<?xi64>
+// CHECK-NEXT:    %[[OUT_SHAPE:.*]] = hlfir.shape_of %[[EXPR]]
+// CHECK-NEXT:    %[[OUT:.*]] = hlfir.elemental %[[OUT_SHAPE]] : (!fir.shape<1>) -> !hlfir.expr<?xi32>
+// CHECK-DAG:       hlfir.assign %[[OUT]] to %[[RES]]#0
+// CHECK-NEXT:      hlfir.destroy %[[OUT]] : !hlfir.expr<?xi32>
+// CHECK-NEXT:      hlfir.destroy %[[EXPR]] : !hlfir.expr<?xi64>
+// CHECK-NEXT:    return
+// CHECK-NEXT:  }
diff --git a/flang/test/HLFIR/count.fir b/flang/test/HLFIR/count.fir
new file mode 100644
index 0000000000000..c25f9b94124af
--- /dev/null
+++ b/flang/test/HLFIR/count.fir
@@ -0,0 +1,83 @@
+// Test hlfir.count operation parse, verify (no errors), and unparse
+
+// RUN: fir-opt %s | fir-opt | FileCheck %s
+
+// mask is an expression of known shape
+func.func @count0(%arg0: !hlfir.expr<2x!fir.logical<4>>) {
+  %count = hlfir.count %arg0 : (!hlfir.expr<2x!fir.logical<4>>) -> i32
+  return
+}
+// CHECK:      func.func @count0(%[[ARRAY:.*]]: !hlfir.expr<2x!fir.logical<4>>) {
+// CHECK-NEXT:   %[[COUNT:.*]] = hlfir.count %[[ARRAY]] : (!hlfir.expr<2x!fir.logical<4>>) -> i32
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
+
+// mask is an expression of assumed shape
+func.func @count1(%arg0: !hlfir.expr<?x!fir.logical<4>>) {
+  %count = hlfir.count %arg0 : (!hlfir.expr<?x!fir.logical<4>>) -> i32
+  return
+}
+// CHECK:      func.func @count1(%[[ARRAY:.*]]: !hlfir.expr<?x!fir.logical<4>>) {
+// CHECK-NEXT:   %[[COUNT:.*]] = hlfir.count %[[ARRAY]] : (!hlfir.expr<?x!fir.logical<4>>) -> i32
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
+
+// mask is a boxed array
+func.func @count2(%arg0: !fir.box<!fir.array<2x!fir.logical<4>>>) {
+  %count = hlfir.count %arg0 : (!fir.box<!fir.array<2x!fir.logical<4>>>) -> i32
+  return
+}
+// CHECK:      func.func @count2(%[[ARRAY:.*]]: !fir.box<!fir.array<2x!fir.logical<4>>>) {
+// CHECK-NEXT:   %[[COUNT:.*]] = hlfir.count %[[ARRAY]] : (!fir.box<!fir.array<2x!fir.logical<4>>>) -> i32
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
+
+// mask is an assumed shape boxed array
+func.func @count3(%arg0: !fir.box<!fir.array<?x!fir.logical<4>>>) {
+  %count = hlfir.count %arg0 : (!fir.box<!fir.array<?x!fir.logical<4>>>) -> i32
+  return
+}
+// CHECK:      func.func @count3(%[[ARRAY:.*]]: !fir.box<!fir.array<?x!fir.logical<4>>>) {
+// CHECK-NEXT:   %[[COUNT:.*]] = hlfir.count %[[ARRAY]] : (!fir.box<!fir.array<?x!fir.logical<4>>>) -> i32
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
+
+// mask is a 2-dimensional array
+func.func @count4(%arg0: !fir.box<!fir.array<?x?x!fir.logical<4>>>){
+  %count = hlfir.count %arg0 : (!fir.box<!fir.array<?x?x!fir.logical<4>>>) -> i32
+  return
+}
+// CHECK:      func.func @count4(%[[ARRAY:.*]]: !fir.box<!fir.array<?x?x!fir.logical<4>>>) {
+// CHECK-NEXT:   %[[COUNT:.*]] = hlfir.count %[[ARRAY]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>) -> i32
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
+
+// mask and dim argument
+func.func @count5(%arg0: !fir.box<!fir.array<?x!fir.logical<4>>>, %arg1: i32) {
+  %count = hlfir.count %arg0 dim %arg1 : (!fir.box<!fir.array<?x!fir.logical<4>>>, i32) -> i32
+  return
+}
+// CHECK:      func.func @count5(%[[ARRAY:.*]]: !fir.box<!fir.array<?x!fir.logical<4>>>, %[[DIM:.*]]: i32) {
+// CHECK-NEXT:   %[[COUNT:.*]] = hlfir.count %[[ARRAY]] dim %[[DIM]] : (!fir.box<!fir.array<?x!fir.logical<4>>>, i32) -> i32
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
+
+// mask is a 2 dimensional array with dim
+func.func @count6(%arg0: !fir.box<!fir.array<?x?x!fir.logical<4>>>, %arg1: i32) {
+  %count = hlfir.count %arg0 dim %arg1 : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, i32) -> i32
+  return
+}
+// CHECK:      func.func @count6(%[[ARRAY:.*]]: !fir.box<!fir.array<?x?x!fir.logical<4>>>, %[[DIM:.*]]: i32) {
+// CHECK-NEXT:   %[[ANY:.*]] = hlfir.count %[[ARRAY]] dim %[[DIM]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, i32) -> i32
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
+
+// mask is of a different logical type
+func.func @count7(%arg0: !fir.box<!fir.array<?x?x!fir.logical<8>>>, %arg1: i32) {
+  %count = hlfir.count %arg0 dim %arg1 : (!fir.box<!fir.array<?x?x!fir.logical<8>>>, i32) -> i32
+  return
+}
+// CHECK:      func.func @count7(%[[ARRAY:.*]]: !fir.box<!fir.array<?x?x!fir.logical<8>>>, %[[DIM:.*]]: i32) {
+// CHECK-NEXT:   %[[ANY:.*]] = hlfir.count %[[ARRAY]] dim %[[DIM]] : (!fir.box<!fir.array<?x?x!fir.logical<8>>>, i32) -> i32
+// CHECK-NEXT:   return
+// CHECK-NEXT: }
\ No newline at end of file
diff --git a/flang/test/HLFIR/invalid.fir b/flang/test/HLFIR/invalid.fir
index 01bccdf80428b..6db1b79e0818f 100644
--- a/flang/test/HLFIR/invalid.fir
+++ b/flang/test/HLFIR/invalid.fir
@@ -368,6 +368,30 @@ func.func @bad_all6(%arg0: !hlfir.expr<?x!fir.logical<4>>) {
   %0 = hlfir.all %arg0 : (!hlfir.expr<?x!fir.logical<4>>) -> !hlfir.expr<!fir.logical<4>>
 }
 
+// -----
+func.func @bad_count1(%arg0: !hlfir.expr<?x?x!fir.logical<4>>, %arg1: i32) {
+  // expected-error@+1 {{'hlfir.count' op result must be an array}}
+  %0 = hlfir.count %arg0 dim %arg1 : (!hlfir.expr<?x?x!fir.logical<4>>, i32) -> !hlfir.expr<i32>
+}
+
+// -----
+func.func @bad_count2(%arg0: !hlfir.expr<?x?x!fir.logical<4>>, %arg1: i32){
+  // expected-error@+1 {{'hlfir.count' op result rank must be one less than MASK}}
+  %0 = hlfir.count %arg0 dim %arg1 : (!hlfir.expr<?x?x!fir.logical<4>>, i32) -> !hlfir.expr<?x?x!fir.logical<4>>
+}
+
+// -----
+func.func @bad_count3(%arg0: !hlfir.expr<?x!fir.logical<4>>, %arg1: i32) {
+  // expected-error@+1 {{'hlfir.count' op result must be of numerical scalar type}}
+  %0 = hlfir.count %arg0 dim %arg1 : (!hlfir.expr<?x!fir.logical<4>>, i32) -> !hlfir.expr<i32>
+}
+
+// -----
+func.func @bad_count4(%arg0: !hlfir.expr<?x!fir.logical<4>>, %arg1: i32) {
+  // expected-error@+1 {{'hlfir.count' op result must be of numerical scalar type}}
+  %0 = hlfir.count %arg0 dim %arg1 : (!hlfir.expr<?x!fir.logical<4>>, i32) -> !fir.logical<4>
+}
+
 // -----
 func.func @bad_product1(%arg0: !hlfir.expr<?xi32>, %arg1: i32, %arg2: !fir.box<!fir.logical<4>>) {
   // expected-error@+1 {{'hlfir.product' op result must have the same element type as ARRAY argument}}
diff --git a/flang/test/Lower/HLFIR/count.f90 b/flang/test/Lower/HLFIR/count.f90
new file mode 100644
index 0000000000000..25c74841514e9
--- /dev/null
+++ b/flang/test/Lower/HLFIR/count.f90
@@ -0,0 +1,82 @@
+! Test lowering of COUNT intrinsic to HLFIR
+! RUN: bbc -emit-hlfir -o - %s 2>&1 | FileCheck %s
+
+! simple 1 argument COUNT
+subroutine count1(a, s)
+  logical :: a(:)
+  integer :: s
+  s = COUNT(a)
+end subroutine
+! CHECK-LABEL: func.func @_QPcount1(
+! CHECK:           %[[ARG0:.*]]: !fir.box<!fir.array<?x!fir.logical<4>>>
+! CHECK:           %[[ARG1:.*]]: !fir.ref<i32>
+! CHECK-DAG:     %[[MASK:.*]]:2 = hlfir.declare %[[ARG0]]
+! CHECK-DAG:     %[[OUT:.*]]:2 = hlfir.declare %[[ARG1]]
+! CHECK-NEXT:    %[[EXPR:.*]] = hlfir.count %[[MASK]]#0 : (!fir.box<!fir.array<?x!fir.logical<4>>>) -> i32
+! CHECK-NEXT:    hlfir.assign %[[EXPR]] to %[[OUT]]#0 : i32, !fir.ref<i32>
+! CHECK-NEXT:    return
+! CHECK-NEXT:  }
+
+! count with by-ref DIM argument
+subroutine count2(a, s, d)
+  logical :: a(:,:)
+  integer :: s(:), d
+  s = COUNT(a, d)
+end subroutine
+! CHECK-LABEL: func.func @_QPcount2(
+! CHECK:           %[[ARG0:.*]]: !fir.box<!fir.array<?x?x!fir.logical<4>>>
+! CHECK:           %[[ARG1:.*]]: !fir.box<!fir.array<?xi32>>
+! CHECK:           %[[ARG2:.*]]: !fir.ref<i32>
+! CHECK-DAG:     %[[MASK:.*]]:2 = hlfir.declare %[[ARG0]]
+! CHECK-DAG:     %[[DIM_REF:.*]]:2 = hlfir.declare %[[ARG2]]
+! CHECK-DAG:     %[[OUT:.*]]:2 = hlfir.declare %[[ARG1]]
+! CHECK-DAG:     %[[DIM:.*]] = fir.load %[[DIM_REF]]#0 : !fir.ref<i32>
+! CHECK-DAG:     %[[EXPR:.*]] = hlfir.count %[[MASK]]#0 dim %[[DIM]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, i32) -> !hlfir.expr<?xi32>
+! CHECK-NEXT:    hlfir.assign %[[EXPR]] to %[[OUT]]#0 : !hlfir.expr<?xi32>, !fir.box<!fir.array<?xi32>>
+! CHECK-NEXT:    hlfir.destroy %[[EXPR]] : !hlfir.expr<?xi32>
+! CHECK-NEXT:    return
+! CHECK-NEXT:  }
+
+! count with DIM argument by-val, mask isn't boxed
+subroutine count3(s)
+  integer :: s(2)
+  logical :: a(2,2) = reshape((/.true.,.false.,.true.,.false./), shape(a))
+  s = COUNT(a, 1)
+end subroutine
+! CHECK-LABEL: func.func @_QPcount3(
+! CHECK:           %[[ARG0:.*]]: !fir.ref<!fir.array<2xi32>>
+! CHECK-DAG:     %[[ADDR:.*]] = fir.address_of{{.*}} : !fir.ref<!fir.array<2x2x!fir.logical<4>>>
+! CHECK-DAG:     %[[MASK_SHAPE:.*]] = fir.shape {{.*}} -> !fir.shape<2>
+! CHECK-DAG:     %[[MASK:.*]]:2 = hlfir.declare %[[ADDR]](%[[MASK_SHAPE]])
+! CHECK-DAG:     %[[OUT_SHAPE:.*]] = fir.shape {{.*}} -> !fir.shape<1>
+! CHECK-DAG:     %[[OUT:.*]]:2 = hlfir.declare %[[ARG0]](%[[OUT_SHAPE]])
+! CHECK-DAG:     %[[C1:.*]] = arith.constant 1 : i32
+! CHECK-DAG:     %[[EXPR:.*]] = hlfir.count %[[MASK]]#0 dim %[[C1]] : (!fir.ref<!fir.array<2x2x!fir.logical<4>>>, i32) -> !hlfir.expr<2xi32>
+! CHECK-DAG:     hlfir.assign %[[EXPR]] to %[[OUT]]#0 : !hlfir.expr<2xi32>, !fir.ref<!fir.array<2xi32>>
+! CHECK-NEXT:    hlfir.destroy %[[EXPR]] : !hlfir.expr<2xi32>
+! CHECK-NEXT:    return
+! CHECK-NEXT:  }
+
+! count with dim and kind arguments
+subroutine count4(a, s, d)
+  logical :: a(:,:)
+  integer :: s(:), d
+  s = COUNT(a, d, 8)
+end subroutine
+! CHECK-LABEL: func.func @_QPcount4(
+! CHECK:           %[[ARG0:.*]]: !fir.box<!fir.array<?x?x!fir.logical<4>>>
+! CHECK:           %[[ARG1:.*]]: !fir.box<!fir.array<?xi32>>
+! CHECK:           %[[ARG2:.*]]: !fir.ref<i32>
+! CHECK-DAG:     %[[MASK:.*]]:2 = hlfir.declare %[[ARG0]]
+! CHECK-DAG:     %[[DIM_REF:.*]]:2 = hlfir.declare %[[ARG2]]
+! CHECK-DAG:     %[[OUT:.*]]:2 = hlfir.declare %[[ARG1]]
+! CHECK-DAG:     %[[C8:.*]] = arith.constant 8 : i32
+! CHECK-DAG:     %[[DIM:.*]] = fir.load %[[DIM_REF]]#0 : !fir.ref<i32>
+! CHECK-DAG:     %[[EXPR:.*]] = hlfir.count %[[MASK]]#0 dim %[[DIM]] kind %[[C8]] : (!fir.box<!fir.array<?x?x!fir.logical<4>>>, i32, i32) -> !hlfir.expr<?xi64>
+! CHECK-DAG:     %[[RES_SHAPE:.*]] = hlfir.shape_of %[[EXPR]]
+! CHECK-DAG:     %[[RES:.*]] = hlfir.elemental %[[RES_SHAPE]] : (!fir.shape<1>) -> !hlfir.expr<?xi32>
+! CHECK-DAG:     hlfir.assign %[[RES]] to %[[OUT]]#0
+! CHECK-NEXT:    hlfir.destroy %[[RES]] : !hlfir.expr<?xi32>
+! CHECK-NEXT:    hlfir.destroy %[[EXPR]] : !hlfir.expr<?xi64>
+! CHECK-NEXT:    return
+! CHECK-NEXT:  }

From 4511d3c33cba90d5d53a1fd6eed89ef9a3f696e1 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Mon, 19 Jun 2023 11:29:46 +0200
Subject: [PATCH 004/130] [InstSimplify] Add test for icmp of uadd.sat/usub.sat
 (NFC)

---
 .../InstSimplify/saturating-add-sub.ll        | 154 ++++++++++++++++++
 1 file changed, 154 insertions(+)

diff --git a/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll b/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll
index b323ce0749f84..616a2060e5339 100644
--- a/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll
+++ b/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll
@@ -794,3 +794,157 @@ define i1 @ssub_icmp_op1_smin(i8 %a) {
   %c = icmp sge i8 %b, 0
   ret i1 %c
 }
+
+define i1 @uadd_uge(i8 %x, i8 %y) {
+; CHECK-LABEL: @uadd_uge(
+; CHECK-NEXT:    [[SAT:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i8 [[SAT]], [[X]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sat = call i8 @llvm.uadd.sat.i8(i8 %x, i8 %y)
+  %cmp = icmp uge i8 %sat, %x
+  ret i1 %cmp
+}
+
+define i1 @uadd_uge_rhs(i8 %x, i8 %y) {
+; CHECK-LABEL: @uadd_uge_rhs(
+; CHECK-NEXT:    [[SAT:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i8 [[SAT]], [[Y]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sat = call i8 @llvm.uadd.sat.i8(i8 %x, i8 %y)
+  %cmp = icmp uge i8 %sat, %y
+  ret i1 %cmp
+}
+
+define i1 @uadd_uge_commuted(i8 %x, i8 %y) {
+; CHECK-LABEL: @uadd_uge_commuted(
+; CHECK-NEXT:    [[SAT:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i8 [[X]], [[SAT]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sat = call i8 @llvm.uadd.sat.i8(i8 %x, i8 %y)
+  %cmp = icmp ule i8 %x, %sat
+  ret i1 %cmp
+}
+
+define i1 @uadd_ult(i8 %x, i8 %y) {
+; CHECK-LABEL: @uadd_ult(
+; CHECK-NEXT:    [[SAT:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[SAT]], [[X]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sat = call i8 @llvm.uadd.sat.i8(i8 %x, i8 %y)
+  %cmp = icmp ult i8 %sat, %x
+  ret i1 %cmp
+}
+
+define <2 x i1> @uadd_uge_vec(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: @uadd_uge_vec(
+; CHECK-NEXT:    [[SAT:%.*]] = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge <2 x i8> [[SAT]], [[X]]
+; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+;
+  %sat = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %x, <2 x i8> %y)
+  %cmp = icmp uge <2 x i8> %sat, %x
+  ret <2 x i1> %cmp
+}
+
+define i1 @uadd_ugt_fail(i8 %x, i8 %y) {
+; CHECK-LABEL: @uadd_ugt_fail(
+; CHECK-NEXT:    [[SAT:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[SAT]], [[X]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sat = call i8 @llvm.uadd.sat.i8(i8 %x, i8 %y)
+  %cmp = icmp ugt i8 %sat, %x
+  ret i1 %cmp
+}
+
+define i1 @uadd_ule_fail(i8 %x, i8 %y) {
+; CHECK-LABEL: @uadd_ule_fail(
+; CHECK-NEXT:    [[SAT:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i8 [[SAT]], [[X]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sat = call i8 @llvm.uadd.sat.i8(i8 %x, i8 %y)
+  %cmp = icmp ule i8 %sat, %x
+  ret i1 %cmp
+}
+
+define i1 @uadd_uge_unrelated_op_fail(i8 %x, i8 %y, i8 %z) {
+; CHECK-LABEL: @uadd_uge_unrelated_op_fail(
+; CHECK-NEXT:    [[SAT:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i8 [[SAT]], [[Z:%.*]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sat = call i8 @llvm.uadd.sat.i8(i8 %x, i8 %y)
+  %cmp = icmp uge i8 %sat, %z
+  ret i1 %cmp
+}
+
+define i1 @usub_ule(i8 %x, i8 %y) {
+; CHECK-LABEL: @usub_ule(
+; CHECK-NEXT:    [[SAT:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i8 [[SAT]], [[X]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sat = call i8 @llvm.usub.sat.i8(i8 %x, i8 %y)
+  %cmp = icmp ule i8 %sat, %x
+  ret i1 %cmp
+}
+
+define i1 @usub_ule_rhs_fail(i8 %x, i8 %y) {
+; CHECK-LABEL: @usub_ule_rhs_fail(
+; CHECK-NEXT:    [[SAT:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i8 [[SAT]], [[Y]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sat = call i8 @llvm.usub.sat.i8(i8 %x, i8 %y)
+  %cmp = icmp ule i8 %sat, %y
+  ret i1 %cmp
+}
+
+define i1 @usub_ule_commuted(i8 %x, i8 %y) {
+; CHECK-LABEL: @usub_ule_commuted(
+; CHECK-NEXT:    [[SAT:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i8 [[X]], [[SAT]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sat = call i8 @llvm.usub.sat.i8(i8 %x, i8 %y)
+  %cmp = icmp uge i8 %x, %sat
+  ret i1 %cmp
+}
+
+define i1 @usub_ugt(i8 %x, i8 %y) {
+; CHECK-LABEL: @usub_ugt(
+; CHECK-NEXT:    [[SAT:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[SAT]], [[X]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sat = call i8 @llvm.usub.sat.i8(i8 %x, i8 %y)
+  %cmp = icmp ugt i8 %sat, %x
+  ret i1 %cmp
+}
+
+define i1 @usub_ult_fail(i8 %x, i8 %y) {
+; CHECK-LABEL: @usub_ult_fail(
+; CHECK-NEXT:    [[SAT:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[SAT]], [[X]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sat = call i8 @llvm.usub.sat.i8(i8 %x, i8 %y)
+  %cmp = icmp ult i8 %sat, %x
+  ret i1 %cmp
+}
+
+define i1 @usub_uge_fail(i8 %x, i8 %y) {
+; CHECK-LABEL: @usub_uge_fail(
+; CHECK-NEXT:    [[SAT:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
+; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i8 [[SAT]], [[X]]
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %sat = call i8 @llvm.usub.sat.i8(i8 %x, i8 %y)
+  %cmp = icmp uge i8 %sat, %x
+  ret i1 %cmp
+}

From f6a8775c5d9fa6cd49f404da70982f2e7b91c88c Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Mon, 19 Jun 2023 11:42:21 +0200
Subject: [PATCH 005/130] [InstSimplify] Fold icmp of uadd.sat/usub.sat
 (PR63381)

Fold uadd.sat(X, Y) uge X and usub.sat(X, Y) ule X to true.

Proof: https://alive2.llvm.org/ce/z/596m9X

Fixes https://github.com/llvm/llvm-project/issues/63381.
---
 llvm/lib/Analysis/InstructionSimplify.cpp     | 36 +++++++++++++++++++
 .../InstSimplify/saturating-add-sub.ll        | 32 +++++------------
 2 files changed, 44 insertions(+), 24 deletions(-)

diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 616873f87f313..48ffa83d45cee 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -3682,6 +3682,36 @@ static Value *simplifyICmpWithDominatingAssume(CmpInst::Predicate Predicate,
   return nullptr;
 }
 
+static Value *simplifyICmpWithIntrinsicOnLHS(CmpInst::Predicate Pred,
+                                             Value *LHS, Value *RHS) {
+  auto *II = dyn_cast<IntrinsicInst>(LHS);
+  if (!II)
+    return nullptr;
+
+  switch (II->getIntrinsicID()) {
+  case Intrinsic::uadd_sat:
+    // uadd.sat(X, Y) uge X, uadd.sat(X, Y) uge Y
+    if (II->getArgOperand(0) == RHS || II->getArgOperand(1) == RHS) {
+      if (Pred == ICmpInst::ICMP_UGE)
+        return ConstantInt::getTrue(getCompareTy(II));
+      if (Pred == ICmpInst::ICMP_ULT)
+        return ConstantInt::getFalse(getCompareTy(II));
+    }
+    return nullptr;
+  case Intrinsic::usub_sat:
+    // usub.sat(X, Y) ule X
+    if (II->getArgOperand(0) == RHS) {
+      if (Pred == ICmpInst::ICMP_ULE)
+        return ConstantInt::getTrue(getCompareTy(II));
+      if (Pred == ICmpInst::ICMP_UGT)
+        return ConstantInt::getFalse(getCompareTy(II));
+    }
+    return nullptr;
+  default:
+    return nullptr;
+  }
+}
+
 /// Given operands for an ICmpInst, see if we can fold the result.
 /// If not, this returns null.
 static Value *simplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
@@ -3946,6 +3976,12 @@ static Value *simplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
   if (Value *V = simplifyICmpWithMinMax(Pred, LHS, RHS, Q, MaxRecurse))
     return V;
 
+  if (Value *V = simplifyICmpWithIntrinsicOnLHS(Pred, LHS, RHS))
+    return V;
+  if (Value *V = simplifyICmpWithIntrinsicOnLHS(
+          ICmpInst::getSwappedPredicate(Pred), RHS, LHS))
+    return V;
+
   if (Value *V = simplifyICmpWithDominatingAssume(Pred, LHS, RHS, Q))
     return V;
 
diff --git a/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll b/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll
index 616a2060e5339..6fb12612f2f72 100644
--- a/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll
+++ b/llvm/test/Transforms/InstSimplify/saturating-add-sub.ll
@@ -797,9 +797,7 @@ define i1 @ssub_icmp_op1_smin(i8 %a) {
 
 define i1 @uadd_uge(i8 %x, i8 %y) {
 ; CHECK-LABEL: @uadd_uge(
-; CHECK-NEXT:    [[SAT:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
-; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i8 [[SAT]], [[X]]
-; CHECK-NEXT:    ret i1 [[CMP]]
+; CHECK-NEXT:    ret i1 true
 ;
   %sat = call i8 @llvm.uadd.sat.i8(i8 %x, i8 %y)
   %cmp = icmp uge i8 %sat, %x
@@ -808,9 +806,7 @@ define i1 @uadd_uge(i8 %x, i8 %y) {
 
 define i1 @uadd_uge_rhs(i8 %x, i8 %y) {
 ; CHECK-LABEL: @uadd_uge_rhs(
-; CHECK-NEXT:    [[SAT:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
-; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i8 [[SAT]], [[Y]]
-; CHECK-NEXT:    ret i1 [[CMP]]
+; CHECK-NEXT:    ret i1 true
 ;
   %sat = call i8 @llvm.uadd.sat.i8(i8 %x, i8 %y)
   %cmp = icmp uge i8 %sat, %y
@@ -819,9 +815,7 @@ define i1 @uadd_uge_rhs(i8 %x, i8 %y) {
 
 define i1 @uadd_uge_commuted(i8 %x, i8 %y) {
 ; CHECK-LABEL: @uadd_uge_commuted(
-; CHECK-NEXT:    [[SAT:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i8 [[X]], [[SAT]]
-; CHECK-NEXT:    ret i1 [[CMP]]
+; CHECK-NEXT:    ret i1 true
 ;
   %sat = call i8 @llvm.uadd.sat.i8(i8 %x, i8 %y)
   %cmp = icmp ule i8 %x, %sat
@@ -830,9 +824,7 @@ define i1 @uadd_uge_commuted(i8 %x, i8 %y) {
 
 define i1 @uadd_ult(i8 %x, i8 %y) {
 ; CHECK-LABEL: @uadd_ult(
-; CHECK-NEXT:    [[SAT:%.*]] = call i8 @llvm.uadd.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i8 [[SAT]], [[X]]
-; CHECK-NEXT:    ret i1 [[CMP]]
+; CHECK-NEXT:    ret i1 false
 ;
   %sat = call i8 @llvm.uadd.sat.i8(i8 %x, i8 %y)
   %cmp = icmp ult i8 %sat, %x
@@ -841,9 +833,7 @@ define i1 @uadd_ult(i8 %x, i8 %y) {
 
 define <2 x i1> @uadd_uge_vec(<2 x i8> %x, <2 x i8> %y) {
 ; CHECK-LABEL: @uadd_uge_vec(
-; CHECK-NEXT:    [[SAT:%.*]] = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> [[X:%.*]], <2 x i8> [[Y:%.*]])
-; CHECK-NEXT:    [[CMP:%.*]] = icmp uge <2 x i8> [[SAT]], [[X]]
-; CHECK-NEXT:    ret <2 x i1> [[CMP]]
+; CHECK-NEXT:    ret <2 x i1> <i1 true, i1 true>
 ;
   %sat = call <2 x i8> @llvm.uadd.sat.v2i8(<2 x i8> %x, <2 x i8> %y)
   %cmp = icmp uge <2 x i8> %sat, %x
@@ -885,9 +875,7 @@ define i1 @uadd_uge_unrelated_op_fail(i8 %x, i8 %y, i8 %z) {
 
 define i1 @usub_ule(i8 %x, i8 %y) {
 ; CHECK-LABEL: @usub_ule(
-; CHECK-NEXT:    [[SAT:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ule i8 [[SAT]], [[X]]
-; CHECK-NEXT:    ret i1 [[CMP]]
+; CHECK-NEXT:    ret i1 true
 ;
   %sat = call i8 @llvm.usub.sat.i8(i8 %x, i8 %y)
   %cmp = icmp ule i8 %sat, %x
@@ -907,9 +895,7 @@ define i1 @usub_ule_rhs_fail(i8 %x, i8 %y) {
 
 define i1 @usub_ule_commuted(i8 %x, i8 %y) {
 ; CHECK-LABEL: @usub_ule_commuted(
-; CHECK-NEXT:    [[SAT:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
-; CHECK-NEXT:    [[CMP:%.*]] = icmp uge i8 [[X]], [[SAT]]
-; CHECK-NEXT:    ret i1 [[CMP]]
+; CHECK-NEXT:    ret i1 true
 ;
   %sat = call i8 @llvm.usub.sat.i8(i8 %x, i8 %y)
   %cmp = icmp uge i8 %x, %sat
@@ -918,9 +904,7 @@ define i1 @usub_ule_commuted(i8 %x, i8 %y) {
 
 define i1 @usub_ugt(i8 %x, i8 %y) {
 ; CHECK-LABEL: @usub_ugt(
-; CHECK-NEXT:    [[SAT:%.*]] = call i8 @llvm.usub.sat.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ugt i8 [[SAT]], [[X]]
-; CHECK-NEXT:    ret i1 [[CMP]]
+; CHECK-NEXT:    ret i1 false
 ;
   %sat = call i8 @llvm.usub.sat.i8(i8 %x, i8 %y)
   %cmp = icmp ugt i8 %sat, %x

From 5400257ded86b65e5b55cbb138e0438b96c9bebf Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas@arm.com>
Date: Mon, 19 Jun 2023 10:45:42 +0100
Subject: [PATCH 006/130] [FuncSpec] Add Freeze and CallBase to the
 InstCostVisitor.

Allows constant folding of such instructions when estimating user bonus.

Differential Revision: https://reviews.llvm.org/D153036
---
 .../Transforms/IPO/FunctionSpecialization.h   |  2 ++
 .../Transforms/IPO/FunctionSpecialization.cpp | 28 +++++++++++++++++++
 .../IPO/FunctionSpecializationTest.cpp        | 23 ++++++++++++---
 3 files changed, 49 insertions(+), 4 deletions(-)

diff --git a/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h b/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h
index d7a2e9f871a28..e24d72fae10fb 100644
--- a/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h
+++ b/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h
@@ -143,6 +143,8 @@ class InstCostVisitor : public InstVisitor<InstCostVisitor, Constant *> {
   Cost estimateBranchInst(BranchInst &I);
 
   Constant *visitInstruction(Instruction &I) { return nullptr; }
+  Constant *visitFreezeInst(FreezeInst &I);
+  Constant *visitCallBase(CallBase &I);
   Constant *visitLoadInst(LoadInst &I);
   Constant *visitGetElementPtrInst(GetElementPtrInst &I);
   Constant *visitSelectInst(SelectInst &I);
diff --git a/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp b/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
index a970253d9b1c8..1792b24936bf8 100644
--- a/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
@@ -222,6 +222,34 @@ Cost InstCostVisitor::estimateBranchInst(BranchInst &I) {
   return estimateBasicBlocks(WorkList, KnownConstants, Solver, BFI, TTI);
 }
 
+Constant *InstCostVisitor::visitFreezeInst(FreezeInst &I) {
+  if (isGuaranteedNotToBeUndefOrPoison(LastVisited->second))
+    return LastVisited->second;
+  return nullptr;
+}
+
+Constant *InstCostVisitor::visitCallBase(CallBase &I) {
+  Function *F = I.getCalledFunction();
+  if (!F || !canConstantFoldCallTo(&I, F))
+    return nullptr;
+
+  SmallVector<Constant *, 8> Operands;
+  Operands.reserve(I.getNumOperands());
+
+  for (unsigned Idx = 0, E = I.getNumOperands() - 1; Idx != E; ++Idx) {
+    Value *V = I.getOperand(Idx);
+    auto *C = dyn_cast<Constant>(V);
+    if (!C)
+      C = findConstantFor(V, KnownConstants);
+    if (!C)
+      return nullptr;
+    Operands.push_back(C);
+  }
+
+  auto Ops = ArrayRef(Operands.begin(), Operands.end());
+  return ConstantFoldCall(&I, F, Ops);
+}
+
 Constant *InstCostVisitor::visitLoadInst(LoadInst &I) {
   if (isa<ConstantPointerNull>(LastVisited->second))
     return nullptr;
diff --git a/llvm/unittests/Transforms/IPO/FunctionSpecializationTest.cpp b/llvm/unittests/Transforms/IPO/FunctionSpecializationTest.cpp
index df44a16987721..bf65b3402f31c 100644
--- a/llvm/unittests/Transforms/IPO/FunctionSpecializationTest.cpp
+++ b/llvm/unittests/Transforms/IPO/FunctionSpecializationTest.cpp
@@ -227,13 +227,21 @@ TEST_F(FunctionSpecializationTest, Misc) {
   const char *ModuleString = R"(
     @g = constant [2 x i32] zeroinitializer, align 4
 
-    define i32 @foo(i8 %a, i1 %cond, ptr %b) {
+    declare i32 @llvm.smax.i32(i32, i32)
+    declare i32 @bar(i32)
+
+    define i32 @foo(i8 %a, i1 %cond, ptr %b, i32 %c) {
       %cmp = icmp eq i8 %a, 10
       %ext = zext i1 %cmp to i32
       %sel = select i1 %cond, i32 %ext, i32 1
       %gep = getelementptr i32, ptr %b, i32 %sel
       %ld = load i32, ptr %gep
-      ret i32 %ld
+      %fr = freeze i32 %ld
+      %smax = call i32 @llvm.smax.i32(i32 %fr, i32 1)
+      %call = call i32 @bar(i32 %smax)
+      %fr2 = freeze i32 %c
+      %add = add i32 %call, %fr2
+      ret i32 %add
     }
   )";
 
@@ -245,6 +253,7 @@ TEST_F(FunctionSpecializationTest, Misc) {
   GlobalVariable *GV = M.getGlobalVariable("g");
   Constant *One = ConstantInt::get(IntegerType::getInt8Ty(M.getContext()), 1);
   Constant *True = ConstantInt::getTrue(M.getContext());
+  Constant *Undef = UndefValue::get(IntegerType::getInt32Ty(M.getContext()));
 
   auto BlockIter = F->front().begin();
   Instruction &Icmp = *BlockIter++;
@@ -252,6 +261,8 @@ TEST_F(FunctionSpecializationTest, Misc) {
   Instruction &Select = *BlockIter++;
   Instruction &Gep = *BlockIter++;
   Instruction &Load = *BlockIter++;
+  Instruction &Freeze = *BlockIter++;
+  Instruction &Smax = *BlockIter++;
 
   // icmp + zext
   Cost Ref = getInstCost(Icmp) + getInstCost(Zext);
@@ -265,9 +276,13 @@ TEST_F(FunctionSpecializationTest, Misc) {
   EXPECT_EQ(Bonus, Ref);
   EXPECT_TRUE(Bonus > 0);
 
-  // gep + load
-  Ref = getInstCost(Gep) + getInstCost(Load);
+  // gep + load + freeze + smax
+  Ref = getInstCost(Gep) + getInstCost(Load) + getInstCost(Freeze) +
+        getInstCost(Smax);
   Bonus = Specializer.getSpecializationBonus(F->getArg(2), GV, Visitor);
   EXPECT_EQ(Bonus, Ref);
   EXPECT_TRUE(Bonus > 0);
+
+  Bonus = Specializer.getSpecializationBonus(F->getArg(3), Undef, Visitor);
+  EXPECT_TRUE(Bonus == 0);
 }

From 6947db2778e0f4799f5311bc80fe7963aa8409c6 Mon Sep 17 00:00:00 2001
From: Russell Greene <russellgreene8@gmail.com>
Date: Mon, 19 Jun 2023 09:59:20 +0000
Subject: [PATCH 007/130] lldb: do more than 1 kilobyte at a time to vastly
 increase binary sync speed

https://github.com/llvm/llvm-project/issues/62750

I setup a simple test with a large .so (~100MiB) that was only present on the target machine
but not present on the local machine, and ran a lldb server on the target and connectd to it.

LLDB properly downloads the file from the remote, but it does so at a very slow speed, even over a hardwired 1Gbps connection!

Increasing the buffer size for downloading these helps quite a bit.

Test setup:

```
$ cat gen.py
print('const char* hugeglobal = ')

for _ in range(1000*500):
    print('  "' + '1234'*50 + '"')

print(';')
print('const char* mystring() { return hugeglobal; }')
$ gen.py > huge.c
$ mkdir libdir
$ gcc -fPIC huge.c -Wl,-soname,libhuge.so -o libdir/libhuge.so -shared
$ cat test.c
#include <string.h>
#include <stdio.h>
extern const char* mystring();
int main() {
        printf("%d\n", strlen(mystring()));
}
$ gcc test.c -L libdir -l huge -Wl,-rpath='$ORIGIN' -o test
$ rsync -a libdir remote:~/
$ ssh remote bash -c "cd ~/libdir && /llvm/buildr/bin/lldb-server platform --server --listen '*:1234'"
```

in another terminal

```
$ rm -rf ~/.lldb # clear cache
$ cat connect.lldb
platform select remote-linux
platform connect connect://10.0.0.14:1234
file test
b main
r
image list
c
q
$ time /llvm/buildr/bin/lldb --source connect.lldb
```

Times with various buffer sizes:

1kiB (current): ~22s
8kiB: ~8s
16kiB: ~4s
32kiB: ~3.5s
64kiB: ~2.8s
128kiB: ~2.6s
256kiB: ~2.1s
512kiB: ~2.1s
1MiB: ~2.1s
2MiB: ~2.1s

I choose 512kiB from this list as it seems to be the place where the returns start diminishing and still isn't that much memory

My  understanding of how this makes such a difference is ReadFile issues a request for each call, and larger buffer means less round trip times. The "ideal" situation is ReadFile() being async and being able to issue multiple of these, but that is much more work for probably little gains.

NOTE: this is my first contribution, so wasn't sure who to choose as a reviewer. Greg Clayton seems to be the most appropriate of those in CODE_OWNERS.txt

Reviewed By: clayborg, jasonmolenda

Differential Revision: https://reviews.llvm.org/D153060
---
 lldb/source/Target/Platform.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lldb/source/Target/Platform.cpp b/lldb/source/Target/Platform.cpp
index 2590197c7149b..4b5d21bede121 100644
--- a/lldb/source/Target/Platform.cpp
+++ b/lldb/source/Target/Platform.cpp
@@ -1630,7 +1630,7 @@ Status Platform::DownloadModuleSlice(const FileSpec &src_file_spec,
     return error;
   }
 
-  std::vector<char> buffer(1024);
+  std::vector<char> buffer(512 * 1024);
   auto offset = src_offset;
   uint64_t total_bytes_read = 0;
   while (total_bytes_read < src_size) {

From 664b7a4cd51d9273888e79688f64cc8bbcbdbe25 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Mon, 19 Jun 2023 12:27:46 +0200
Subject: [PATCH 008/130] [SCCP] Fix conversion of range to constant for
 vectors (PR63380)

The ConstantRange specifies the range of the scalar elements in the
vector. When converting into a Constant, we need to create a vector
splat with the correct type. For that purpose, pass in the expected
type for the constant.

Fixes https://github.com/llvm/llvm-project/issues/63380.
---
 .../llvm/Transforms/Utils/SCCPSolver.h        |  2 +-
 llvm/lib/Transforms/Utils/SCCPSolver.cpp      | 64 +++++++++++--------
 llvm/test/Transforms/SCCP/intrinsics.ll       | 14 ++++
 3 files changed, 52 insertions(+), 28 deletions(-)

diff --git a/llvm/include/llvm/Transforms/Utils/SCCPSolver.h b/llvm/include/llvm/Transforms/Utils/SCCPSolver.h
index 3754b51f4722d..7930d95e1deaf 100644
--- a/llvm/include/llvm/Transforms/Utils/SCCPSolver.h
+++ b/llvm/include/llvm/Transforms/Utils/SCCPSolver.h
@@ -160,7 +160,7 @@ class SCCPSolver {
 
   /// Helper to return a Constant if \p LV is either a constant or a constant
   /// range with a single element.
-  Constant *getConstant(const ValueLatticeElement &LV) const;
+  Constant *getConstant(const ValueLatticeElement &LV, Type *Ty) const;
 
   /// Return either a Constant or nullptr for a given Value.
   Constant *getConstantOrNull(Value *V) const;
diff --git a/llvm/lib/Transforms/Utils/SCCPSolver.cpp b/llvm/lib/Transforms/Utils/SCCPSolver.cpp
index 902651ab84f68..24d1a46cfd40f 100644
--- a/llvm/lib/Transforms/Utils/SCCPSolver.cpp
+++ b/llvm/lib/Transforms/Utils/SCCPSolver.cpp
@@ -394,8 +394,8 @@ class SCCPInstVisitor : public InstVisitor<SCCPInstVisitor> {
   LLVMContext &Ctx;
 
 private:
-  ConstantInt *getConstantInt(const ValueLatticeElement &IV) const {
-    return dyn_cast_or_null<ConstantInt>(getConstant(IV));
+  ConstantInt *getConstantInt(const ValueLatticeElement &IV, Type *Ty) const {
+    return dyn_cast_or_null<ConstantInt>(getConstant(IV, Ty));
   }
 
   // pushToWorkList - Helper for markConstant/markOverdefined
@@ -778,7 +778,7 @@ class SCCPInstVisitor : public InstVisitor<SCCPInstVisitor> {
 
   bool isStructLatticeConstant(Function *F, StructType *STy);
 
-  Constant *getConstant(const ValueLatticeElement &LV) const;
+  Constant *getConstant(const ValueLatticeElement &LV, Type *Ty) const;
 
   Constant *getConstantOrNull(Value *V) const;
 
@@ -881,14 +881,18 @@ bool SCCPInstVisitor::isStructLatticeConstant(Function *F, StructType *STy) {
   return true;
 }
 
-Constant *SCCPInstVisitor::getConstant(const ValueLatticeElement &LV) const {
-  if (LV.isConstant())
-    return LV.getConstant();
+Constant *SCCPInstVisitor::getConstant(const ValueLatticeElement &LV,
+                                       Type *Ty) const {
+  if (LV.isConstant()) {
+    Constant *C = LV.getConstant();
+    assert(C->getType() == Ty && "Type mismatch");
+    return C;
+  }
 
   if (LV.isConstantRange()) {
     const auto &CR = LV.getConstantRange();
     if (CR.getSingleElement())
-      return ConstantInt::get(Ctx, *CR.getSingleElement());
+      return ConstantInt::get(Ty, *CR.getSingleElement());
   }
   return nullptr;
 }
@@ -904,7 +908,7 @@ Constant *SCCPInstVisitor::getConstantOrNull(Value *V) const {
     for (unsigned I = 0, E = ST->getNumElements(); I != E; ++I) {
       ValueLatticeElement LV = LVs[I];
       ConstVals.push_back(SCCPSolver::isConstant(LV)
-                              ? getConstant(LV)
+                              ? getConstant(LV, ST->getElementType(I))
                               : UndefValue::get(ST->getElementType(I)));
     }
     Const = ConstantStruct::get(ST, ConstVals);
@@ -912,7 +916,7 @@ Constant *SCCPInstVisitor::getConstantOrNull(Value *V) const {
     const ValueLatticeElement &LV = getLatticeValueFor(V);
     if (SCCPSolver::isOverdefined(LV))
       return nullptr;
-    Const = SCCPSolver::isConstant(LV) ? getConstant(LV)
+    Const = SCCPSolver::isConstant(LV) ? getConstant(LV, V->getType())
                                        : UndefValue::get(V->getType());
   }
   assert(Const && "Constant is nullptr here!");
@@ -1007,7 +1011,7 @@ void SCCPInstVisitor::getFeasibleSuccessors(Instruction &TI,
     }
 
     ValueLatticeElement BCValue = getValueState(BI->getCondition());
-    ConstantInt *CI = getConstantInt(BCValue);
+    ConstantInt *CI = getConstantInt(BCValue, BI->getCondition()->getType());
     if (!CI) {
       // Overdefined condition variables, and branches on unfoldable constant
       // conditions, mean the branch could go either way.
@@ -1033,7 +1037,8 @@ void SCCPInstVisitor::getFeasibleSuccessors(Instruction &TI,
       return;
     }
     const ValueLatticeElement &SCValue = getValueState(SI->getCondition());
-    if (ConstantInt *CI = getConstantInt(SCValue)) {
+    if (ConstantInt *CI =
+            getConstantInt(SCValue, SI->getCondition()->getType())) {
       Succs[SI->findCaseValue(CI)->getSuccessorIndex()] = true;
       return;
     }
@@ -1064,7 +1069,8 @@ void SCCPInstVisitor::getFeasibleSuccessors(Instruction &TI,
   if (auto *IBR = dyn_cast<IndirectBrInst>(&TI)) {
     // Casts are folded by visitCastInst.
     ValueLatticeElement IBRValue = getValueState(IBR->getAddress());
-    BlockAddress *Addr = dyn_cast_or_null<BlockAddress>(getConstant(IBRValue));
+    BlockAddress *Addr = dyn_cast_or_null<BlockAddress>(
+        getConstant(IBRValue, IBR->getAddress()->getType()));
     if (!Addr) { // Overdefined or unknown condition?
       // All destinations are executable!
       if (!IBRValue.isUnknownOrUndef())
@@ -1219,7 +1225,7 @@ void SCCPInstVisitor::visitCastInst(CastInst &I) {
   if (OpSt.isUnknownOrUndef())
     return;
 
-  if (Constant *OpC = getConstant(OpSt)) {
+  if (Constant *OpC = getConstant(OpSt, I.getOperand(0)->getType())) {
     // Fold the constant as we build.
     Constant *C = ConstantFoldCastOperand(I.getOpcode(), OpC, I.getType(), DL);
     markConstant(&I, C);
@@ -1354,7 +1360,8 @@ void SCCPInstVisitor::visitSelectInst(SelectInst &I) {
   if (CondValue.isUnknownOrUndef())
     return;
 
-  if (ConstantInt *CondCB = getConstantInt(CondValue)) {
+  if (ConstantInt *CondCB =
+          getConstantInt(CondValue, I.getCondition()->getType())) {
     Value *OpVal = CondCB->isZero() ? I.getFalseValue() : I.getTrueValue();
     mergeInValue(&I, getValueState(OpVal));
     return;
@@ -1387,8 +1394,8 @@ void SCCPInstVisitor::visitUnaryOperator(Instruction &I) {
     return;
 
   if (SCCPSolver::isConstant(V0State))
-    if (Constant *C = ConstantFoldUnaryOpOperand(I.getOpcode(),
-                                                 getConstant(V0State), DL))
+    if (Constant *C = ConstantFoldUnaryOpOperand(
+            I.getOpcode(), getConstant(V0State, I.getType()), DL))
       return (void)markConstant(IV, &I, C);
 
   markOverdefined(&I);
@@ -1412,8 +1419,8 @@ void SCCPInstVisitor::visitFreezeInst(FreezeInst &I) {
     return;
 
   if (SCCPSolver::isConstant(V0State) &&
-      isGuaranteedNotToBeUndefOrPoison(getConstant(V0State)))
-    return (void)markConstant(IV, &I, getConstant(V0State));
+      isGuaranteedNotToBeUndefOrPoison(getConstant(V0State, I.getType())))
+    return (void)markConstant(IV, &I, getConstant(V0State, I.getType()));
 
   markOverdefined(&I);
 }
@@ -1437,10 +1444,12 @@ void SCCPInstVisitor::visitBinaryOperator(Instruction &I) {
   // If either of the operands is a constant, try to fold it to a constant.
   // TODO: Use information from notconstant better.
   if ((V1State.isConstant() || V2State.isConstant())) {
-    Value *V1 = SCCPSolver::isConstant(V1State) ? getConstant(V1State)
-                                                : I.getOperand(0);
-    Value *V2 = SCCPSolver::isConstant(V2State) ? getConstant(V2State)
-                                                : I.getOperand(1);
+    Value *V1 = SCCPSolver::isConstant(V1State)
+                    ? getConstant(V1State, I.getOperand(0)->getType())
+                    : I.getOperand(0);
+    Value *V2 = SCCPSolver::isConstant(V2State)
+                    ? getConstant(V2State, I.getOperand(1)->getType())
+                    : I.getOperand(1);
     Value *R = simplifyBinOp(I.getOpcode(), V1, V2, SimplifyQuery(DL));
     auto *C = dyn_cast_or_null<Constant>(R);
     if (C) {
@@ -1518,7 +1527,7 @@ void SCCPInstVisitor::visitGetElementPtrInst(GetElementPtrInst &I) {
     if (SCCPSolver::isOverdefined(State))
       return (void)markOverdefined(&I);
 
-    if (Constant *C = getConstant(State)) {
+    if (Constant *C = getConstant(State, I.getOperand(i)->getType())) {
       Operands.push_back(C);
       continue;
     }
@@ -1584,7 +1593,7 @@ void SCCPInstVisitor::visitLoadInst(LoadInst &I) {
   ValueLatticeElement &IV = ValueState[&I];
 
   if (SCCPSolver::isConstant(PtrVal)) {
-    Constant *Ptr = getConstant(PtrVal);
+    Constant *Ptr = getConstant(PtrVal, I.getOperand(0)->getType());
 
     // load null is undefined.
     if (isa<ConstantPointerNull>(Ptr)) {
@@ -1647,7 +1656,7 @@ void SCCPInstVisitor::handleCallOverdefined(CallBase &CB) {
       if (SCCPSolver::isOverdefined(State))
         return (void)markOverdefined(&CB);
       assert(SCCPSolver::isConstant(State) && "Unknown state!");
-      Operands.push_back(getConstant(State));
+      Operands.push_back(getConstant(State, A->getType()));
     }
 
     if (SCCPSolver::isOverdefined(getValueState(&CB)))
@@ -2067,8 +2076,9 @@ bool SCCPSolver::isStructLatticeConstant(Function *F, StructType *STy) {
   return Visitor->isStructLatticeConstant(F, STy);
 }
 
-Constant *SCCPSolver::getConstant(const ValueLatticeElement &LV) const {
-  return Visitor->getConstant(LV);
+Constant *SCCPSolver::getConstant(const ValueLatticeElement &LV,
+                                  Type *Ty) const {
+  return Visitor->getConstant(LV, Ty);
 }
 
 Constant *SCCPSolver::getConstantOrNull(Value *V) const {
diff --git a/llvm/test/Transforms/SCCP/intrinsics.ll b/llvm/test/Transforms/SCCP/intrinsics.ll
index 3fc7637ab7327..5edb31738e685 100644
--- a/llvm/test/Transforms/SCCP/intrinsics.ll
+++ b/llvm/test/Transforms/SCCP/intrinsics.ll
@@ -122,3 +122,17 @@ exit:
   %p_umax = call i8 @llvm.umax.i8(i8 %p, i8 1)
   ret i8 %p_umax
 }
+
+define <4 x i32> @pr63380(<4 x i32> %input) {
+; CHECK-LABEL: @pr63380(
+; CHECK-NEXT:    [[CTLZ_1:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[INPUT:%.*]], i1 false)
+; CHECK-NEXT:    [[CTLZ_2:%.*]] = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> [[CTLZ_1]], i1 true)
+; CHECK-NEXT:    ret <4 x i32> <i32 27, i32 27, i32 27, i32 27>
+;
+  %ctlz.1 = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %input, i1 false)
+  %ctlz.2 = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %ctlz.1, i1 true)
+  %ctlz.3 = call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %ctlz.2, i1 true)
+  ret <4 x i32> %ctlz.3
+}
+
+declare <4 x i32> @llvm.ctlz.v4i32(<4 x i32>, i1 immarg)

From 65a2cde77195887f308d4f14be2e3ae18c5c5c0a Mon Sep 17 00:00:00 2001
From: Mehdi Amini <joker.eph@gmail.com>
Date: Mon, 19 Jun 2023 11:23:50 +0200
Subject: [PATCH 009/130] Remove extra closing `)` from MLIR --view-op-graph
 GraphViz emission

This was a spurious closing parenthese.
---
 mlir/lib/Transforms/ViewOpGraph.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/mlir/lib/Transforms/ViewOpGraph.cpp b/mlir/lib/Transforms/ViewOpGraph.cpp
index def8a1443b1aa..7689aa061a09d 100644
--- a/mlir/lib/Transforms/ViewOpGraph.cpp
+++ b/mlir/lib/Transforms/ViewOpGraph.cpp
@@ -228,7 +228,6 @@ class PrintOpPass : public impl::ViewOpGraphBase<PrintOpPass> {
         llvm::raw_string_ostream ss(buf);
         interleaveComma(op->getResultTypes(), ss);
         os << truncateString(ss.str()) << ")";
-        os << ")";
       }
 
       // Print attributes.

From 9ef73f2f58ecfce21bdeb27f68b224d9be97cfe6 Mon Sep 17 00:00:00 2001
From: Mehdi Amini <joker.eph@gmail.com>
Date: Mon, 19 Jun 2023 11:58:49 +0200
Subject: [PATCH 010/130] Emit MLIR trampoline wrapper function with LLVM
 private linkage

The wrapper, as most of compiler-generated functions, are intended to serve the
IR for the current module. The safest linkage is to keep these private to avoid
any possible collision with other modules.

Differential Revision: https://reviews.llvm.org/D153255
---
 mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp  |  2 ++
 ...emit-c-wrappers-for-external-functions.mlir | 18 +++++++++---------
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
index 5867d9fa02b06..3ae997d713104 100644
--- a/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
+++ b/mlir/lib/Conversion/FuncToLLVM/FuncToLLVM.cpp
@@ -208,6 +208,8 @@ static void wrapExternalFunction(OpBuilder &builder, Location loc,
       wrapperType, LLVM::Linkage::External, /*dsoLocal*/ false,
       /*cconv*/ LLVM::CConv::C, attributes);
 
+  // The wrapper that we synthetize here should only be visible in this module.
+  newFuncOp.setLinkage(LLVM::Linkage::Private);
   builder.setInsertionPointToStart(newFuncOp.addEntryBlock());
 
   // Get a ValueRange containing arguments.
diff --git a/mlir/test/Conversion/FuncToLLVM/emit-c-wrappers-for-external-functions.mlir b/mlir/test/Conversion/FuncToLLVM/emit-c-wrappers-for-external-functions.mlir
index 3e8841120463f..027d29b0bf079 100644
--- a/mlir/test/Conversion/FuncToLLVM/emit-c-wrappers-for-external-functions.mlir
+++ b/mlir/test/Conversion/FuncToLLVM/emit-c-wrappers-for-external-functions.mlir
@@ -1,38 +1,38 @@
 // RUN: mlir-opt -llvm-request-c-wrappers -convert-func-to-llvm='use-opaque-pointers=1' %s | FileCheck %s
 
-// CHECK: llvm.func @res_attrs_with_memref_return() -> (!llvm.struct{{.*}} {test.returnOne})
+// CHECK: llvm.func private @res_attrs_with_memref_return() -> (!llvm.struct{{.*}} {test.returnOne})
 // CHECK-LABEL: llvm.func @_mlir_ciface_res_attrs_with_memref_return
 // CHECK-SAME: !llvm.ptr
 // CHECK-NOT: test.returnOne
 func.func private @res_attrs_with_memref_return() -> (memref<f32> {test.returnOne})
 
-// CHECK: llvm.func @res_attrs_with_value_return() -> (f32 {test.returnOne = 1 : i64})
+// CHECK: llvm.func private @res_attrs_with_value_return() -> (f32 {test.returnOne = 1 : i64})
 // CHECK-LABEL: llvm.func @_mlir_ciface_res_attrs_with_value_return
 // CHECK-SAME: -> (f32 {test.returnOne = 1 : i64})
 func.func private @res_attrs_with_value_return() -> (f32 {test.returnOne = 1})
 
-// CHECK: llvm.func @multiple_return() -> !llvm.struct<{{.*}}>
+// CHECK: llvm.func private @multiple_return() -> !llvm.struct<{{.*}}>
 // CHECK-LABEL: llvm.func @_mlir_ciface_multiple_return
 // CHECK-NOT: test.returnOne
 // CHECK-NOT: test.returnTwo
 // CHECK-NOT: test.returnThree
 func.func private @multiple_return() -> (memref<f32> {test.returnOne = 1}, f32 {test.returnTwo = 2, test.returnThree = 3})
 
-// CHECK: llvm.func @multiple_return_missing_res_attr() -> !llvm.struct<{{.*}}>
+// CHECK: llvm.func private @multiple_return_missing_res_attr() -> !llvm.struct<{{.*}}>
 // CHECK-LABEL: llvm.func @_mlir_ciface_multiple_return_missing_res_attr
 // CHECK-NOT: test.returnOne
 // CHECK-NOT: test.returnTwo
 // CHECK-NOT: test.returnThree
 func.func private @multiple_return_missing_res_attr() -> (memref<f32> {test.returnOne = 1}, i64, f32 {test.returnTwo = 2, test.returnThree = 3})
 
-// CHECK: llvm.func @one_arg_attr_no_res_attrs_with_memref_return({{.*}}) -> !llvm.struct{{.*}}
+// CHECK: llvm.func private @one_arg_attr_no_res_attrs_with_memref_return({{.*}}) -> !llvm.struct{{.*}}
 // CHECK-LABEL: llvm.func @_mlir_ciface_one_arg_attr_no_res_attrs_with_memref_return
 // CHECK-SAME: !llvm.ptr
 // CHECK-SAME: !llvm.ptr
 // CHECK-SAME: {test.argOne = 1 : i64})
 func.func private @one_arg_attr_no_res_attrs_with_memref_return(%arg0: memref<f32> {test.argOne = 1}) -> memref<f32>
 
-// CHECK: llvm.func @one_arg_attr_one_res_attr_with_memref_return({{.*}}) -> (!llvm.struct<{{.*}}> {test.returnOne = 1 : i64})
+// CHECK: llvm.func private @one_arg_attr_one_res_attr_with_memref_return({{.*}}) -> (!llvm.struct<{{.*}}> {test.returnOne = 1 : i64})
 // CHECK-LABEL: llvm.func @_mlir_ciface_one_arg_attr_one_res_attr_with_memref_return
 // CHECK-SAME: !llvm.ptr
 // CHECK-NOT: test.returnOne
@@ -40,14 +40,14 @@ func.func private @one_arg_attr_no_res_attrs_with_memref_return(%arg0: memref<f3
 // CHECK-SAME: {test.argOne = 1 : i64})
 func.func private @one_arg_attr_one_res_attr_with_memref_return(%arg0: memref<f32> {test.argOne = 1}) -> (memref<f32> {test.returnOne = 1})
 
-// CHECK: llvm.func @one_arg_attr_one_res_attr_with_value_return({{.*}}) -> (f32 {test.returnOne = 1 : i64})
+// CHECK: llvm.func private @one_arg_attr_one_res_attr_with_value_return({{.*}}) -> (f32 {test.returnOne = 1 : i64})
 // CHECK-LABEL: llvm.func @_mlir_ciface_one_arg_attr_one_res_attr_with_value_return
 // CHECK-SAME: !llvm.ptr
 // CHECK-SAME: {test.argOne = 1 : i64}
 // CHECK-SAME: -> (f32 {test.returnOne = 1 : i64})
 func.func private @one_arg_attr_one_res_attr_with_value_return(%arg0: memref<f32> {test.argOne = 1}) -> (f32 {test.returnOne = 1})
 
-// CHECK: llvm.func @multiple_arg_attr_multiple_res_attr({{.*}}) -> !llvm.struct<{{.*}}>
+// CHECK: llvm.func private @multiple_arg_attr_multiple_res_attr({{.*}}) -> !llvm.struct<{{.*}}>
 // CHECK-LABEL: llvm.func @_mlir_ciface_multiple_arg_attr_multiple_res_attr
 // CHECK-SAME: !llvm.ptr
 // CHECK-NOT: test.returnOne
@@ -58,7 +58,7 @@ func.func private @one_arg_attr_one_res_attr_with_value_return(%arg0: memref<f32
 // CHECK-SAME: i32 {test.argTwo = 2 : i64})
 func.func private @multiple_arg_attr_multiple_res_attr(%arg0: memref<f32> {test.argZero = 0}, %arg1: f32, %arg2: i32 {test.argTwo = 2}) -> (f32, memref<i32> {test.returnOne = 1}, i32 {test.returnTwo = 2})
 
-// CHECK: llvm.func weak @drop_linkage_attr() -> (!llvm.struct{{.*}} {test.returnOne})
+// CHECK: llvm.func private @drop_linkage_attr() -> (!llvm.struct{{.*}} {test.returnOne})
 // CHECK-LABEL: llvm.func @_mlir_ciface_drop_linkage_attr
 // CHECK-SAME: !llvm.ptr
 // CHECK-NOT: llvm.linkage

From 6f2e92c10cebca51f9bc98b99fa3b5583ff396c4 Mon Sep 17 00:00:00 2001
From: Alexandre Ganea <alex_toresh@yahoo.fr>
Date: Mon, 19 Jun 2023 07:32:34 -0400
Subject: [PATCH 011/130] Re-land [LLD] Allow usage of LLD as a library

This reverts commit aa495214b39d475bab24b468de7a7c676ce9e366.

As discussed in https://github.com/llvm/llvm-project/issues/53475 this patch
allows for using LLD-as-a-lib. It also lets clients link only the drivers that
they want (see unit tests).

This also adds the unit test infra as in the other LLVM projects. Among the
test coverage, I've added the original issue from @krzysz00, see:
https://github.com/ROCmSoftwarePlatform/D108850-lld-bug-reproduction

Important note: this doesn't allow (yet) linking in parallel. This will come a
bit later hopefully, in subsequent patches, for COFF at least.

Differential revision: https://reviews.llvm.org/D119049
---
 lld/CMakeLists.txt                      |   2 +
 lld/COFF/Driver.cpp                     |   2 +-
 lld/Common/CMakeLists.txt               |   1 +
 lld/Common/DriverDispatcher.cpp         | 203 ++++++++++++++++++++++++
 lld/ELF/Driver.cpp                      |  11 +-
 lld/MachO/Driver.cpp                    |   9 +-
 lld/MinGW/Driver.cpp                    |  14 +-
 lld/docs/NewLLD.rst                     |   2 +-
 lld/docs/index.rst                      |   2 +-
 lld/include/lld/Common/Driver.h         |  71 +++++----
 lld/test/CMakeLists.txt                 |  16 +-
 lld/test/Unit/lit.cfg.py                |  47 ++++++
 lld/test/Unit/lit.site.cfg.py.in        |  12 ++
 lld/tools/lld/lld.cpp                   | 191 ++++------------------
 lld/unittests/AsLibAll/AllDrivers.cpp   |  35 ++++
 lld/unittests/AsLibAll/CMakeLists.txt   |  17 ++
 lld/unittests/AsLibELF/CMakeLists.txt   |  14 ++
 lld/unittests/AsLibELF/Inputs/kernel1.o | Bin 0 -> 1688 bytes
 lld/unittests/AsLibELF/Inputs/kernel2.o | Bin 0 -> 1688 bytes
 lld/unittests/AsLibELF/ROCm.cpp         |  74 +++++++++
 lld/unittests/AsLibELF/SomeDrivers.cpp  |  37 +++++
 lld/unittests/CMakeLists.txt            |   8 +
 lld/wasm/Driver.cpp                     |   2 +-
 llvm/cmake/modules/AddLLVM.cmake        |  10 ++
 24 files changed, 570 insertions(+), 210 deletions(-)
 create mode 100644 lld/Common/DriverDispatcher.cpp
 create mode 100644 lld/test/Unit/lit.cfg.py
 create mode 100644 lld/test/Unit/lit.site.cfg.py.in
 create mode 100644 lld/unittests/AsLibAll/AllDrivers.cpp
 create mode 100644 lld/unittests/AsLibAll/CMakeLists.txt
 create mode 100644 lld/unittests/AsLibELF/CMakeLists.txt
 create mode 100644 lld/unittests/AsLibELF/Inputs/kernel1.o
 create mode 100644 lld/unittests/AsLibELF/Inputs/kernel2.o
 create mode 100644 lld/unittests/AsLibELF/ROCm.cpp
 create mode 100644 lld/unittests/AsLibELF/SomeDrivers.cpp
 create mode 100644 lld/unittests/CMakeLists.txt

diff --git a/lld/CMakeLists.txt b/lld/CMakeLists.txt
index 518289a6328b6..1ffcf2d9a2564 100644
--- a/lld/CMakeLists.txt
+++ b/lld/CMakeLists.txt
@@ -191,6 +191,8 @@ add_subdirectory(Common)
 add_subdirectory(tools/lld)
 
 if (LLVM_INCLUDE_TESTS)
+  add_custom_target(LLDUnitTests)
+  llvm_add_unittests(LLD_UNITTESTS_ADDED)
   add_subdirectory(test)
 endif()
 
diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp
index d4115a0e85eb2..f8e2c17099c0c 100644
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@@ -64,7 +64,7 @@ namespace lld::coff {
 
 bool link(ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS,
           llvm::raw_ostream &stderrOS, bool exitEarly, bool disableOutput) {
-  // This driver-specific context will be freed later by lldMain().
+  // This driver-specific context will be freed later by unsafeLldMain().
   auto *ctx = new COFFLinkerContext;
 
   ctx->e.initialize(stdoutOS, stderrOS, exitEarly, disableOutput);
diff --git a/lld/Common/CMakeLists.txt b/lld/Common/CMakeLists.txt
index 71df89c28aea0..c975da2a49b6b 100644
--- a/lld/Common/CMakeLists.txt
+++ b/lld/Common/CMakeLists.txt
@@ -23,6 +23,7 @@ set_source_files_properties("${version_inc}"
 add_lld_library(lldCommon
   Args.cpp
   CommonLinkerContext.cpp
+  DriverDispatcher.cpp
   DWARF.cpp
   ErrorHandler.cpp
   Filesystem.cpp
diff --git a/lld/Common/DriverDispatcher.cpp b/lld/Common/DriverDispatcher.cpp
new file mode 100644
index 0000000000000..379a4c6ddabea
--- /dev/null
+++ b/lld/Common/DriverDispatcher.cpp
@@ -0,0 +1,203 @@
+//===- DriverDispatcher.cpp - Support using LLD as a library --------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "lld/Common/CommonLinkerContext.h"
+#include "lld/Common/Driver.h"
+#include "lld/Common/ErrorHandler.h"
+#include "lld/Common/Memory.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/CrashRecoveryContext.h"
+#include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Process.h"
+#include "llvm/TargetParser/Host.h"
+#include "llvm/TargetParser/Triple.h"
+#include <cstdlib>
+
+using namespace lld;
+using namespace llvm;
+using namespace llvm::sys;
+
+static void err(const Twine &s) { llvm::errs() << s << "\n"; }
+
+static Flavor getFlavor(StringRef s) {
+  return StringSwitch<Flavor>(s)
+      .CasesLower("ld", "ld.lld", "gnu", Gnu)
+      .CasesLower("wasm", "ld-wasm", Wasm)
+      .CaseLower("link", WinLink)
+      .CasesLower("ld64", "ld64.lld", "darwin", Darwin)
+      .Default(Invalid);
+}
+
+static cl::TokenizerCallback getDefaultQuotingStyle() {
+  if (Triple(sys::getProcessTriple()).getOS() == Triple::Win32)
+    return cl::TokenizeWindowsCommandLine;
+  return cl::TokenizeGNUCommandLine;
+}
+
+static bool isPETargetName(StringRef s) {
+  return s == "i386pe" || s == "i386pep" || s == "thumb2pe" || s == "arm64pe";
+}
+
+static std::optional<bool> isPETarget(llvm::ArrayRef<const char *> args) {
+  for (auto it = args.begin(); it + 1 != args.end(); ++it) {
+    if (StringRef(*it) != "-m")
+      continue;
+    return isPETargetName(*(it + 1));
+  }
+
+  // Expand response files (arguments in the form of @<filename>)
+  // to allow detecting the -m argument from arguments in them.
+  SmallVector<const char *, 256> expandedArgs(args.data(),
+                                              args.data() + args.size());
+  BumpPtrAllocator a;
+  StringSaver saver(a);
+  cl::ExpansionContext ectx(saver.getAllocator(), getDefaultQuotingStyle());
+  if (Error e = ectx.expandResponseFiles(expandedArgs)) {
+    err(toString(std::move(e)));
+    return std::nullopt;
+  }
+
+  for (auto it = expandedArgs.begin(); it + 1 != expandedArgs.end(); ++it) {
+    if (StringRef(*it) != "-m")
+      continue;
+    return isPETargetName(*(it + 1));
+  }
+
+#ifdef LLD_DEFAULT_LD_LLD_IS_MINGW
+  return true;
+#else
+  return false;
+#endif
+}
+
+static Flavor parseProgname(StringRef progname) {
+  // Use GNU driver for "ld" by default.
+  if (progname == "ld")
+    return Gnu;
+
+  // Progname may be something like "lld-gnu". Parse it.
+  SmallVector<StringRef, 3> v;
+  progname.split(v, "-");
+  for (StringRef s : v)
+    if (Flavor f = getFlavor(s))
+      return f;
+  return Invalid;
+}
+
+static Flavor
+parseFlavorWithoutMinGW(llvm::SmallVectorImpl<const char *> &argsV) {
+  // Parse -flavor option.
+  if (argsV.size() > 1 && argsV[1] == StringRef("-flavor")) {
+    if (argsV.size() <= 2) {
+      err("missing arg value for '-flavor'");
+      return Invalid;
+    }
+    Flavor f = getFlavor(argsV[2]);
+    if (f == Invalid) {
+      err("Unknown flavor: " + StringRef(argsV[2]));
+      return Invalid;
+    }
+    argsV.erase(argsV.begin() + 1, argsV.begin() + 3);
+    return f;
+  }
+
+  // Deduct the flavor from argv[0].
+  StringRef arg0 = path::filename(argsV[0]);
+  if (arg0.ends_with_insensitive(".exe"))
+    arg0 = arg0.drop_back(4);
+  Flavor f = parseProgname(arg0);
+  if (f == Invalid) {
+    err("lld is a generic driver.\n"
+        "Invoke ld.lld (Unix), ld64.lld (macOS), lld-link (Windows), wasm-ld"
+        " (WebAssembly) instead");
+    return Invalid;
+  }
+  return f;
+}
+
+static Flavor parseFlavor(llvm::SmallVectorImpl<const char *> &argsV) {
+  Flavor f = parseFlavorWithoutMinGW(argsV);
+  if (f == Gnu) {
+    auto isPE = isPETarget(argsV);
+    if (!isPE)
+      return Invalid;
+    if (*isPE)
+      return MinGW;
+  }
+  return f;
+}
+
+static Driver whichDriver(llvm::SmallVectorImpl<const char *> &argsV,
+                          llvm::ArrayRef<DriverDef> drivers) {
+  Flavor f = parseFlavor(argsV);
+  auto it =
+      llvm::find_if(drivers, [=](auto &driverdef) { return driverdef.f == f; });
+  if (it == drivers.end()) {
+    // Driver is invalid or not available in this build.
+    return [](llvm::ArrayRef<const char *>, llvm::raw_ostream &,
+              llvm::raw_ostream &, bool, bool) { return false; };
+  }
+  return it->d;
+}
+
+namespace lld {
+bool inTestOutputDisabled = false;
+
+/// Universal linker main(). This linker emulates the gnu, darwin, or
+/// windows linker based on the argv[0] or -flavor option.
+int unsafeLldMain(llvm::ArrayRef<const char *> args,
+                  llvm::raw_ostream &stdoutOS, llvm::raw_ostream &stderrOS,
+                  llvm::ArrayRef<DriverDef> drivers, bool exitEarly) {
+  SmallVector<const char *, 256> argsV(args);
+  Driver d = whichDriver(argsV, drivers);
+  // Run the driver. If an error occurs, false will be returned.
+  int r = !d(argsV, stdoutOS, stderrOS, exitEarly, inTestOutputDisabled);
+  // At this point 'r' is either 1 for error, and 0 for no error.
+
+  // Call exit() if we can to avoid calling destructors.
+  if (exitEarly)
+    exitLld(r);
+
+  // Delete the global context and clear the global context pointer, so that it
+  // cannot be accessed anymore.
+  CommonLinkerContext::destroy();
+
+  return r;
+}
+} // namespace lld
+
+Result lld::lldMain(llvm::ArrayRef<const char *> args,
+                    llvm::raw_ostream &stdoutOS, llvm::raw_ostream &stderrOS,
+                    llvm::ArrayRef<DriverDef> drivers) {
+  int r = 0;
+  {
+    // The crash recovery is here only to be able to recover from arbitrary
+    // control flow when fatal() is called (through setjmp/longjmp or
+    // __try/__except).
+    llvm::CrashRecoveryContext crc;
+    if (!crc.RunSafely([&]() {
+          r = unsafeLldMain(args, stdoutOS, stderrOS, drivers,
+                            /*exitEarly=*/false);
+        }))
+      return {crc.RetCode, /*canRunAgain=*/false};
+  }
+
+  // Cleanup memory and reset everything back in pristine condition. This path
+  // is only taken when LLD is in test, or when it is used as a library.
+  llvm::CrashRecoveryContext crc;
+  if (!crc.RunSafely([&]() { CommonLinkerContext::destroy(); })) {
+    // The memory is corrupted beyond any possible recovery.
+    return {r, /*canRunAgain=*/false};
+  }
+  return {r, /*canRunAgain=*/true};
+}
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index db5e9bc5ab04b..4eb10fc3c91e1 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -107,10 +107,11 @@ void Ctx::reset() {
   needsTlsLd.store(false, std::memory_order_relaxed);
 }
 
-bool elf::link(ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS,
-               llvm::raw_ostream &stderrOS, bool exitEarly,
-               bool disableOutput) {
-  // This driver-specific context will be freed later by lldMain().
+namespace lld {
+namespace elf {
+bool link(ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS,
+          llvm::raw_ostream &stderrOS, bool exitEarly, bool disableOutput) {
+  // This driver-specific context will be freed later by unsafeLldMain().
   auto *ctx = new CommonLinkerContext;
 
   ctx->e.initialize(stdoutOS, stderrOS, exitEarly, disableOutput);
@@ -147,6 +148,8 @@ bool elf::link(ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS,
 
   return errorCount() == 0;
 }
+} // namespace elf
+} // namespace lld
 
 // Parses a linker -m option.
 static std::tuple<ELFKind, uint16_t, uint8_t> parseEmulation(StringRef emul) {
diff --git a/lld/MachO/Driver.cpp b/lld/MachO/Driver.cpp
index 87664b5c2ede3..8fa3ee0828797 100644
--- a/lld/MachO/Driver.cpp
+++ b/lld/MachO/Driver.cpp
@@ -1367,9 +1367,10 @@ static void handleExplicitExports() {
   }
 }
 
-bool macho::link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
-                 llvm::raw_ostream &stderrOS, bool exitEarly,
-                 bool disableOutput) {
+namespace lld {
+namespace macho {
+bool link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
+          llvm::raw_ostream &stderrOS, bool exitEarly, bool disableOutput) {
   // This driver-specific context will be freed later by lldMain().
   auto *ctx = new CommonLinkerContext;
 
@@ -1968,3 +1969,5 @@ bool macho::link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
 
   return errorCount() == 0;
 }
+} // namespace macho
+} // namespace lld
diff --git a/lld/MinGW/Driver.cpp b/lld/MinGW/Driver.cpp
index c5a32c3a90bb8..a5b83d2f12ca1 100644
--- a/lld/MinGW/Driver.cpp
+++ b/lld/MinGW/Driver.cpp
@@ -157,11 +157,17 @@ searchLibrary(StringRef name, ArrayRef<StringRef> searchPaths, bool bStatic) {
   return "";
 }
 
+namespace lld {
+namespace coff {
+bool link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
+          llvm::raw_ostream &stderrOS, bool exitEarly, bool disableOutput);
+}
+
+namespace mingw {
 // Convert Unix-ish command line arguments to Windows-ish ones and
 // then call coff::link.
-bool mingw::link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
-                 llvm::raw_ostream &stderrOS, bool exitEarly,
-                 bool disableOutput) {
+bool link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
+          llvm::raw_ostream &stderrOS, bool exitEarly, bool disableOutput) {
   auto *ctx = new CommonLinkerContext;
   ctx->e.initialize(stdoutOS, stderrOS, exitEarly, disableOutput);
 
@@ -482,3 +488,5 @@ bool mingw::link(ArrayRef<const char *> argsArr, llvm::raw_ostream &stdoutOS,
 
   return coff::link(vec, stdoutOS, stderrOS, exitEarly, disableOutput);
 }
+} // namespace mingw
+} // namespace lld
diff --git a/lld/docs/NewLLD.rst b/lld/docs/NewLLD.rst
index 1b1c87067f512..18b0cc83db519 100644
--- a/lld/docs/NewLLD.rst
+++ b/lld/docs/NewLLD.rst
@@ -5,7 +5,7 @@ The ELF Linker as a Library
 ---------------------------
 
 You can embed LLD to your program by linking against it and calling the linker's
-entry point function lld::elf::link.
+entry point function ``lld::lldMain``.
 
 The current policy is that it is your responsibility to give trustworthy object
 files. The function is guaranteed to return as long as you do not pass corrupted
diff --git a/lld/docs/index.rst b/lld/docs/index.rst
index a50d5c0b3453d..bf7cf4846e854 100644
--- a/lld/docs/index.rst
+++ b/lld/docs/index.rst
@@ -36,7 +36,7 @@ Features
   external linkers. All you have to do is to construct object files
   and command line arguments just like you would do to invoke an
   external linker and then call the linker's main function,
-  ``lld::elf::link``, from your code.
+  ``lld::lldMain``, from your code.
 
 - It is small. We are using LLVM libObject library to read from object
   files, so it is not a completely fair comparison, but as of February
diff --git a/lld/include/lld/Common/Driver.h b/lld/include/lld/Common/Driver.h
index 19573e6e100dd..8520e6e7e257c 100644
--- a/lld/include/lld/Common/Driver.h
+++ b/lld/include/lld/Common/Driver.h
@@ -13,8 +13,25 @@
 #include "llvm/Support/raw_ostream.h"
 
 namespace lld {
-struct SafeReturn {
-  int ret;
+enum Flavor {
+  Invalid,
+  Gnu,     // -flavor gnu
+  MinGW,   // -flavor gnu MinGW
+  WinLink, // -flavor link
+  Darwin,  // -flavor darwin
+  Wasm,    // -flavor wasm
+};
+
+using Driver = bool (*)(llvm::ArrayRef<const char *>, llvm::raw_ostream &,
+                        llvm::raw_ostream &, bool, bool);
+
+struct DriverDef {
+  Flavor f;
+  Driver d;
+};
+
+struct Result {
+  int retCode;
   bool canRunAgain;
 };
 
@@ -24,33 +41,29 @@ struct SafeReturn {
 // and re-entry would not be possible anymore. Use exitLld() in that case to
 // properly exit your application and avoid intermittent crashes on exit caused
 // by cleanup.
-SafeReturn safeLldMain(int argc, const char **argv, llvm::raw_ostream &stdoutOS,
-                       llvm::raw_ostream &stderrOS);
-
-namespace coff {
-bool link(llvm::ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS,
-          llvm::raw_ostream &stderrOS, bool exitEarly, bool disableOutput);
-}
-
-namespace mingw {
-bool link(llvm::ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS,
-          llvm::raw_ostream &stderrOS, bool exitEarly, bool disableOutput);
-}
-
-namespace elf {
-bool link(llvm::ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS,
-          llvm::raw_ostream &stderrOS, bool exitEarly, bool disableOutput);
-}
-
-namespace macho {
-bool link(llvm::ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS,
-          llvm::raw_ostream &stderrOS, bool exitEarly, bool disableOutput);
-}
-
-namespace wasm {
-bool link(llvm::ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS,
-          llvm::raw_ostream &stderrOS, bool exitEarly, bool disableOutput);
-}
+Result lldMain(llvm::ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS,
+               llvm::raw_ostream &stderrOS, llvm::ArrayRef<DriverDef> drivers);
 } // namespace lld
 
+// With this macro, library users must specify which drivers they use, provide
+// that information to lldMain() in the `drivers` param, and link the
+// corresponding driver library in their executable.
+#define LLD_HAS_DRIVER(name)                                                   \
+  namespace lld {                                                              \
+  namespace name {                                                             \
+  bool link(llvm::ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS,    \
+            llvm::raw_ostream &stderrOS, bool exitEarly, bool disableOutput);  \
+  }                                                                            \
+  }
+
+// An array which declares that all LLD drivers are linked in your executable.
+// Must be used along with LLD_HAS_DRIVERS. See examples in LLD unittests.
+#define LLD_ALL_DRIVERS                                                        \
+  {                                                                            \
+    {lld::WinLink, &lld::coff::link}, {lld::Gnu, &lld::elf::link},             \
+        {lld::MinGW, &lld::mingw::link}, {lld::Darwin, &lld::macho::link}, {   \
+      lld::Wasm, &lld::wasm::link                                              \
+    }                                                                          \
+  }
+
 #endif
diff --git a/lld/test/CMakeLists.txt b/lld/test/CMakeLists.txt
index 042bfd9140b6b..f1cfacba9dd16 100644
--- a/lld/test/CMakeLists.txt
+++ b/lld/test/CMakeLists.txt
@@ -14,12 +14,23 @@ configure_lit_site_cfg(
   ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py
   MAIN_CONFIG
   ${CMAKE_CURRENT_SOURCE_DIR}/lit.cfg.py
-  )
+)
+configure_lit_site_cfg(
+  ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.site.cfg.py.in
+  ${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg.py
+  MAIN_CONFIG
+  ${CMAKE_CURRENT_SOURCE_DIR}/Unit/lit.cfg.py
+  PATHS
+  "SHLIBDIR"
+  "LLD_BINARY_DIR"
+  "LLD_SOURCE_DIR"
+)
 
 set(LLD_TEST_DEPS lld)
 if (NOT LLD_BUILT_STANDALONE)
   list(APPEND LLD_TEST_DEPS
     FileCheck
+    LLDUnitTests
     count
     dsymutil
     llc
@@ -60,6 +71,7 @@ add_lit_testsuite(check-lld "Running lld test suite"
   ${CMAKE_CURRENT_BINARY_DIR}
   DEPENDS ${LLD_TEST_DEPS}
   )
+set_target_properties(check-lld PROPERTIES FOLDER "lld tests")
 
 add_custom_target(lld-test-depends DEPENDS ${LLD_TEST_DEPS})
 set_target_properties(lld-test-depends PROPERTIES FOLDER "lld tests")
@@ -68,8 +80,6 @@ add_lit_testsuites(LLD ${CMAKE_CURRENT_SOURCE_DIR}
   DEPENDS ${LLD_TEST_DEPS}
   )
 
-set_target_properties(check-lld PROPERTIES FOLDER "lld tests")
-
 # Add a legacy target spelling: lld-test
 add_custom_target(lld-test)
 add_dependencies(lld-test check-lld)
diff --git a/lld/test/Unit/lit.cfg.py b/lld/test/Unit/lit.cfg.py
new file mode 100644
index 0000000000000..ffbbddeeed5ec
--- /dev/null
+++ b/lld/test/Unit/lit.cfg.py
@@ -0,0 +1,47 @@
+# -*- Python -*-
+
+# Configuration file for the 'lit' test runner.
+
+import os
+import subprocess
+
+import lit.formats
+
+# name: The name of this test suite.
+config.name = "LLD-Unit"
+
+# suffixes: A list of file extensions to treat as test files.
+config.suffixes = []
+
+# test_source_root: The root path where tests are located.
+# test_exec_root: The root path where tests should be run.
+config.test_exec_root = os.path.join(config.lld_obj_root, "unittests")
+config.test_source_root = config.test_exec_root
+
+# testFormat: The test format to use to interpret tests.
+config.test_format = lit.formats.GoogleTest(config.llvm_build_mode, "Tests")
+
+# Propagate the temp directory. Windows requires this because it uses \Windows\
+# if none of these are present.
+if "TMP" in os.environ:
+    config.environment["TMP"] = os.environ["TMP"]
+if "TEMP" in os.environ:
+    config.environment["TEMP"] = os.environ["TEMP"]
+
+# Propagate HOME as it can be used to override incorrect homedir in passwd
+# that causes the tests to fail.
+if "HOME" in os.environ:
+    config.environment["HOME"] = os.environ["HOME"]
+
+# Win32 seeks DLLs along %PATH%.
+if sys.platform in ["win32", "cygwin"] and os.path.isdir(config.shlibdir):
+    config.environment["PATH"] = os.path.pathsep.join((
+            config.shlibdir, config.environment["PATH"]))
+
+# Win32 may use %SYSTEMDRIVE% during file system shell operations, so propogate.
+if sys.platform == "win32" and "SYSTEMDRIVE" in os.environ:
+    config.environment["SYSTEMDRIVE"] = os.environ["SYSTEMDRIVE"]
+
+# Expand the LLD source path so that unittests can use associated input files.
+# (see AsLibELF/ROCm.cpp test)
+config.environment["LLD_SRC_DIR"] = config.lld_src_dir
diff --git a/lld/test/Unit/lit.site.cfg.py.in b/lld/test/Unit/lit.site.cfg.py.in
new file mode 100644
index 0000000000000..204ac5f204d7b
--- /dev/null
+++ b/lld/test/Unit/lit.site.cfg.py.in
@@ -0,0 +1,12 @@
+@LIT_SITE_CFG_IN_HEADER@
+
+import sys
+
+config.llvm_build_mode = lit_config.substitute("@LLVM_BUILD_MODE@")
+config.shlibdir = lit_config.substitute(path(r"@SHLIBDIR@"))
+config.lld_obj_root = path(r"@LLD_BINARY_DIR@")
+config.lld_src_dir = path(r"@LLD_SOURCE_DIR@")
+
+# Let the main config do the real work.
+lit_config.load_config(
+    config, os.path.join(config.lld_src_dir, "test/Unit/lit.cfg.py"))
diff --git a/lld/tools/lld/lld.cpp b/lld/tools/lld/lld.cpp
index 2fc9c3a669146..2c30bc905106e 100644
--- a/lld/tools/lld/lld.cpp
+++ b/lld/tools/lld/lld.cpp
@@ -25,7 +25,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "lld/Common/CommonLinkerContext.h"
 #include "lld/Common/Driver.h"
 #include "lld/Common/ErrorHandler.h"
 #include "lld/Common/Memory.h"
@@ -49,161 +48,15 @@ using namespace lld;
 using namespace llvm;
 using namespace llvm::sys;
 
-enum Flavor {
-  Invalid,
-  Gnu,     // -flavor gnu
-  WinLink, // -flavor link
-  Darwin,  // -flavor darwin
-  Wasm,    // -flavor wasm
-};
+namespace lld {
+extern bool inTestOutputDisabled;
 
-[[noreturn]] static void die(const Twine &s) {
-  llvm::errs() << s << "\n";
-  exit(1);
-}
-
-static Flavor getFlavor(StringRef s) {
-  return StringSwitch<Flavor>(s)
-      .CasesLower("ld", "ld.lld", "gnu", Gnu)
-      .CasesLower("wasm", "ld-wasm", Wasm)
-      .CaseLower("link", WinLink)
-      .CasesLower("ld64", "ld64.lld", "darwin", Darwin)
-      .Default(Invalid);
-}
-
-static cl::TokenizerCallback getDefaultQuotingStyle() {
-  if (Triple(sys::getProcessTriple()).getOS() == Triple::Win32)
-    return cl::TokenizeWindowsCommandLine;
-  return cl::TokenizeGNUCommandLine;
-}
-
-static bool isPETargetName(StringRef s) {
-  return s == "i386pe" || s == "i386pep" || s == "thumb2pe" || s == "arm64pe";
-}
-
-static bool isPETarget(std::vector<const char *> &v) {
-  for (auto it = v.begin(); it + 1 != v.end(); ++it) {
-    if (StringRef(*it) != "-m")
-      continue;
-    return isPETargetName(*(it + 1));
-  }
-  // Expand response files (arguments in the form of @<filename>)
-  // to allow detecting the -m argument from arguments in them.
-  SmallVector<const char *, 256> expandedArgs(v.data(), v.data() + v.size());
-  BumpPtrAllocator a;
-  StringSaver saver(a);
-  cl::ExpansionContext ECtx(saver.getAllocator(), getDefaultQuotingStyle());
-  if (Error Err = ECtx.expandResponseFiles(expandedArgs))
-    die(toString(std::move(Err)));
-  for (auto it = expandedArgs.begin(); it + 1 != expandedArgs.end(); ++it) {
-    if (StringRef(*it) != "-m")
-      continue;
-    return isPETargetName(*(it + 1));
-  }
-
-#ifdef LLD_DEFAULT_LD_LLD_IS_MINGW
-  return true;
-#else
-  return false;
-#endif
-}
-
-static Flavor parseProgname(StringRef progname) {
-  // Use GNU driver for "ld" by default.
-  if (progname == "ld")
-    return Gnu;
-
-  // Progname may be something like "lld-gnu". Parse it.
-  SmallVector<StringRef, 3> v;
-  progname.split(v, "-");
-  for (StringRef s : v)
-    if (Flavor f = getFlavor(s))
-      return f;
-  return Invalid;
-}
-
-static Flavor parseFlavor(std::vector<const char *> &v) {
-  // Parse -flavor option.
-  if (v.size() > 1 && v[1] == StringRef("-flavor")) {
-    if (v.size() <= 2)
-      die("missing arg value for '-flavor'");
-    Flavor f = getFlavor(v[2]);
-    if (f == Invalid)
-      die("Unknown flavor: " + StringRef(v[2]));
-    v.erase(v.begin() + 1, v.begin() + 3);
-    return f;
-  }
-
-  // Deduct the flavor from argv[0].
-  StringRef arg0 = path::filename(v[0]);
-  if (arg0.ends_with_insensitive(".exe"))
-    arg0 = arg0.drop_back(4);
-  return parseProgname(arg0);
-}
-
-bool inTestOutputDisabled = false;
-
-/// Universal linker main(). This linker emulates the gnu, darwin, or
-/// windows linker based on the argv[0] or -flavor option.
-static int lldMain(int argc, const char **argv, llvm::raw_ostream &stdoutOS,
-                   llvm::raw_ostream &stderrOS, bool exitEarly = true) {
-  std::vector<const char *> args(argv, argv + argc);
-  auto link = [&args]() {
-    Flavor f = parseFlavor(args);
-    if (f == Gnu && isPETarget(args))
-      return mingw::link;
-    else if (f == Gnu)
-      return elf::link;
-    else if (f == WinLink)
-      return coff::link;
-    else if (f == Darwin)
-      return macho::link;
-    else if (f == Wasm)
-      return lld::wasm::link;
-    else
-      die("lld is a generic driver.\n"
-          "Invoke ld.lld (Unix), ld64.lld (macOS), lld-link (Windows), wasm-ld"
-          " (WebAssembly) instead");
-  }();
-  // Run the driver. If an error occurs, false will be returned.
-  bool r = link(args, stdoutOS, stderrOS, exitEarly, inTestOutputDisabled);
-
-  // Call exit() if we can to avoid calling destructors.
-  if (exitEarly)
-    exitLld(!r ? 1 : 0);
-
-  // Delete the global context and clear the global context pointer, so that it
-  // cannot be accessed anymore.
-  CommonLinkerContext::destroy();
-
-  return !r ? 1 : 0;
-}
-
-// Similar to lldMain except that exceptions are caught.
-SafeReturn lld::safeLldMain(int argc, const char **argv,
-                            llvm::raw_ostream &stdoutOS,
-                            llvm::raw_ostream &stderrOS) {
-  int r = 0;
-  {
-    // The crash recovery is here only to be able to recover from arbitrary
-    // control flow when fatal() is called (through setjmp/longjmp or
-    // __try/__except).
-    llvm::CrashRecoveryContext crc;
-    if (!crc.RunSafely([&]() {
-          r = lldMain(argc, argv, stdoutOS, stderrOS, /*exitEarly=*/false);
-        }))
-      return {crc.RetCode, /*canRunAgain=*/false};
-  }
-
-  // Cleanup memory and reset everything back in pristine condition. This path
-  // is only taken when LLD is in test, or when it is used as a library.
-  llvm::CrashRecoveryContext crc;
-  if (!crc.RunSafely([&]() { CommonLinkerContext::destroy(); })) {
-    // The memory is corrupted beyond any possible recovery.
-    return {r, /*canRunAgain=*/false};
-  }
-  return {r, /*canRunAgain=*/true};
-}
+// Bypass the crash recovery handler, which is only meant to be used in
+// LLD-as-lib scenarios.
+int unsafeLldMain(llvm::ArrayRef<const char *> args,
+                  llvm::raw_ostream &stdoutOS, llvm::raw_ostream &stderrOS,
+                  llvm::ArrayRef<DriverDef> drivers, bool exitEarly);
+} // namespace lld
 
 // When in lit tests, tells how many times the LLD tool should re-execute the
 // main loop with the same inputs. When not in test, returns a value of 0 which
@@ -215,6 +68,12 @@ static unsigned inTestVerbosity() {
   return v;
 }
 
+LLD_HAS_DRIVER(coff)
+LLD_HAS_DRIVER(elf)
+LLD_HAS_DRIVER(mingw)
+LLD_HAS_DRIVER(macho)
+LLD_HAS_DRIVER(wasm)
+
 int lld_main(int argc, char **argv, const llvm::ToolContext &) {
   InitLLVM x(argc, argv);
   sys::Process::UseANSIEscapeCodes(true);
@@ -225,11 +84,16 @@ int lld_main(int argc, char **argv, const llvm::ToolContext &) {
     LLVM_BUILTIN_TRAP;
   }
 
+  ArrayRef<const char *> args(argv, argv + argc);
+
   // Not running in lit tests, just take the shortest codepath with global
   // exception handling and no memory cleanup on exit.
-  if (!inTestVerbosity())
-    return lldMain(argc, const_cast<const char **>(argv), llvm::outs(),
-                   llvm::errs());
+  if (!inTestVerbosity()) {
+    int r =
+        lld::unsafeLldMain(args, llvm::outs(), llvm::errs(), LLD_ALL_DRIVERS,
+                           /*exitEarly=*/true);
+    return r;
+  }
 
   std::optional<int> mainRet;
   CrashRecoveryContext::Enable();
@@ -239,16 +103,15 @@ int lld_main(int argc, char **argv, const llvm::ToolContext &) {
     inTestOutputDisabled = (i != 1);
 
     // Execute one iteration.
-    auto r = safeLldMain(argc, const_cast<const char **>(argv), llvm::outs(),
-                         llvm::errs());
+    auto r = lldMain(args, llvm::outs(), llvm::errs(), LLD_ALL_DRIVERS);
     if (!r.canRunAgain)
-      exitLld(r.ret); // Exit now, can't re-execute again.
+      exitLld(r.retCode); // Exit now, can't re-execute again.
 
     if (!mainRet) {
-      mainRet = r.ret;
-    } else if (r.ret != *mainRet) {
+      mainRet = r.retCode;
+    } else if (r.retCode != *mainRet) {
       // Exit now, to fail the tests if the result is different between runs.
-      return r.ret;
+      return r.retCode;
     }
   }
   return *mainRet;
diff --git a/lld/unittests/AsLibAll/AllDrivers.cpp b/lld/unittests/AsLibAll/AllDrivers.cpp
new file mode 100644
index 0000000000000..fef3d8ca6f016
--- /dev/null
+++ b/lld/unittests/AsLibAll/AllDrivers.cpp
@@ -0,0 +1,35 @@
+//===- AllDrivers.cpp -------------------------------------------*- C++ -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// This test shows a typical case where all LLD drivers are linked into the
+// application binary. This is very similar to how lld.exe binary is linked,
+// except that here we cleanup the internal LLD memory context after each call.
+//===----------------------------------------------------------------------===//
+
+#include "lld/Common/Driver.h"
+#include "gmock/gmock.h"
+
+LLD_HAS_DRIVER(coff)
+LLD_HAS_DRIVER(elf)
+LLD_HAS_DRIVER(mingw)
+LLD_HAS_DRIVER(macho)
+LLD_HAS_DRIVER(wasm)
+
+static bool lldInvoke(std::vector<const char *> args) {
+  args.push_back("--version");
+  lld::Result r =
+      lld::lldMain(args, llvm::outs(), llvm::errs(), LLD_ALL_DRIVERS);
+  return !r.retCode && r.canRunAgain;
+}
+
+TEST(AsLib, AllDrivers) {
+  EXPECT_TRUE(lldInvoke({"ld.lld"}));
+  EXPECT_TRUE(lldInvoke({"ld64.lld"}));
+  EXPECT_TRUE(lldInvoke({"ld", "-m", "i386pe"})); // MinGW
+  EXPECT_TRUE(lldInvoke({"lld-link"}));
+  EXPECT_TRUE(lldInvoke({"wasm-ld"}));
+}
diff --git a/lld/unittests/AsLibAll/CMakeLists.txt b/lld/unittests/AsLibAll/CMakeLists.txt
new file mode 100644
index 0000000000000..a88dc3eb854d4
--- /dev/null
+++ b/lld/unittests/AsLibAll/CMakeLists.txt
@@ -0,0 +1,17 @@
+# Test usage of LLD as a library.
+# This test covers a typical case where all the LLD drivers are pulled into the
+# application executable.
+
+add_lld_unittests(LLDAsLibAllTests
+  AllDrivers.cpp
+)
+
+target_link_libraries(LLDAsLibAllTests
+  PRIVATE
+  lldCommon
+  lldCOFF
+  lldELF
+  lldMachO
+  lldMinGW
+  lldWasm
+)
diff --git a/lld/unittests/AsLibELF/CMakeLists.txt b/lld/unittests/AsLibELF/CMakeLists.txt
new file mode 100644
index 0000000000000..cffd9da7006ea
--- /dev/null
+++ b/lld/unittests/AsLibELF/CMakeLists.txt
@@ -0,0 +1,14 @@
+# Test usage of LLD as a library.
+# This test covers a more singular case where only one LLD driver is used in the
+# target application executable.
+
+add_lld_unittests(LLDAsLibELFTests
+  ROCm.cpp
+  SomeDrivers.cpp
+)
+
+target_link_libraries(LLDAsLibELFTests
+  PRIVATE
+  lldCommon
+  lldELF
+)
diff --git a/lld/unittests/AsLibELF/Inputs/kernel1.o b/lld/unittests/AsLibELF/Inputs/kernel1.o
new file mode 100644
index 0000000000000000000000000000000000000000..ba9b3056d3af37ff8512f2efb44405339adce780
GIT binary patch
literal 1688
zcmd5+y>8S%5FY!EV7dTJ%1BTg#jPT$&>0C5iUiT5L$dbHYh#`Nw7Yk?OF;rj848e)
zc!B7UPPr%rh=?M20p5TjkHW0Ib{uo!kZ4$GHT%tcJ3BL;@$S=&HPbL`6Y^!iB^df#
z6`z_pTDbvWXGa^Xnat{~|Nm^l!i9xlz6JCC?#1D`VK!fbX+GR*Hdg`CpLs3boX^Xo
zHIJa}Hk`mAo^NS<6XhBfe;MWD%_r+GwgC30G;&`FN_vdPEEMnG9TCcX@$q0vsOxej
zgd<YgWn-Vkj0fE@Njy(52}eW(JItDp0S)`i=>@TS?1xE*hEAvNd5q8VV9oWgUp%%r
zgZjBMa<ZpO*3w_GmjA>$BR)_1sUw&lu~<4@Fl4Ty5{0i>iLf@Mk%oaETVH<DD|+i2
ziRjSrLMol@g!i;twEQrzbTN(CB%(p=<aPf}QXUMbWTo@U%M&8}lsny|A4}^r2O<r^
zFjv8th*8u@!fC}J>ACo2%4?0&98|BJkZn3(9#1e&6|#M-TR~!yOr9a~yKx(Z+SnuQ
zw(kuet*jO;Ghkc<N&LZ>HN?8-lu&(lr%_BjK+(SsP=6a5ijpl`%SA15{IjVqFc8Ux
z5-M2&i4)1l`tz-}khI&&ag~8}03sxpw1aXKWs<Xy5}vqJQvK9F2^`)^7FVup{U5_I
zySD1(QdsSUYb>oqQPit6*BoqW>Q#kvrmcB)c~R?Y_!(jqs$J8!bo@(NufC)DT9(Fz
x{w`u=tFOIUoQ1uvzJ~^NzaD>a|7Fdf!z$cman~rB?NYav%1<rzuG;nZe*wVPAlm={

literal 0
HcmV?d00001

diff --git a/lld/unittests/AsLibELF/Inputs/kernel2.o b/lld/unittests/AsLibELF/Inputs/kernel2.o
new file mode 100644
index 0000000000000000000000000000000000000000..ba9b3056d3af37ff8512f2efb44405339adce780
GIT binary patch
literal 1688
zcmd5+y>8S%5FY!EV7dTJ%1BTg#jPT$&>0C5iUiT5L$dbHYh#`Nw7Yk?OF;rj848e)
zc!B7UPPr%rh=?M20p5TjkHW0Ib{uo!kZ4$GHT%tcJ3BL;@$S=&HPbL`6Y^!iB^df#
z6`z_pTDbvWXGa^Xnat{~|Nm^l!i9xlz6JCC?#1D`VK!fbX+GR*Hdg`CpLs3boX^Xo
zHIJa}Hk`mAo^NS<6XhBfe;MWD%_r+GwgC30G;&`FN_vdPEEMnG9TCcX@$q0vsOxej
zgd<YgWn-Vkj0fE@Njy(52}eW(JItDp0S)`i=>@TS?1xE*hEAvNd5q8VV9oWgUp%%r
zgZjBMa<ZpO*3w_GmjA>$BR)_1sUw&lu~<4@Fl4Ty5{0i>iLf@Mk%oaETVH<DD|+i2
ziRjSrLMol@g!i;twEQrzbTN(CB%(p=<aPf}QXUMbWTo@U%M&8}lsny|A4}^r2O<r^
zFjv8th*8u@!fC}J>ACo2%4?0&98|BJkZn3(9#1e&6|#M-TR~!yOr9a~yKx(Z+SnuQ
zw(kuet*jO;Ghkc<N&LZ>HN?8-lu&(lr%_BjK+(SsP=6a5ijpl`%SA15{IjVqFc8Ux
z5-M2&i4)1l`tz-}khI&&ag~8}03sxpw1aXKWs<Xy5}vqJQvK9F2^`)^7FVup{U5_I
zySD1(QdsSUYb>oqQPit6*BoqW>Q#kvrmcB)c~R?Y_!(jqs$J8!bo@(NufC)DT9(Fz
x{w`u=tFOIUoQ1uvzJ~^NzaD>a|7Fdf!z$cman~rB?NYav%1<rzuG;nZe*wVPAlm={

literal 0
HcmV?d00001

diff --git a/lld/unittests/AsLibELF/ROCm.cpp b/lld/unittests/AsLibELF/ROCm.cpp
new file mode 100644
index 0000000000000..00bde87ad78d7
--- /dev/null
+++ b/lld/unittests/AsLibELF/ROCm.cpp
@@ -0,0 +1,74 @@
+//===- ROCm.cpp -------------------------------------------------*- C++ -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// The purpose of this test is to showcase a more singular usage of LLD as a
+// library, where only one LLD driver is being used (and linked in the target
+// application). We also expect that linking twice the same object files
+// would yield a successfull result. When used as library, LLD always cleans its
+// internal memory context after each linker call.
+//===----------------------------------------------------------------------===//
+
+// When this flag is on, we actually need the MinGW driver library, not the
+// ELF one. Here our test only covers the case where the ELF driver is linked
+// into the unit test binary.
+#ifndef LLD_DEFAULT_LD_LLD_IS_MINGW
+
+#include "lld/Common/Driver.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FileUtilities.h"
+#include "llvm/Support/Path.h"
+#include "gmock/gmock.h"
+#include <algorithm>
+
+static std::string expand(const char *path) {
+  if (!llvm::StringRef(path).contains("%"))
+    return std::string(path);
+
+  llvm::SmallString<256> thisPath;
+  thisPath.append(getenv("LLD_SRC_DIR"));
+  llvm::sys::path::append(thisPath, "unittests", "AsLibELF");
+
+  std::string expanded(path);
+  expanded.replace(expanded.find("%S"), 2, thisPath.data(), thisPath.size());
+  return expanded;
+}
+
+LLD_HAS_DRIVER(elf)
+
+static bool lldInvoke(const char *inPath, const char *outPath) {
+  std::vector<const char *> args{"ld.lld", "-shared", inPath, "-o", outPath};
+  lld::Result s = lld::lldMain(args, llvm::outs(), llvm::errs(),
+                               {{lld::Gnu, &lld::elf::link}});
+  return !s.retCode && s.canRunAgain;
+}
+
+static bool runLinker(const char *path) {
+  // Create a temp file for HSA code object.
+  int tempHsacoFD = -1;
+  llvm::SmallString<128> tempHsacoFilename;
+  if (llvm::sys::fs::createTemporaryFile("kernel", "hsaco", tempHsacoFD,
+                                         tempHsacoFilename)) {
+    return false;
+  }
+  llvm::FileRemover cleanupHsaco(tempHsacoFilename);
+  // Invoke lld. Expect a true return value from lld.
+  std::string expandedPath = expand(path);
+  if (!lldInvoke(expandedPath.data(), tempHsacoFilename.c_str())) {
+    llvm::errs() << "Failed to link: " << expandedPath << "\n";
+    return false;
+  }
+  return true;
+}
+
+TEST(AsLib, ROCm) {
+  EXPECT_TRUE(runLinker("%S/Inputs/kernel1.o"));
+  EXPECT_TRUE(runLinker("%S/Inputs/kernel2.o"));
+  EXPECT_TRUE(runLinker("%S/Inputs/kernel1.o"));
+  EXPECT_TRUE(runLinker("%S/Inputs/kernel2.o"));
+}
+#endif
diff --git a/lld/unittests/AsLibELF/SomeDrivers.cpp b/lld/unittests/AsLibELF/SomeDrivers.cpp
new file mode 100644
index 0000000000000..05027541a3af0
--- /dev/null
+++ b/lld/unittests/AsLibELF/SomeDrivers.cpp
@@ -0,0 +1,37 @@
+//===- SomeDrivers.cpp ------------------------------------------*- C++ -*-===//
+//
+// This file is licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+// In this test we showcase the fact that only one LLD driver can be invoked -
+// the ELF driver that was linked in the test binary. Calling other drivers
+// would return a failure. When using LLD as a library, any driver can be
+// linked into your application.
+//===----------------------------------------------------------------------===//
+
+#include "lld/Common/Driver.h"
+#include "gmock/gmock.h"
+
+LLD_HAS_DRIVER(elf)
+
+static bool lldInvoke(const char *lldExe) {
+  std::vector<const char *> args{lldExe, "--version"};
+  lld::Result s = lld::lldMain(args, llvm::outs(), llvm::errs(),
+                               {{lld::Gnu, &lld::elf::link}});
+  return !s.retCode && s.canRunAgain;
+}
+
+TEST(AsLib, SomeDrivers) {
+  // When this flag is on, we actually need the MinGW driver library, not the
+  // ELF one. Here our test only covers the case where the ELF driver is linked
+  // into the unit test binary.
+#ifndef LLD_DEFAULT_LD_LLD_IS_MINGW
+  EXPECT_TRUE(lldInvoke("ld.lld")); // ELF
+#endif
+  // These drivers are not linked in this unit test.
+  EXPECT_FALSE(lldInvoke("ld64.lld")); // Mach-O
+  EXPECT_FALSE(lldInvoke("lld-link")); // COFF
+  EXPECT_FALSE(lldInvoke("wasm-ld"));  // Wasm
+}
diff --git a/lld/unittests/CMakeLists.txt b/lld/unittests/CMakeLists.txt
new file mode 100644
index 0000000000000..ac878fa019083
--- /dev/null
+++ b/lld/unittests/CMakeLists.txt
@@ -0,0 +1,8 @@
+set_target_properties(LLDUnitTests PROPERTIES FOLDER "lld tests")
+
+function(add_lld_unittests test_dirname)
+  add_unittest(LLDUnitTests ${test_dirname} ${ARGN})
+endfunction()
+
+add_subdirectory(AsLibAll)
+add_subdirectory(AsLibELF)
diff --git a/lld/wasm/Driver.cpp b/lld/wasm/Driver.cpp
index dcb76886d3a43..8e2c93772abda 100644
--- a/lld/wasm/Driver.cpp
+++ b/lld/wasm/Driver.cpp
@@ -84,7 +84,7 @@ class LinkerDriver {
 
 bool link(ArrayRef<const char *> args, llvm::raw_ostream &stdoutOS,
           llvm::raw_ostream &stderrOS, bool exitEarly, bool disableOutput) {
-  // This driver-specific context will be freed later by lldMain().
+  // This driver-specific context will be freed later by unsafeLldMain().
   auto *ctx = new CommonLinkerContext;
 
   ctx->e.initialize(stdoutOS, stderrOS, exitEarly, disableOutput);
diff --git a/llvm/cmake/modules/AddLLVM.cmake b/llvm/cmake/modules/AddLLVM.cmake
index bc3deef614458..926336e4a7aa8 100644
--- a/llvm/cmake/modules/AddLLVM.cmake
+++ b/llvm/cmake/modules/AddLLVM.cmake
@@ -2469,3 +2469,13 @@ function(setup_host_tool tool_name setting_name exe_var_name target_var_name)
     add_custom_target(${target_var_name} DEPENDS ${exe_name})
   endif()
 endfunction()
+
+# Adds the unittests folder if gtest is available.
+function(llvm_add_unittests tests_added)
+  if (EXISTS ${LLVM_THIRD_PARTY_DIR}/unittest/googletest/include/gtest/gtest.h)
+    add_subdirectory(unittests)
+    set(${tests_added} ON PARENT_SCOPE)
+  else()
+    message(WARNING "gtest not found, unittests will not be available")
+  endif()
+endfunction()

From 7e229217f4215b519b886e7881bae4da3742a7d2 Mon Sep 17 00:00:00 2001
From: David Spickett <david.spickett@linaro.org>
Date: Mon, 19 Jun 2023 11:52:06 +0100
Subject: [PATCH 012/130] [lldb][AArch64] Add thread local storage tpidr
 register

This register is used as the pointer to the current thread
local storage block and is read from NT_ARM_TLS on Linux.

Though tpidr will be present on all AArch64 Linux, I am soon
going to add a second register tpidr2 to this set.

tpidr is only present when SME is implemented, therefore the
NT_ARM_TLS set will change size. This is why I've added this
as a dynamic register set to save changes later.

Reviewed By: omjavaid

Differential Revision: https://reviews.llvm.org/D152516
---
 .../NativeRegisterContextLinux_arm64.cpp      | 71 ++++++++++++++++++-
 .../Linux/NativeRegisterContextLinux_arm64.h  | 12 ++++
 .../Utility/RegisterInfoPOSIX_arm64.cpp       | 34 +++++++++
 .../Process/Utility/RegisterInfoPOSIX_arm64.h |  6 ++
 .../API/linux/aarch64/tls_register/Makefile   |  3 +
 .../TestAArch64LinuxTLSRegister.py            | 66 +++++++++++++++++
 .../API/linux/aarch64/tls_register/main.c     | 24 +++++++
 7 files changed, 215 insertions(+), 1 deletion(-)
 create mode 100644 lldb/test/API/linux/aarch64/tls_register/Makefile
 create mode 100644 lldb/test/API/linux/aarch64/tls_register/TestAArch64LinuxTLSRegister.py
 create mode 100644 lldb/test/API/linux/aarch64/tls_register/main.c

diff --git a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp
index 0ec152f0643a4..c9384a651e81b 100644
--- a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp
+++ b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.cpp
@@ -86,6 +86,8 @@ NativeRegisterContextLinux::CreateHostNativeRegisterContextLinux(
     if (auxv_at_hwcap2 && (*auxv_at_hwcap2 & HWCAP2_MTE))
       opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskMTE);
 
+    opt_regsets.Set(RegisterInfoPOSIX_arm64::eRegsetMaskTLS);
+
     auto register_info_up =
         std::make_unique<RegisterInfoPOSIX_arm64>(target_arch, opt_regsets);
     return std::make_unique<NativeRegisterContextLinux_arm64>(
@@ -116,6 +118,7 @@ NativeRegisterContextLinux_arm64::NativeRegisterContextLinux_arm64(
   ::memset(&m_pac_mask, 0, sizeof(m_pac_mask));
 
   m_mte_ctrl_reg = 0;
+  m_tls_tpidr_reg = 0;
 
   // 16 is just a maximum value, query hardware for actual watchpoint count
   m_max_hwp_supported = 16;
@@ -129,6 +132,7 @@ NativeRegisterContextLinux_arm64::NativeRegisterContextLinux_arm64(
   m_sve_header_is_valid = false;
   m_pac_mask_is_valid = false;
   m_mte_ctrl_is_valid = false;
+  m_tls_tpidr_is_valid = false;
 
   if (GetRegisterInfo().IsSVEEnabled())
     m_sve_state = SVEState::Unknown;
@@ -232,8 +236,15 @@ NativeRegisterContextLinux_arm64::ReadRegister(const RegisterInfo *reg_info,
       assert(offset < GetSVEBufferSize());
       src = (uint8_t *)GetSVEBuffer() + offset;
     }
-  } else if (IsSVE(reg)) {
+  } else if (IsTLS(reg)) {
+    error = ReadTLSTPIDR();
+    if (error.Fail())
+      return error;
 
+    offset = reg_info->byte_offset - GetRegisterInfo().GetTLSOffset();
+    assert(offset < GetTLSTPIDRSize());
+    src = (uint8_t *)GetTLSTPIDR() + offset;
+  } else if (IsSVE(reg)) {
     if (m_sve_state == SVEState::Disabled || m_sve_state == SVEState::Unknown)
       return Status("SVE disabled or not supported");
 
@@ -450,6 +461,17 @@ Status NativeRegisterContextLinux_arm64::WriteRegister(
     ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
 
     return WriteMTEControl();
+  } else if (IsTLS(reg)) {
+    error = ReadTLSTPIDR();
+    if (error.Fail())
+      return error;
+
+    offset = reg_info->byte_offset - GetRegisterInfo().GetTLSOffset();
+    assert(offset < GetTLSTPIDRSize());
+    dst = (uint8_t *)GetTLSTPIDR() + offset;
+    ::memcpy(dst, reg_value.GetBytes(), reg_info->byte_size);
+
+    return WriteTLSTPIDR();
   }
 
   return Status("Failed to write register value");
@@ -490,6 +512,12 @@ Status NativeRegisterContextLinux_arm64::ReadAllRegisterValues(
       return error;
   }
 
+  // tpidr is always present but there will be more in future.
+  reg_data_byte_size += GetTLSTPIDRSize();
+  error = ReadTLSTPIDR();
+  if (error.Fail())
+    return error;
+
   data_sp.reset(new DataBufferHeap(reg_data_byte_size, 0));
   uint8_t *dst = data_sp->GetBytes();
 
@@ -507,6 +535,8 @@ Status NativeRegisterContextLinux_arm64::ReadAllRegisterValues(
   if (GetRegisterInfo().IsMTEEnabled())
     ::memcpy(dst, GetMTEControl(), GetMTEControlSize());
 
+  ::memcpy(dst, GetTLSTPIDR(), GetTLSTPIDRSize());
+
   return error;
 }
 
@@ -641,6 +671,10 @@ bool NativeRegisterContextLinux_arm64::IsMTE(unsigned reg) const {
   return GetRegisterInfo().IsMTEReg(reg);
 }
 
+bool NativeRegisterContextLinux_arm64::IsTLS(unsigned reg) const {
+  return GetRegisterInfo().IsTLSReg(reg);
+}
+
 llvm::Error NativeRegisterContextLinux_arm64::ReadHardwareDebugInfo() {
   if (!m_refresh_hwdebug_info) {
     return llvm::Error::success();
@@ -784,6 +818,7 @@ void NativeRegisterContextLinux_arm64::InvalidateAllRegisters() {
   m_sve_header_is_valid = false;
   m_pac_mask_is_valid = false;
   m_mte_ctrl_is_valid = false;
+  m_tls_tpidr_is_valid = false;
 
   // Update SVE registers in case there is change in configuration.
   ConfigureRegisterContext();
@@ -914,6 +949,40 @@ Status NativeRegisterContextLinux_arm64::WriteMTEControl() {
   return WriteRegisterSet(&ioVec, GetMTEControlSize(), NT_ARM_TAGGED_ADDR_CTRL);
 }
 
+Status NativeRegisterContextLinux_arm64::ReadTLSTPIDR() {
+  Status error;
+
+  if (m_tls_tpidr_is_valid)
+    return error;
+
+  struct iovec ioVec;
+  ioVec.iov_base = GetTLSTPIDR();
+  ioVec.iov_len = GetTLSTPIDRSize();
+
+  error = ReadRegisterSet(&ioVec, GetTLSTPIDRSize(), NT_ARM_TLS);
+
+  if (error.Success())
+    m_tls_tpidr_is_valid = true;
+
+  return error;
+}
+
+Status NativeRegisterContextLinux_arm64::WriteTLSTPIDR() {
+  Status error;
+
+  error = ReadTLSTPIDR();
+  if (error.Fail())
+    return error;
+
+  struct iovec ioVec;
+  ioVec.iov_base = GetTLSTPIDR();
+  ioVec.iov_len = GetTLSTPIDRSize();
+
+  m_tls_tpidr_is_valid = false;
+
+  return WriteRegisterSet(&ioVec, GetTLSTPIDRSize(), NT_ARM_TLS);
+}
+
 void NativeRegisterContextLinux_arm64::ConfigureRegisterContext() {
   // ConfigureRegisterContext gets called from InvalidateAllRegisters
   // on every stop and configures SVE vector length.
diff --git a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.h b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.h
index 14f669a3ce98f..2b5442578cd27 100644
--- a/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.h
+++ b/lldb/source/Plugins/Process/Linux/NativeRegisterContextLinux_arm64.h
@@ -83,6 +83,7 @@ class NativeRegisterContextLinux_arm64
   bool m_fpu_is_valid;
   bool m_sve_buffer_is_valid;
   bool m_mte_ctrl_is_valid;
+  bool m_tls_tpidr_is_valid;
 
   bool m_sve_header_is_valid;
   bool m_pac_mask_is_valid;
@@ -107,6 +108,8 @@ class NativeRegisterContextLinux_arm64
 
   uint64_t m_mte_ctrl_reg;
 
+  uint64_t m_tls_tpidr_reg;
+
   bool IsGPR(unsigned reg) const;
 
   bool IsFPR(unsigned reg) const;
@@ -125,9 +128,14 @@ class NativeRegisterContextLinux_arm64
 
   Status WriteMTEControl();
 
+  Status ReadTLSTPIDR();
+
+  Status WriteTLSTPIDR();
+
   bool IsSVE(unsigned reg) const;
   bool IsPAuth(unsigned reg) const;
   bool IsMTE(unsigned reg) const;
+  bool IsTLS(unsigned reg) const;
 
   uint64_t GetSVERegVG() { return m_sve_header.vl / 8; }
 
@@ -139,6 +147,8 @@ class NativeRegisterContextLinux_arm64
 
   void *GetMTEControl() { return &m_mte_ctrl_reg; }
 
+  void *GetTLSTPIDR() { return &m_tls_tpidr_reg; }
+
   void *GetSVEBuffer() { return m_sve_ptrace_payload.data(); };
 
   size_t GetSVEHeaderSize() { return sizeof(m_sve_header); }
@@ -149,6 +159,8 @@ class NativeRegisterContextLinux_arm64
 
   size_t GetMTEControlSize() { return sizeof(m_mte_ctrl_reg); }
 
+  size_t GetTLSTPIDRSize() { return sizeof(m_tls_tpidr_reg); }
+
   llvm::Error ReadHardwareDebugInfo() override;
 
   llvm::Error WriteHardwareDebugRegs(DREGType hwbType) override;
diff --git a/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.cpp b/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.cpp
index 497f0b4114dfb..af5bbda7bfcf5 100644
--- a/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.cpp
+++ b/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.cpp
@@ -78,12 +78,16 @@ static lldb_private::RegisterInfo g_register_infos_pauth[] = {
 static lldb_private::RegisterInfo g_register_infos_mte[] = {
     DEFINE_EXTENSION_REG(mte_ctrl)};
 
+static lldb_private::RegisterInfo g_register_infos_tls[] = {
+    DEFINE_EXTENSION_REG(tpidr)};
+
 // Number of register sets provided by this context.
 enum {
   k_num_gpr_registers = gpr_w28 - gpr_x0 + 1,
   k_num_fpr_registers = fpu_fpcr - fpu_v0 + 1,
   k_num_sve_registers = sve_ffr - sve_vg + 1,
   k_num_mte_register = 1,
+  k_num_tls_register = 1,
   k_num_pauth_register = 2,
   k_num_register_sets_default = 2,
   k_num_register_sets = 3
@@ -189,6 +193,9 @@ static const lldb_private::RegisterSet g_reg_set_pauth_arm64 = {
 static const lldb_private::RegisterSet g_reg_set_mte_arm64 = {
     "MTE Control Register", "mte", k_num_mte_register, nullptr};
 
+static const lldb_private::RegisterSet g_reg_set_tls_arm64 = {
+    "Thread Local Storage Registers", "tls", k_num_tls_register, nullptr};
+
 RegisterInfoPOSIX_arm64::RegisterInfoPOSIX_arm64(
     const lldb_private::ArchSpec &target_arch, lldb_private::Flags opt_regsets)
     : lldb_private::RegisterInfoAndSetInterface(target_arch),
@@ -229,6 +236,10 @@ RegisterInfoPOSIX_arm64::RegisterInfoPOSIX_arm64(
       if (m_opt_regsets.AllSet(eRegsetMaskMTE))
         AddRegSetMTE();
 
+      // tpidr is always present, but in future there will be others so this is
+      // done as a dynamic set.
+      AddRegSetTLS();
+
       m_register_info_count = m_dynamic_reg_infos.size();
       m_register_info_p = m_dynamic_reg_infos.data();
       m_register_set_p = m_dynamic_reg_sets.data();
@@ -312,6 +323,21 @@ void RegisterInfoPOSIX_arm64::AddRegSetMTE() {
   m_dynamic_reg_sets.back().registers = m_mte_regnum_collection.data();
 }
 
+void RegisterInfoPOSIX_arm64::AddRegSetTLS() {
+  uint32_t tls_regnum = m_dynamic_reg_infos.size();
+  m_tls_regnum_collection.push_back(tls_regnum);
+  m_dynamic_reg_infos.push_back(g_register_infos_tls[0]);
+  m_dynamic_reg_infos[tls_regnum].byte_offset =
+      m_dynamic_reg_infos[tls_regnum - 1].byte_offset +
+      m_dynamic_reg_infos[tls_regnum - 1].byte_size;
+  m_dynamic_reg_infos[tls_regnum].kinds[lldb::eRegisterKindLLDB] = tls_regnum;
+
+  m_per_regset_regnum_range[m_register_set_count] =
+      std::make_pair(tls_regnum, tls_regnum + 1);
+  m_dynamic_reg_sets.push_back(g_reg_set_tls_arm64);
+  m_dynamic_reg_sets.back().registers = m_tls_regnum_collection.data();
+}
+
 uint32_t RegisterInfoPOSIX_arm64::ConfigureVectorLength(uint32_t sve_vq) {
   // sve_vq contains SVE Quad vector length in context of AArch64 SVE.
   // SVE register infos if enabled cannot be disabled by selecting sve_vq = 0.
@@ -403,6 +429,10 @@ bool RegisterInfoPOSIX_arm64::IsMTEReg(unsigned reg) const {
   return llvm::is_contained(m_mte_regnum_collection, reg);
 }
 
+bool RegisterInfoPOSIX_arm64::IsTLSReg(unsigned reg) const {
+  return llvm::is_contained(m_tls_regnum_collection, reg);
+}
+
 uint32_t RegisterInfoPOSIX_arm64::GetRegNumSVEZ0() const { return sve_z0; }
 
 uint32_t RegisterInfoPOSIX_arm64::GetRegNumSVEFFR() const { return sve_ffr; }
@@ -420,3 +450,7 @@ uint32_t RegisterInfoPOSIX_arm64::GetPAuthOffset() const {
 uint32_t RegisterInfoPOSIX_arm64::GetMTEOffset() const {
   return m_register_info_p[m_mte_regnum_collection[0]].byte_offset;
 }
+
+uint32_t RegisterInfoPOSIX_arm64::GetTLSOffset() const {
+  return m_register_info_p[m_tls_regnum_collection[0]].byte_offset;
+}
diff --git a/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h b/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h
index 4c52de99fde63..465d3f5b9f3bb 100644
--- a/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h
+++ b/lldb/source/Plugins/Process/Utility/RegisterInfoPOSIX_arm64.h
@@ -28,6 +28,7 @@ class RegisterInfoPOSIX_arm64
     eRegsetMaskSVE = 1,
     eRegsetMaskPAuth = 2,
     eRegsetMaskMTE = 4,
+    eRegsetMaskTLS = 8,
     eRegsetMaskDynamic = ~1,
   };
 
@@ -102,6 +103,8 @@ class RegisterInfoPOSIX_arm64
 
   void AddRegSetMTE();
 
+  void AddRegSetTLS();
+
   uint32_t ConfigureVectorLength(uint32_t sve_vq);
 
   bool VectorSizeIsValid(uint32_t vq) {
@@ -121,6 +124,7 @@ class RegisterInfoPOSIX_arm64
   bool IsSVERegVG(unsigned reg) const;
   bool IsPAuthReg(unsigned reg) const;
   bool IsMTEReg(unsigned reg) const;
+  bool IsTLSReg(unsigned reg) const;
 
   uint32_t GetRegNumSVEZ0() const;
   uint32_t GetRegNumSVEFFR() const;
@@ -129,6 +133,7 @@ class RegisterInfoPOSIX_arm64
   uint32_t GetRegNumSVEVG() const;
   uint32_t GetPAuthOffset() const;
   uint32_t GetMTEOffset() const;
+  uint32_t GetTLSOffset() const;
 
 private:
   typedef std::map<uint32_t, std::vector<lldb_private::RegisterInfo>>
@@ -155,6 +160,7 @@ class RegisterInfoPOSIX_arm64
 
   std::vector<uint32_t> pauth_regnum_collection;
   std::vector<uint32_t> m_mte_regnum_collection;
+  std::vector<uint32_t> m_tls_regnum_collection;
 };
 
 #endif
diff --git a/lldb/test/API/linux/aarch64/tls_register/Makefile b/lldb/test/API/linux/aarch64/tls_register/Makefile
new file mode 100644
index 0000000000000..10495940055b6
--- /dev/null
+++ b/lldb/test/API/linux/aarch64/tls_register/Makefile
@@ -0,0 +1,3 @@
+C_SOURCES := main.c
+
+include Makefile.rules
diff --git a/lldb/test/API/linux/aarch64/tls_register/TestAArch64LinuxTLSRegister.py b/lldb/test/API/linux/aarch64/tls_register/TestAArch64LinuxTLSRegister.py
new file mode 100644
index 0000000000000..d4ab5d70cc764
--- /dev/null
+++ b/lldb/test/API/linux/aarch64/tls_register/TestAArch64LinuxTLSRegister.py
@@ -0,0 +1,66 @@
+"""
+Test lldb's ability to read and write the AArch64 TLS register tpidr.
+"""
+
+import lldb
+from lldbsuite.test.decorators import *
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class AArch64LinuxTLSRegister(TestBase):
+    NO_DEBUG_INFO_TESTCASE = True
+
+    @skipUnlessArch("aarch64")
+    @skipUnlessPlatform(["linux"])
+    def test_tls(self):
+        self.build()
+        self.runCmd("file " + self.getBuildArtifact("a.out"), CURRENT_EXECUTABLE_SET)
+
+        lldbutil.run_break_set_by_file_and_line(
+            self,
+            "main.c",
+            line_number("main.c", "// Set break point at this line."),
+            num_expected_locations=1,
+        )
+
+        lldbutil.run_break_set_by_file_and_line(
+            self,
+            "main.c",
+            line_number("main.c", "// Set break point 2 at this line."),
+            num_expected_locations=1,
+        )
+
+        self.runCmd("run", RUN_SUCCEEDED)
+
+        if self.process().GetState() == lldb.eStateExited:
+            self.fail("Test program failed to run.")
+
+        self.expect(
+            "thread list",
+            STOPPED_DUE_TO_BREAKPOINT,
+            substrs=["stopped", "stop reason = breakpoint"],
+        )
+
+        # Since we can't predict what the value will be, the program has set
+        # a target value for us to find.
+
+        regs = self.thread().GetSelectedFrame().GetRegisters()
+        tls_regs = regs.GetFirstValueByName("Thread Local Storage Registers")
+        self.assertTrue(tls_regs.IsValid(), "No TLS registers found.")
+        tpidr = tls_regs.GetChildMemberWithName("tpidr")
+        self.assertTrue(tpidr.IsValid(), "No tpidr register found.")
+
+        self.assertEqual(tpidr.GetValueAsUnsigned(), 0x1122334455667788)
+
+        # Set our own value for the program to find.
+        self.expect("register write tpidr 0x{:x}".format(0x8877665544332211))
+        self.expect("continue")
+
+        self.expect(
+            "thread list",
+            STOPPED_DUE_TO_BREAKPOINT,
+            substrs=["stopped", "stop reason = breakpoint"],
+        )
+
+        self.expect("p tpidr_was_set", substrs=["true"])
\ No newline at end of file
diff --git a/lldb/test/API/linux/aarch64/tls_register/main.c b/lldb/test/API/linux/aarch64/tls_register/main.c
new file mode 100644
index 0000000000000..7a5fd3f5fcbdf
--- /dev/null
+++ b/lldb/test/API/linux/aarch64/tls_register/main.c
@@ -0,0 +1,24 @@
+#include <stdbool.h>
+#include <stdint.h>
+
+int main() {
+  // Save tpidr to restore later.
+  uint64_t tpidr = 0;
+  __asm__ volatile("mrs %0, tpidr_el0" : "=r"(tpidr));
+
+  // Set a pattern for lldb to find.
+  uint64_t pattern = 0x1122334455667788;
+  __asm__ volatile("msr tpidr_el0, %0" ::"r"(pattern));
+
+  // Set break point at this line.
+  // lldb will now set its own pattern for us to find.
+
+  uint64_t new_tpidr = pattern;
+  __asm__ volatile("mrs %0, tpidr_el0" : "=r"(new_tpidr));
+  volatile bool tpidr_was_set = new_tpidr == 0x8877665544332211;
+
+  // Restore the original.
+  __asm__ volatile("msr tpidr_el0, %0" ::"r"(tpidr));
+
+  return 0; // Set break point 2 at this line.
+}

From a032dc139ddaf4bfccdc4d2dfe073411118cb7e0 Mon Sep 17 00:00:00 2001
From: Akash Banerjee <Akash.Banerjee@amd.com>
Date: Mon, 19 Jun 2023 12:46:15 +0100
Subject: [PATCH 013/130] [MLIR][OpenMP] Refactoring createTargetData in
 OMPIRBuilder

Key changes:
  - Refactor the createTargetData function to make use of the emitOffloadingArrays and emitOffloadingArraysArgument functions to generate code.
  - Added a new emitIfClause helper function to allow handling if clauses in a similar fashion to Clang.
  - Updated the MLIR side of code to account for changes to createTargetData.

Depends on D149872

Reviewed By: jdoerfert

Differential Revision: https://reviews.llvm.org/D146557
---
 .../llvm/Frontend/OpenMP/OMPIRBuilder.h       |  55 ++++-
 llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp     | 219 ++++++++++++++----
 .../Frontend/OpenMPIRBuilderTest.cpp          | 214 +++++++----------
 .../OpenMP/OpenMPToLLVMIRTranslation.cpp      | 162 ++++++-------
 mlir/test/Target/LLVMIR/omptarget-llvm.mlir   | 105 ++++-----
 5 files changed, 419 insertions(+), 336 deletions(-)

diff --git a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
index ea459b2d8fe37..7657ad32e1a5a 100644
--- a/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
+++ b/llvm/include/llvm/Frontend/OpenMP/OMPIRBuilder.h
@@ -1478,6 +1478,25 @@ class OpenMPIRBuilder {
   /// Computes the size of type in bytes.
   Value *getSizeInBytes(Value *BasePtr);
 
+  // Emit a branch from the current block to the Target block only if
+  // the current block has a terminator.
+  void emitBranch(BasicBlock *Target);
+
+  // If BB has no use then delete it and return. Else place BB after the current
+  // block, if possible, or else at the end of the function. Also add a branch
+  // from current block to BB if current block does not have a terminator.
+  void emitBlock(BasicBlock *BB, Function *CurFn, bool IsFinished = false);
+
+  /// Emits code for OpenMP 'if' clause using specified \a BodyGenCallbackTy
+  /// Here is the logic:
+  /// if (Cond) {
+  ///   ThenGen();
+  /// } else {
+  ///   ElseGen();
+  /// }
+  void emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen,
+                    BodyGenCallbackTy ElseGen, InsertPointTy AllocaIP = {});
+
   /// Create the global variable holding the offload mappings information.
   GlobalVariable *createOffloadMaptypes(SmallVectorImpl<uint64_t> &Mappings,
                                         std::string VarName);
@@ -1987,29 +2006,41 @@ class OpenMPIRBuilder {
                                          StringRef EntryFnName,
                                          StringRef EntryFnIDName,
                                          int32_t NumTeams, int32_t NumThreads);
+  /// Type of BodyGen to use for region codegen
+  ///
+  /// Priv: If device pointer privatization is required, emit the body of the
+  /// region here. It will have to be duplicated: with and without
+  /// privatization.
+  /// DupNoPriv: If we need device pointer privatization, we need
+  /// to emit the body of the region with no privatization in the 'else' branch
+  /// of the conditional.
+  /// NoPriv: If we don't require privatization of device
+  /// pointers, we emit the body in between the runtime calls. This avoids
+  /// duplicating the body code.
+  enum BodyGenTy { Priv, DupNoPriv, NoPriv };
 
   /// Generator for '#omp target data'
   ///
   /// \param Loc The location where the target data construct was encountered.
+  /// \param AllocaIP The insertion points to be used for alloca instructions.
   /// \param CodeGenIP The insertion point at which the target directive code
   /// should be placed.
-  /// \param MapTypeFlags BitVector storing the mapType flags for the
-  /// mapOperands.
-  /// \param MapNames Names for the mapOperands.
-  /// \param MapperAllocas Pointers to the AllocInsts for the map clause.
   /// \param IsBegin If true then emits begin mapper call otherwise emits
   /// end mapper call.
   /// \param DeviceID Stores the DeviceID from the device clause.
   /// \param IfCond Value which corresponds to the if clause condition.
-  /// \param ProcessMapOpCB Callback that generates code for the map clause.
-  /// \param BodyGenCB Callback that will generate the region code.
+  /// \param Info Stores all information realted to the Target Data directive.
+  /// \param GenMapInfoCB Callback that populates the MapInfos and returns.
+  /// \param BodyGenCB Optional Callback to generate the region code.
   OpenMPIRBuilder::InsertPointTy createTargetData(
-      const LocationDescription &Loc, OpenMPIRBuilder::InsertPointTy CodeGenIP,
-      SmallVectorImpl<uint64_t> &MapTypeFlags,
-      SmallVectorImpl<Constant *> &MapNames,
-      struct MapperAllocas &MapperAllocas, bool IsBegin, int64_t DeviceID,
-      Value *IfCond, BodyGenCallbackTy ProcessMapOpCB,
-      BodyGenCallbackTy BodyGenCB = {});
+      const LocationDescription &Loc, InsertPointTy AllocaIP,
+      InsertPointTy CodeGenIP, Value *DeviceID, Value *IfCond,
+      TargetDataInfo &Info,
+      function_ref<MapInfosTy &(InsertPointTy CodeGenIP)> GenMapInfoCB,
+      omp::RuntimeFunction *MapperFunc = nullptr,
+      function_ref<InsertPointTy(InsertPointTy CodeGenIP,
+                                 BodyGenTy BodyGenType)>
+          BodyGenCB = nullptr);
 
   using TargetBodyGenCallbackTy = function_ref<InsertPointTy(
       InsertPointTy AllocaIP, InsertPointTy CodeGenIP)>;
diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
index b4d5c256aa1a7..e3aa3910a4178 100644
--- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
+++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp
@@ -4078,73 +4078,121 @@ Constant *OpenMPIRBuilder::registerTargetRegionFunction(
 }
 
 OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::createTargetData(
-    const LocationDescription &Loc, OpenMPIRBuilder::InsertPointTy CodeGenIP,
-    SmallVectorImpl<uint64_t> &MapTypeFlags,
-    SmallVectorImpl<Constant *> &MapNames, struct MapperAllocas &MapperAllocas,
-    bool IsBegin, int64_t DeviceID, Value *IfCond,
-    BodyGenCallbackTy ProcessMapOpCB, BodyGenCallbackTy BodyGenCB) {
+    const LocationDescription &Loc, InsertPointTy AllocaIP,
+    InsertPointTy CodeGenIP, Value *DeviceID, Value *IfCond,
+    TargetDataInfo &Info,
+    function_ref<MapInfosTy &(InsertPointTy CodeGenIP)> GenMapInfoCB,
+    omp::RuntimeFunction *MapperFunc,
+    function_ref<InsertPointTy(InsertPointTy CodeGenIP, BodyGenTy BodyGenType)>
+        BodyGenCB) {
   if (!updateToLocation(Loc))
     return InsertPointTy();
 
   Builder.restoreIP(CodeGenIP);
+  bool IsStandAlone = !BodyGenCB;
 
-  // LLVM utilities like blocks with terminators.
-  // The UI acts as a resume point for code insertion after the BodyGen
-  auto *UI = Builder.CreateUnreachable();
-  if (IfCond) {
-    auto *ThenTI =
-        SplitBlockAndInsertIfThen(IfCond, UI, /* Unreachable */ false);
-    ThenTI->getParent()->setName("omp_if.then");
-    Builder.SetInsertPoint(ThenTI);
-  } else {
-    Builder.SetInsertPoint(UI);
-  }
+  // Generate the code for the opening of the data environment. Capture all the
+  // arguments of the runtime call by reference because they are used in the
+  // closing of the region.
+  auto BeginThenGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+    emitOffloadingArrays(AllocaIP, Builder.saveIP(),
+                         GenMapInfoCB(Builder.saveIP()), Info,
+                         /*IsNonContiguous=*/true);
 
-  ProcessMapOpCB(Builder.saveIP(), Builder.saveIP());
+    TargetDataRTArgs RTArgs;
+    emitOffloadingArraysArgument(Builder, RTArgs, Info);
 
-  uint32_t SrcLocStrSize;
-  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
-  Value *srcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
+    // Emit the number of elements in the offloading arrays.
+    Value *PointerNum = Builder.getInt32(Info.NumberOfPtrs);
 
-  GlobalVariable *MapTypesGV =
-      createOffloadMaptypes(MapTypeFlags, ".offload_maptypes");
-  Value *MapTypesArg = Builder.CreateConstInBoundsGEP2_32(
-      ArrayType::get(Builder.getInt64Ty(), MapTypeFlags.size()), MapTypesGV,
-      /*Idx0=*/0, /*Idx1=*/0);
+    // Source location for the ident struct
+    uint32_t SrcLocStrSize;
+    Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
+    Value *SrcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
+
+    Value *OffloadingArgs[] = {SrcLocInfo,           DeviceID,
+                               PointerNum,           RTArgs.BasePointersArray,
+                               RTArgs.PointersArray, RTArgs.SizesArray,
+                               RTArgs.MapTypesArray, RTArgs.MapNamesArray,
+                               RTArgs.MappersArray};
+
+    if (IsStandAlone) {
+      assert(MapperFunc && "MapperFunc missing for standalone target data");
+      Builder.CreateCall(getOrCreateRuntimeFunctionPtr(*MapperFunc),
+                         OffloadingArgs);
+    } else {
+      Function *BeginMapperFunc = getOrCreateRuntimeFunctionPtr(
+          omp::OMPRTL___tgt_target_data_begin_mapper);
 
-  GlobalVariable *MapNamesGV =
-      createOffloadMapnames(MapNames, ".offload_mapnames");
-  Value *MapNamesArg = Builder.CreateConstInBoundsGEP2_32(
-      ArrayType::get(Builder.getInt8PtrTy(), MapNames.size()), MapNamesGV,
-      /*Idx0=*/0, /*Idx1=*/0);
+      Builder.CreateCall(BeginMapperFunc, OffloadingArgs);
+
+      // If device pointer privatization is required, emit the body of the
+      // region here. It will have to be duplicated: with and without
+      // privatization.
+      Builder.restoreIP(BodyGenCB(Builder.saveIP(), BodyGenTy::Priv));
+    }
+  };
+
+  // If we need device pointer privatization, we need to emit the body of the
+  // region with no privatization in the 'else' branch of the conditional.
+  // Otherwise, we don't have to do anything.
+  auto BeginElseGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+    Builder.restoreIP(BodyGenCB(Builder.saveIP(), BodyGenTy::DupNoPriv));
+  };
+
+  // Generate code for the closing of the data region.
+  auto EndThenGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
+    TargetDataRTArgs RTArgs;
+    emitOffloadingArraysArgument(Builder, RTArgs, Info, /*EmitDebug=*/false,
+                                 /*ForEndCall=*/true);
+
+    // Emit the number of elements in the offloading arrays.
+    Value *PointerNum = Builder.getInt32(Info.NumberOfPtrs);
+
+    // Source location for the ident struct
+    uint32_t SrcLocStrSize;
+    Constant *SrcLocStr = getOrCreateSrcLocStr(Loc, SrcLocStrSize);
+    Value *SrcLocInfo = getOrCreateIdent(SrcLocStr, SrcLocStrSize);
+
+    Value *OffloadingArgs[] = {SrcLocInfo,           DeviceID,
+                               PointerNum,           RTArgs.BasePointersArray,
+                               RTArgs.PointersArray, RTArgs.SizesArray,
+                               RTArgs.MapTypesArray, RTArgs.MapNamesArray,
+                               RTArgs.MappersArray};
+    Function *EndMapperFunc =
+        getOrCreateRuntimeFunctionPtr(omp::OMPRTL___tgt_target_data_end_mapper);
+
+    Builder.CreateCall(EndMapperFunc, OffloadingArgs);
+  };
 
-  Function *beginMapperFunc =
-      getOrCreateRuntimeFunctionPtr(omp::OMPRTL___tgt_target_data_begin_mapper);
-  Function *endMapperFunc =
-      getOrCreateRuntimeFunctionPtr(omp::OMPRTL___tgt_target_data_end_mapper);
+  // We don't have to do anything to close the region if the if clause evaluates
+  // to false.
+  auto EndElseGen = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {};
 
   if (BodyGenCB) {
-    // Create call to start the data region.
-    emitMapperCall(Builder.saveIP(), beginMapperFunc, srcLocInfo, MapTypesArg,
-                   MapNamesArg, MapperAllocas, DeviceID, MapTypeFlags.size());
+    if (IfCond) {
+      emitIfClause(IfCond, BeginThenGen, BeginElseGen, AllocaIP);
+    } else {
+      BeginThenGen(AllocaIP, Builder.saveIP());
+    }
 
-    BodyGenCB(Builder.saveIP(), Builder.saveIP());
+    // If we don't require privatization of device pointers, we emit the body in
+    // between the runtime calls. This avoids duplicating the body code.
+    Builder.restoreIP(BodyGenCB(Builder.saveIP(), BodyGenTy::NoPriv));
 
-    Builder.SetInsertPoint(UI->getParent());
-    // Create call to end the data region.
-    emitMapperCall(Builder.saveIP(), endMapperFunc, srcLocInfo, MapTypesArg,
-                   MapNamesArg, MapperAllocas, DeviceID, MapTypeFlags.size());
+    if (IfCond) {
+      emitIfClause(IfCond, EndThenGen, EndElseGen, AllocaIP);
+    } else {
+      EndThenGen(AllocaIP, Builder.saveIP());
+    }
   } else {
-    emitMapperCall(Builder.saveIP(), IsBegin ? beginMapperFunc : endMapperFunc,
-                   srcLocInfo, MapTypesArg, MapNamesArg, MapperAllocas,
-                   DeviceID, MapTypeFlags.size());
+    if (IfCond) {
+      emitIfClause(IfCond, BeginThenGen, EndElseGen, AllocaIP);
+    } else {
+      BeginThenGen(AllocaIP, Builder.saveIP());
+    }
   }
 
-  // Update the insertion point and remove the terminator we introduced.
-  Builder.SetInsertPoint(UI->getParent());
-  if (IfCond)
-    UI->getParent()->setName("omp_if.end");
-  UI->eraseFromParent();
   return Builder.saveIP();
 }
 
@@ -4668,6 +4716,77 @@ void OpenMPIRBuilder::emitOffloadingArrays(
   emitNonContiguousDescriptor(AllocaIP, CodeGenIP, CombinedInfo, Info);
 }
 
+void OpenMPIRBuilder::emitBranch(BasicBlock *Target) {
+  BasicBlock *CurBB = Builder.GetInsertBlock();
+
+  if (!CurBB || CurBB->getTerminator()) {
+    // If there is no insert point or the previous block is already
+    // terminated, don't touch it.
+  } else {
+    // Otherwise, create a fall-through branch.
+    Builder.CreateBr(Target);
+  }
+
+  Builder.ClearInsertionPoint();
+}
+
+void OpenMPIRBuilder::emitBlock(BasicBlock *BB, Function *CurFn,
+                                bool IsFinished) {
+  BasicBlock *CurBB = Builder.GetInsertBlock();
+
+  // Fall out of the current block (if necessary).
+  emitBranch(BB);
+
+  if (IsFinished && BB->use_empty()) {
+    BB->eraseFromParent();
+    return;
+  }
+
+  // Place the block after the current block, if possible, or else at
+  // the end of the function.
+  if (CurBB && CurBB->getParent())
+    CurFn->insert(std::next(CurBB->getIterator()), BB);
+  else
+    CurFn->insert(CurFn->end(), BB);
+  Builder.SetInsertPoint(BB);
+}
+
+void OpenMPIRBuilder::emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen,
+                                   BodyGenCallbackTy ElseGen,
+                                   InsertPointTy AllocaIP) {
+  // If the condition constant folds and can be elided, try to avoid emitting
+  // the condition and the dead arm of the if/else.
+  if (auto *CI = dyn_cast<ConstantInt>(Cond)) {
+    auto CondConstant = CI->getSExtValue();
+    if (CondConstant)
+      ThenGen(AllocaIP, Builder.saveIP());
+    else
+      ElseGen(AllocaIP, Builder.saveIP());
+    return;
+  }
+
+  Function *CurFn = Builder.GetInsertBlock()->getParent();
+
+  // Otherwise, the condition did not fold, or we couldn't elide it.  Just
+  // emit the conditional branch.
+  BasicBlock *ThenBlock = BasicBlock::Create(M.getContext(), "omp_if.then");
+  BasicBlock *ElseBlock = BasicBlock::Create(M.getContext(), "omp_if.else");
+  BasicBlock *ContBlock = BasicBlock::Create(M.getContext(), "omp_if.end");
+  Builder.CreateCondBr(Cond, ThenBlock, ElseBlock);
+  // Emit the 'then' code.
+  emitBlock(ThenBlock, CurFn);
+  ThenGen(AllocaIP, Builder.saveIP());
+  emitBranch(ContBlock);
+  // Emit the 'else' code if present.
+  // There is no need to emit line number for unconditional branch.
+  emitBlock(ElseBlock, CurFn);
+  ElseGen(AllocaIP, Builder.saveIP());
+  // There is no need to emit line number for unconditional branch.
+  emitBranch(ContBlock);
+  // Emit the continuation block for code after the if.
+  emitBlock(ContBlock, CurFn, /*IsFinished=*/true);
+}
+
 bool OpenMPIRBuilder::checkAndEmitFlushAfterAtomic(
     const LocationDescription &Loc, llvm::AtomicOrdering AO, AtomicKind AK) {
   assert(!(AO == AtomicOrdering::NotAtomic ||
diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
index 3897329e492d6..776b04d508dac 100644
--- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
@@ -4886,18 +4886,9 @@ TEST_F(OpenMPIRBuilderTest, TargetEnterData) {
   OMPBuilder.initialize();
   F->setName("func");
   IRBuilder<> Builder(BB);
-
   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
 
-  unsigned NumDataOperands = 1;
   int64_t DeviceID = 2;
-  struct OpenMPIRBuilder::MapperAllocas MapperAllocas;
-  SmallVector<uint64_t> MapTypeFlagsTo = {1};
-  SmallVector<Constant *> MapNames;
-  auto *I8PtrTy = Builder.getInt8PtrTy();
-  auto *ArrI8PtrTy = ArrayType::get(I8PtrTy, NumDataOperands);
-  auto *I64Ty = Builder.getInt64Ty();
-  auto *ArrI64Ty = ArrayType::get(I64Ty, NumDataOperands);
 
   AllocaInst *Val1 =
       Builder.CreateAlloca(Builder.getInt32Ty(), Builder.getInt64(1));
@@ -4905,44 +4896,34 @@ TEST_F(OpenMPIRBuilderTest, TargetEnterData) {
 
   IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
                                     F->getEntryBlock().getFirstInsertionPt());
-  OMPBuilder.createMapperAllocas(Builder.saveIP(), AllocaIP, NumDataOperands,
-                                 MapperAllocas);
 
+  llvm::OpenMPIRBuilder::MapInfosTy CombinedInfo;
   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
-  auto ProcessMapOpCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
-    Value *DataValue = Val1;
-    Value *DataPtrBase;
-    Value *DataPtr;
-    DataPtrBase = DataValue;
-    DataPtr = DataValue;
-    Builder.restoreIP(CodeGenIP);
+  auto GenMapInfoCB =
+      [&](InsertPointTy codeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {
+    // Get map clause information.
+    Builder.restoreIP(codeGenIP);
 
-    Value *Null = Constant::getNullValue(DataValue->getType()->getPointerTo());
-    Value *SizeGep =
-        Builder.CreateGEP(DataValue->getType(), Null, Builder.getInt32(1));
-    Value *SizePtrToInt = Builder.CreatePtrToInt(SizeGep, I64Ty);
-
-    Value *PtrBaseGEP =
-        Builder.CreateInBoundsGEP(ArrI8PtrTy, MapperAllocas.ArgsBase,
-                                  {Builder.getInt32(0), Builder.getInt32(0)});
-    Value *PtrBaseCast = Builder.CreateBitCast(
-        PtrBaseGEP, DataPtrBase->getType()->getPointerTo());
-    Builder.CreateStore(DataPtrBase, PtrBaseCast);
-    Value *PtrGEP =
-        Builder.CreateInBoundsGEP(ArrI8PtrTy, MapperAllocas.Args,
-                                  {Builder.getInt32(0), Builder.getInt32(0)});
-    Value *PtrCast =
-        Builder.CreateBitCast(PtrGEP, DataPtr->getType()->getPointerTo());
-    Builder.CreateStore(DataPtr, PtrCast);
-    Value *SizeGEP =
-        Builder.CreateInBoundsGEP(ArrI64Ty, MapperAllocas.ArgSizes,
-                                  {Builder.getInt32(0), Builder.getInt32(0)});
-    Builder.CreateStore(SizePtrToInt, SizeGEP);
+    CombinedInfo.BasePointers.emplace_back(Val1);
+    CombinedInfo.Pointers.emplace_back(Val1);
+    CombinedInfo.Sizes.emplace_back(Builder.getInt64(4));
+    CombinedInfo.Types.emplace_back(llvm::omp::OpenMPOffloadMappingFlags(1));
+    uint32_t temp;
+    CombinedInfo.Names.emplace_back(
+        OMPBuilder.getOrCreateSrcLocStr("unknown", temp));
+    return CombinedInfo;
   };
 
+  llvm::OpenMPIRBuilder::TargetDataInfo Info(
+      /*RequiresDevicePointerInfo=*/false,
+      /*SeparateBeginEndCalls=*/true);
+
+  OMPBuilder.Config.setIsTargetCodegen(true);
+
+  llvm::omp::RuntimeFunction RTLFunc = OMPRTL___tgt_target_data_begin_mapper;
   Builder.restoreIP(OMPBuilder.createTargetData(
-      Loc, Builder.saveIP(), MapTypeFlagsTo, MapNames, MapperAllocas,
-      /* IsBegin= */ true, DeviceID, /* IfCond= */ nullptr, ProcessMapOpCB));
+      Loc, AllocaIP, Builder.saveIP(), Builder.getInt64(DeviceID),
+      /* IfCond= */ nullptr, Info, GenMapInfoCB, &RTLFunc));
 
   CallInst *TargetDataCall = dyn_cast<CallInst>(&BB->back());
   EXPECT_NE(TargetDataCall, nullptr);
@@ -4962,18 +4943,9 @@ TEST_F(OpenMPIRBuilderTest, TargetExitData) {
   OMPBuilder.initialize();
   F->setName("func");
   IRBuilder<> Builder(BB);
-
   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
 
-  unsigned NumDataOperands = 1;
   int64_t DeviceID = 2;
-  struct OpenMPIRBuilder::MapperAllocas MapperAllocas;
-  SmallVector<uint64_t> MapTypeFlagsFrom = {2};
-  SmallVector<Constant *> MapNames;
-  auto *I8PtrTy = Builder.getInt8PtrTy();
-  auto *ArrI8PtrTy = ArrayType::get(I8PtrTy, NumDataOperands);
-  auto *I64Ty = Builder.getInt64Ty();
-  auto *ArrI64Ty = ArrayType::get(I64Ty, NumDataOperands);
 
   AllocaInst *Val1 =
       Builder.CreateAlloca(Builder.getInt32Ty(), Builder.getInt64(1));
@@ -4981,44 +4953,34 @@ TEST_F(OpenMPIRBuilderTest, TargetExitData) {
 
   IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
                                     F->getEntryBlock().getFirstInsertionPt());
-  OMPBuilder.createMapperAllocas(Builder.saveIP(), AllocaIP, NumDataOperands,
-                                 MapperAllocas);
 
+  llvm::OpenMPIRBuilder::MapInfosTy CombinedInfo;
   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
-  auto ProcessMapOpCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
-    Value *DataValue = Val1;
-    Value *DataPtrBase;
-    Value *DataPtr;
-    DataPtrBase = DataValue;
-    DataPtr = DataValue;
-    Builder.restoreIP(CodeGenIP);
+  auto GenMapInfoCB =
+      [&](InsertPointTy codeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {
+    // Get map clause information.
+    Builder.restoreIP(codeGenIP);
 
-    Value *Null = Constant::getNullValue(DataValue->getType()->getPointerTo());
-    Value *SizeGep =
-        Builder.CreateGEP(DataValue->getType(), Null, Builder.getInt32(1));
-    Value *SizePtrToInt = Builder.CreatePtrToInt(SizeGep, I64Ty);
-
-    Value *PtrBaseGEP =
-        Builder.CreateInBoundsGEP(ArrI8PtrTy, MapperAllocas.ArgsBase,
-                                  {Builder.getInt32(0), Builder.getInt32(0)});
-    Value *PtrBaseCast = Builder.CreateBitCast(
-        PtrBaseGEP, DataPtrBase->getType()->getPointerTo());
-    Builder.CreateStore(DataPtrBase, PtrBaseCast);
-    Value *PtrGEP =
-        Builder.CreateInBoundsGEP(ArrI8PtrTy, MapperAllocas.Args,
-                                  {Builder.getInt32(0), Builder.getInt32(0)});
-    Value *PtrCast =
-        Builder.CreateBitCast(PtrGEP, DataPtr->getType()->getPointerTo());
-    Builder.CreateStore(DataPtr, PtrCast);
-    Value *SizeGEP =
-        Builder.CreateInBoundsGEP(ArrI64Ty, MapperAllocas.ArgSizes,
-                                  {Builder.getInt32(0), Builder.getInt32(0)});
-    Builder.CreateStore(SizePtrToInt, SizeGEP);
+    CombinedInfo.BasePointers.emplace_back(Val1);
+    CombinedInfo.Pointers.emplace_back(Val1);
+    CombinedInfo.Sizes.emplace_back(Builder.getInt64(4));
+    CombinedInfo.Types.emplace_back(llvm::omp::OpenMPOffloadMappingFlags(2));
+    uint32_t temp;
+    CombinedInfo.Names.emplace_back(
+        OMPBuilder.getOrCreateSrcLocStr("unknown", temp));
+    return CombinedInfo;
   };
 
+  llvm::OpenMPIRBuilder::TargetDataInfo Info(
+      /*RequiresDevicePointerInfo=*/false,
+      /*SeparateBeginEndCalls=*/true);
+
+  OMPBuilder.Config.setIsTargetCodegen(true);
+
+  llvm::omp::RuntimeFunction RTLFunc = OMPRTL___tgt_target_data_end_mapper;
   Builder.restoreIP(OMPBuilder.createTargetData(
-      Loc, Builder.saveIP(), MapTypeFlagsFrom, MapNames, MapperAllocas,
-      /* IsBegin= */ false, DeviceID, /* IfCond= */ nullptr, ProcessMapOpCB));
+      Loc, AllocaIP, Builder.saveIP(), Builder.getInt64(DeviceID),
+      /* IfCond= */ nullptr, Info, GenMapInfoCB, &RTLFunc));
 
   CallInst *TargetDataCall = dyn_cast<CallInst>(&BB->back());
   EXPECT_NE(TargetDataCall, nullptr);
@@ -5038,18 +5000,9 @@ TEST_F(OpenMPIRBuilderTest, TargetDataRegion) {
   OMPBuilder.initialize();
   F->setName("func");
   IRBuilder<> Builder(BB);
-
   OpenMPIRBuilder::LocationDescription Loc({Builder.saveIP(), DL});
 
-  unsigned NumDataOperands = 1;
   int64_t DeviceID = 2;
-  struct OpenMPIRBuilder::MapperAllocas MapperAllocas;
-  SmallVector<uint64_t> MapTypeFlagsToFrom = {3};
-  SmallVector<Constant *> MapNames;
-  auto *I8PtrTy = Builder.getInt8PtrTy();
-  auto *ArrI8PtrTy = ArrayType::get(I8PtrTy, NumDataOperands);
-  auto *I64Ty = Builder.getInt64Ty();
-  auto *ArrI64Ty = ArrayType::get(I64Ty, NumDataOperands);
 
   AllocaInst *Val1 =
       Builder.CreateAlloca(Builder.getInt32Ty(), Builder.getInt64(1));
@@ -5057,57 +5010,52 @@ TEST_F(OpenMPIRBuilderTest, TargetDataRegion) {
 
   IRBuilder<>::InsertPoint AllocaIP(&F->getEntryBlock(),
                                     F->getEntryBlock().getFirstInsertionPt());
-  OMPBuilder.createMapperAllocas(Builder.saveIP(), AllocaIP, NumDataOperands,
-                                 MapperAllocas);
 
+  llvm::OpenMPIRBuilder::MapInfosTy CombinedInfo;
   using InsertPointTy = OpenMPIRBuilder::InsertPointTy;
-  auto ProcessMapOpCB = [&](InsertPointTy AllocaIP, InsertPointTy CodeGenIP) {
-    Value *DataValue = Val1;
-    Value *DataPtrBase;
-    Value *DataPtr;
-    DataPtrBase = DataValue;
-    DataPtr = DataValue;
-    Builder.restoreIP(CodeGenIP);
+  auto GenMapInfoCB =
+      [&](InsertPointTy codeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {
+    // Get map clause information.
+    Builder.restoreIP(codeGenIP);
 
-    Value *Null = Constant::getNullValue(DataValue->getType()->getPointerTo());
-    Value *SizeGep =
-        Builder.CreateGEP(DataValue->getType(), Null, Builder.getInt32(1));
-    Value *SizePtrToInt = Builder.CreatePtrToInt(SizeGep, I64Ty);
-
-    Value *PtrBaseGEP =
-        Builder.CreateInBoundsGEP(ArrI8PtrTy, MapperAllocas.ArgsBase,
-                                  {Builder.getInt32(0), Builder.getInt32(0)});
-    Value *PtrBaseCast = Builder.CreateBitCast(
-        PtrBaseGEP, DataPtrBase->getType()->getPointerTo());
-    Builder.CreateStore(DataPtrBase, PtrBaseCast);
-    Value *PtrGEP =
-        Builder.CreateInBoundsGEP(ArrI8PtrTy, MapperAllocas.Args,
-                                  {Builder.getInt32(0), Builder.getInt32(0)});
-    Value *PtrCast =
-        Builder.CreateBitCast(PtrGEP, DataPtr->getType()->getPointerTo());
-    Builder.CreateStore(DataPtr, PtrCast);
-    Value *SizeGEP =
-        Builder.CreateInBoundsGEP(ArrI64Ty, MapperAllocas.ArgSizes,
-                                  {Builder.getInt32(0), Builder.getInt32(0)});
-    Builder.CreateStore(SizePtrToInt, SizeGEP);
+    CombinedInfo.BasePointers.emplace_back(Val1);
+    CombinedInfo.Pointers.emplace_back(Val1);
+    CombinedInfo.Sizes.emplace_back(Builder.getInt64(4));
+    CombinedInfo.Types.emplace_back(llvm::omp::OpenMPOffloadMappingFlags(3));
+    uint32_t temp;
+    CombinedInfo.Names.emplace_back(
+        OMPBuilder.getOrCreateSrcLocStr("unknown", temp));
+    return CombinedInfo;
   };
 
-  auto BodyCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
-    Builder.restoreIP(codeGenIP);
-    auto *SI = Builder.CreateStore(Builder.getInt32(99), Val1);
-    auto *newBB = SplitBlock(Builder.GetInsertBlock(), SI);
-    Builder.SetInsertPoint(newBB);
-    auto *UI = &Builder.GetInsertBlock()->back();
-    SplitBlock(Builder.GetInsertBlock(), UI);
+  llvm::OpenMPIRBuilder::TargetDataInfo Info(
+      /*RequiresDevicePointerInfo=*/false,
+      /*SeparateBeginEndCalls=*/true);
+
+  OMPBuilder.Config.setIsTargetCodegen(true);
+
+  auto BodyCB = [&](InsertPointTy CodeGenIP, int BodyGenType) {
+    if (BodyGenType == 3) {
+      Builder.restoreIP(CodeGenIP);
+      CallInst *TargetDataCall = dyn_cast<CallInst>(&BB->back());
+      EXPECT_NE(TargetDataCall, nullptr);
+      EXPECT_EQ(TargetDataCall->arg_size(), 9U);
+      EXPECT_EQ(TargetDataCall->getCalledFunction()->getName(),
+                "__tgt_target_data_begin_mapper");
+      EXPECT_TRUE(TargetDataCall->getOperand(1)->getType()->isIntegerTy(64));
+      EXPECT_TRUE(TargetDataCall->getOperand(2)->getType()->isIntegerTy(32));
+      EXPECT_TRUE(TargetDataCall->getOperand(8)->getType()->isPointerTy());
+      Builder.restoreIP(CodeGenIP);
+      Builder.CreateStore(Builder.getInt32(99), Val1);
+    }
+    return Builder.saveIP();
   };
 
   Builder.restoreIP(OMPBuilder.createTargetData(
-      Loc, Builder.saveIP(), MapTypeFlagsToFrom, MapNames, MapperAllocas,
-      /* IsBegin= */ false, DeviceID, /* IfCond= */ nullptr, ProcessMapOpCB,
-      BodyCB));
+      Loc, AllocaIP, Builder.saveIP(), Builder.getInt64(DeviceID),
+      /* IfCond= */ nullptr, Info, GenMapInfoCB, nullptr, BodyCB));
 
-  CallInst *TargetDataCall =
-      dyn_cast<CallInst>(&Builder.GetInsertBlock()->back());
+  CallInst *TargetDataCall = dyn_cast<CallInst>(&BB->back());
   EXPECT_NE(TargetDataCall, nullptr);
   EXPECT_EQ(TargetDataCall->arg_size(), 9U);
   EXPECT_EQ(TargetDataCall->getCalledFunction()->getName(),
diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
index c3ffd9903d282..a0050397a079a 100644
--- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp
@@ -1361,78 +1361,55 @@ convertOmpThreadprivate(Operation &opInst, llvm::IRBuilderBase &builder,
   return success();
 }
 
-/// Process MapOperands for Target Data directives.
-static LogicalResult processMapOperand(
-    llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation,
-    const SmallVector<Value> &mapOperands, const ArrayAttr &mapTypes,
-    SmallVector<uint64_t> &mapTypeFlags,
-    SmallVectorImpl<llvm::Constant *> &mapNames,
-    struct llvm::OpenMPIRBuilder::MapperAllocas &mapperAllocas) {
-  auto numMapOperands = mapOperands.size();
-  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
-  llvm::PointerType *i8PtrTy = builder.getInt8PtrTy();
-  llvm::ArrayType *arrI8PtrTy = llvm::ArrayType::get(i8PtrTy, numMapOperands);
-  llvm::IntegerType *i64Ty = builder.getInt64Ty();
-  llvm::ArrayType *arrI64Ty = llvm::ArrayType::get(i64Ty, numMapOperands);
+int64_t getSizeInBytes(DataLayout &DL, const mlir::Type &type) {
+  if (isa<LLVM::LLVMPointerType>(type))
+    return DL.getTypeSize(cast<LLVM::LLVMPointerType>(type).getElementType());
+
+  return 0;
+}
 
+static void genMapInfos(llvm::IRBuilderBase &builder,
+                        LLVM::ModuleTranslation &moduleTranslation,
+                        DataLayout &DL,
+                        llvm::OpenMPIRBuilder::MapInfosTy &combinedInfo,
+                        const SmallVector<Value> &mapOperands,
+                        const ArrayAttr &mapTypes) {
+  // Get map clause information.
+  llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
   unsigned index = 0;
   for (const auto &mapOp : mapOperands) {
-    const auto &mapTypeOp = mapTypes[index];
-
-    llvm::Value *mapOpValue = moduleTranslation.lookupValue(mapOp);
-    llvm::Value *mapOpPtrBase;
-    llvm::Value *mapOpPtr;
-    llvm::Value *mapOpSize;
-
-    if (isa<LLVM::LLVMPointerType>(mapOp.getType())) {
-      mapOpPtrBase = mapOpValue;
-      mapOpPtr = mapOpValue;
-      mapOpSize = ompBuilder->getSizeInBytes(mapOpValue);
-    } else {
-      return failure();
+    if (!mapOp.getType().isa<LLVM::LLVMPointerType>()) {
+      // TODO: Only LLVMPointerTypes are handled.
+      combinedInfo.BasePointers.clear();
+      combinedInfo.Pointers.clear();
+      combinedInfo.Sizes.clear();
+      combinedInfo.Types.clear();
+      combinedInfo.Names.clear();
+      return;
     }
 
-    // Store base pointer extracted from operand into the i-th position of
-    // argBase.
-    llvm::Value *ptrBaseGEP = builder.CreateInBoundsGEP(
-        arrI8PtrTy, mapperAllocas.ArgsBase,
-        {builder.getInt32(0), builder.getInt32(index)});
-    llvm::Value *ptrBaseCast = builder.CreateBitCast(
-        ptrBaseGEP, mapOpPtrBase->getType()->getPointerTo());
-    builder.CreateStore(mapOpPtrBase, ptrBaseCast);
-
-    // Store pointer extracted from operand into the i-th position of args.
-    llvm::Value *ptrGEP = builder.CreateInBoundsGEP(
-        arrI8PtrTy, mapperAllocas.Args,
-        {builder.getInt32(0), builder.getInt32(index)});
-    llvm::Value *ptrCast =
-        builder.CreateBitCast(ptrGEP, mapOpPtr->getType()->getPointerTo());
-    builder.CreateStore(mapOpPtr, ptrCast);
-
-    // Store size extracted from operand into the i-th position of argSizes.
-    llvm::Value *sizeGEP = builder.CreateInBoundsGEP(
-        arrI64Ty, mapperAllocas.ArgSizes,
-        {builder.getInt32(0), builder.getInt32(index)});
-    builder.CreateStore(mapOpSize, sizeGEP);
-
-    mapTypeFlags.push_back(dyn_cast<mlir::IntegerAttr>(mapTypeOp).getInt());
-    llvm::Constant *mapName =
-        mlir::LLVM::createMappingInformation(mapOp.getLoc(), *ompBuilder);
-    mapNames.push_back(mapName);
-    ++index;
+    llvm::Value *mapOpValue = moduleTranslation.lookupValue(mapOp);
+    combinedInfo.BasePointers.emplace_back(mapOpValue);
+    combinedInfo.Pointers.emplace_back(mapOpValue);
+    combinedInfo.Names.emplace_back(
+        mlir::LLVM::createMappingInformation(mapOp.getLoc(), *ompBuilder));
+    combinedInfo.Types.emplace_back(llvm::omp::OpenMPOffloadMappingFlags(
+        mapTypes[index].dyn_cast<mlir::IntegerAttr>().getInt()));
+    combinedInfo.Sizes.emplace_back(
+        builder.getInt64(getSizeInBytes(DL, mapOp.getType())));
+    index++;
   }
-
-  return success();
 }
 
 static LogicalResult
 convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder,
                      LLVM::ModuleTranslation &moduleTranslation) {
-  unsigned numMapOperands;
   llvm::Value *ifCond = nullptr;
   int64_t deviceID = llvm::omp::OMP_DEVICEID_UNDEF;
   SmallVector<Value> mapOperands;
   ArrayAttr mapTypes;
+  llvm::omp::RuntimeFunction RTLFn;
+  DataLayout DL = DataLayout(op->getParentOfType<ModuleOp>());
 
   llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
 
@@ -1453,7 +1430,6 @@ convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder,
                         dyn_cast<mlir::IntegerAttr>(constOp.getValue()))
                   deviceID = intAttr.getInt();
 
-            numMapOperands = dataOp.getMapOperands().size();
             mapOperands = dataOp.getMapOperands();
             mapTypes = dataOp.getMapTypes();
             return success();
@@ -1471,8 +1447,7 @@ convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder,
                 if (auto intAttr =
                         dyn_cast<mlir::IntegerAttr>(constOp.getValue()))
                   deviceID = intAttr.getInt();
-
-            numMapOperands = enterDataOp.getMapOperands().size();
+            RTLFn = llvm::omp::OMPRTL___tgt_target_data_begin_mapper;
             mapOperands = enterDataOp.getMapOperands();
             mapTypes = enterDataOp.getMapTypes();
             return success();
@@ -1491,7 +1466,7 @@ convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder,
                         dyn_cast<mlir::IntegerAttr>(constOp.getValue()))
                   deviceID = intAttr.getInt();
 
-            numMapOperands = exitDataOp.getMapOperands().size();
+            RTLFn = llvm::omp::OMPRTL___tgt_target_data_end_mapper;
             mapOperands = exitDataOp.getMapOperands();
             mapTypes = exitDataOp.getMapTypes();
             return success();
@@ -1504,46 +1479,55 @@ convertOmpTargetData(Operation *op, llvm::IRBuilderBase &builder,
   if (failed(result))
     return failure();
 
-  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
-  llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
-      findAllocaInsertPoint(builder, moduleTranslation);
-
-  struct llvm::OpenMPIRBuilder::MapperAllocas mapperAllocas;
-  SmallVector<uint64_t> mapTypeFlags;
-  SmallVector<llvm::Constant *> mapNames;
-  ompBuilder->createMapperAllocas(builder.saveIP(), allocaIP, numMapOperands,
-                                  mapperAllocas);
-
   using InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;
-  LogicalResult processMapOpStatus = success();
-  auto processMapOpCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
+
+  // Fill up the arrays with all the mapped variables.
+  llvm::OpenMPIRBuilder::MapInfosTy combinedInfo;
+  auto genMapInfoCB =
+      [&](InsertPointTy codeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {
     builder.restoreIP(codeGenIP);
-    processMapOpStatus =
-        processMapOperand(builder, moduleTranslation, mapOperands, mapTypes,
-                          mapTypeFlags, mapNames, mapperAllocas);
+    genMapInfos(builder, moduleTranslation, DL, combinedInfo, mapOperands,
+                mapTypes);
+    return combinedInfo;
   };
 
   LogicalResult bodyGenStatus = success();
-  auto bodyCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) {
-    // DataOp has only one region associated with it.
-    auto &region = cast<omp::DataOp>(op).getRegion();
-    builder.restoreIP(codeGenIP);
-    bodyGenStatus = inlineConvertOmpRegions(region, "omp.data.region", builder,
-                                            moduleTranslation);
+  using BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy;
+  auto bodyGenCB = [&](InsertPointTy codeGenIP, BodyGenTy bodyGenType) {
+    switch (bodyGenType) {
+    case BodyGenTy::Priv:
+      break;
+    case BodyGenTy::DupNoPriv:
+      break;
+    case BodyGenTy::NoPriv: {
+      // DataOp has only one region associated with it.
+      auto &region = cast<omp::DataOp>(op).getRegion();
+      builder.restoreIP(codeGenIP);
+      bodyGenStatus = inlineConvertOmpRegions(region, "omp.data.region",
+                                              builder, moduleTranslation);
+    }
+    }
+    return builder.saveIP();
   };
 
+  llvm::OpenMPIRBuilder::LocationDescription ompLoc(builder);
+  llvm::OpenMPIRBuilder::InsertPointTy allocaIP =
+      findAllocaInsertPoint(builder, moduleTranslation);
+
+  // TODO: Add support for DevicePointerInfo
+  llvm::OpenMPIRBuilder::TargetDataInfo info(
+      /*RequiresDevicePointerInfo=*/false,
+      /*SeparateBeginEndCalls=*/true);
   if (isa<omp::DataOp>(op)) {
     builder.restoreIP(ompBuilder->createTargetData(
-        ompLoc, builder.saveIP(), mapTypeFlags, mapNames, mapperAllocas,
-        /*IsBegin=*/false, deviceID, ifCond, processMapOpCB, bodyCB));
+        ompLoc, allocaIP, builder.saveIP(), builder.getInt64(deviceID), ifCond,
+        info, genMapInfoCB, nullptr, bodyGenCB));
   } else {
     builder.restoreIP(ompBuilder->createTargetData(
-        ompLoc, builder.saveIP(), mapTypeFlags, mapNames, mapperAllocas,
-        isa<omp::EnterDataOp>(op), deviceID, ifCond, processMapOpCB));
+        ompLoc, allocaIP, builder.saveIP(), builder.getInt64(deviceID), ifCond,
+        info, genMapInfoCB, &RTLFn));
   }
 
-  if (failed(processMapOpStatus))
-    return processMapOpStatus;
   return bodyGenStatus;
 }
 
@@ -1554,7 +1538,7 @@ LogicalResult convertFlagsAttr(Operation *op, mlir::omp::FlagsAttr attribute,
                                LLVM::ModuleTranslation &moduleTranslation) {
   if (!cast<mlir::ModuleOp>(op))
     return failure();
-  
+
   llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder();
 
   ompBuilder->createGlobalFlag(
diff --git a/mlir/test/Target/LLVMIR/omptarget-llvm.mlir b/mlir/test/Target/LLVMIR/omptarget-llvm.mlir
index 6d6aa11875fb5..d39741fd1160a 100644
--- a/mlir/test/Target/LLVMIR/omptarget-llvm.mlir
+++ b/mlir/test/Target/LLVMIR/omptarget-llvm.mlir
@@ -11,29 +11,28 @@ llvm.func @_QPopenmp_target_data() {
   llvm.return
 }
 
+// CHECK:         @.offload_sizes = private unnamed_addr constant [1 x i64] [i64 4]
 // CHECK:         @.offload_maptypes = private unnamed_addr constant [1 x i64] [i64 3]
 // CHECK-LABEL: define void @_QPopenmp_target_data() {
 // CHECK:         %[[VAL_0:.*]] = alloca [1 x ptr], align 8
 // CHECK:         %[[VAL_1:.*]] = alloca [1 x ptr], align 8
-// CHECK:         %[[VAL_2:.*]] = alloca [1 x i64], align 8
+// CHECK:         %[[VAL_2:.*]] = alloca [1 x ptr], align 8
 // CHECK:         %[[VAL_3:.*]] = alloca i32, i64 1, align 4
 // CHECK:         br label %[[VAL_4:.*]]
-// CHECK:       [[VAL_4]]:
-// CHECK:         %[[VAL_5:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
-// CHECK:         store ptr %[[VAL_3]], ptr %[[VAL_5]], align 8
-// CHECK:         %[[VAL_6:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
+// CHECK:       entry:                                            ; preds = %[[VAL_5:.*]]
+// CHECK:         %[[VAL_6:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
 // CHECK:         store ptr %[[VAL_3]], ptr %[[VAL_6]], align 8
-// CHECK:         %[[VAL_7:.*]] = getelementptr inbounds [1 x i64], ptr %[[VAL_2]], i32 0, i32 0
-// CHECK:         store i64 ptrtoint (ptr getelementptr (ptr, ptr null, i32 1) to i64), ptr %[[VAL_7]], align 4
-// CHECK:         %[[VAL_8:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
-// CHECK:         %[[VAL_9:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
-// CHECK:         %[[VAL_10:.*]] = getelementptr inbounds [1 x i64], ptr %[[VAL_2]], i32 0, i32 0
-// CHECK:         call void @__tgt_target_data_begin_mapper(ptr @2, i64 -1, i32 1, ptr %[[VAL_8]], ptr %[[VAL_9]], ptr %[[VAL_10]], ptr @.offload_maptypes, ptr @.offload_mapnames, ptr null)
+// CHECK:         %[[VAL_7:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
+// CHECK:         store ptr %[[VAL_3]], ptr %[[VAL_7]], align 8
+// CHECK:         %[[VAL_8:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_2]], i64 0, i64 0
+// CHECK:         store ptr null, ptr %[[VAL_8]], align 8
+// CHECK:         %[[VAL_9:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
+// CHECK:         %[[VAL_10:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
+// CHECK:         call void @__tgt_target_data_begin_mapper(ptr @2, i64 -1, i32 1, ptr %[[VAL_9]], ptr %[[VAL_10]], ptr @.offload_sizes, ptr @.offload_maptypes, ptr null, ptr null)
 // CHECK:         store i32 99, ptr %[[VAL_3]], align 4
 // CHECK:         %[[VAL_11:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
 // CHECK:         %[[VAL_12:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
-// CHECK:         %[[VAL_13:.*]] = getelementptr inbounds [1 x i64], ptr %[[VAL_2]], i32 0, i32 0
-// CHECK:         call void @__tgt_target_data_end_mapper(ptr @2, i64 -1, i32 1, ptr %[[VAL_11]], ptr %[[VAL_12]], ptr %[[VAL_13]], ptr @.offload_maptypes, ptr @.offload_mapnames, ptr null)
+// CHECK:         call void @__tgt_target_data_end_mapper(ptr @2, i64 -1, i32 1, ptr %[[VAL_11]], ptr %[[VAL_12]], ptr @.offload_sizes, ptr @.offload_maptypes, ptr null, ptr null)
 // CHECK:         ret void
 
 // -----
@@ -51,35 +50,34 @@ llvm.func @_QPopenmp_target_data_region(%1 : !llvm.ptr<array<1024 x i32>>) {
   llvm.return
 }
 
+// CHECK:         @.offload_sizes = private unnamed_addr constant [1 x i64] [i64 4096]
 // CHECK:         @.offload_maptypes = private unnamed_addr constant [1 x i64] [i64 2]
 // CHECK-LABEL: define void @_QPopenmp_target_data_region
 // CHECK:         (ptr %[[ARG_0:.*]]) {
 // CHECK:         %[[VAL_0:.*]] = alloca [1 x ptr], align 8
 // CHECK:         %[[VAL_1:.*]] = alloca [1 x ptr], align 8
-// CHECK:         %[[VAL_2:.*]] = alloca [1 x i64], align 8
+// CHECK:         %[[VAL_2:.*]] = alloca [1 x ptr], align 8
 // CHECK:         br label %[[VAL_3:.*]]
-// CHECK:       [[VAL_3]]:
-// CHECK:         %[[VAL_4:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
-// CHECK:         store ptr %[[VAL_5:.*]], ptr %[[VAL_4]], align 8
-// CHECK:         %[[VAL_6:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
-// CHECK:         store ptr %[[VAL_5]], ptr %[[VAL_6]], align 8
-// CHECK:         %[[VAL_7:.*]] = getelementptr inbounds [1 x i64], ptr %[[VAL_2]], i32 0, i32 0
-// CHECK:         store i64 ptrtoint (ptr getelementptr (ptr, ptr null, i32 1) to i64), ptr %[[VAL_7]], align 4
-// CHECK:         %[[VAL_8:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
-// CHECK:         %[[VAL_9:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
-// CHECK:         %[[VAL_10:.*]] = getelementptr inbounds [1 x i64], ptr %[[VAL_2]], i32 0, i32 0
-// CHECK:         call void @__tgt_target_data_begin_mapper(ptr @2, i64 -1, i32 1, ptr %[[VAL_8]], ptr %[[VAL_9]], ptr %[[VAL_10]], ptr @.offload_maptypes, ptr @.offload_mapnames, ptr null)
-// CHECK:         %[[VAL_11:.*]] = getelementptr [1024 x i32], ptr %[[VAL_5]], i32 0, i64 0
+// CHECK:       entry:                                            ; preds = %[[VAL_4:.*]]
+// CHECK:         %[[VAL_5:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
+// CHECK:         store ptr %[[VAL_6:.*]], ptr %[[VAL_5]], align 8
+// CHECK:         %[[VAL_7:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
+// CHECK:         store ptr %[[VAL_6]], ptr %[[VAL_7]], align 8
+// CHECK:         %[[VAL_8:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_2]], i64 0, i64 0
+// CHECK:         store ptr null, ptr %[[VAL_8]], align 8
+// CHECK:         %[[VAL_9:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
+// CHECK:         %[[VAL_10:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
+// CHECK:         call void @__tgt_target_data_begin_mapper(ptr @2, i64 -1, i32 1, ptr %[[VAL_9]], ptr %[[VAL_10]], ptr @.offload_sizes, ptr @.offload_maptypes, ptr null, ptr null)
+// CHECK:         %[[VAL_11:.*]] = getelementptr [1024 x i32], ptr %[[VAL_6]], i32 0, i64 0
 // CHECK:         store i32 99, ptr %[[VAL_11]], align 4
 // CHECK:         %[[VAL_12:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_0]], i32 0, i32 0
 // CHECK:         %[[VAL_13:.*]] = getelementptr inbounds [1 x ptr], ptr %[[VAL_1]], i32 0, i32 0
-// CHECK:         %[[VAL_14:.*]] = getelementptr inbounds [1 x i64], ptr %[[VAL_2]], i32 0, i32 0
-// CHECK:         call void @__tgt_target_data_end_mapper(ptr @2, i64 -1, i32 1, ptr %[[VAL_12]], ptr %[[VAL_13]], ptr %[[VAL_14]], ptr @.offload_maptypes, ptr @.offload_mapnames, ptr null)
+// CHECK:         call void @__tgt_target_data_end_mapper(ptr @2, i64 -1, i32 1, ptr %[[VAL_12]], ptr %[[VAL_13]], ptr @.offload_sizes, ptr @.offload_maptypes, ptr null, ptr null)
 // CHECK:         ret void
 
 // -----
 
-llvm.func @_QPomp_target_enter_exit(%1 : !llvm.ptr<array<1024 x i32>>, %3 : !llvm.ptr<array<1024 x i32>>) {
+llvm.func @_QPomp_target_enter_exit(%1 : !llvm.ptr<array<1024 x i32>>, %3 : !llvm.ptr<array<512 x i32>>) {
   %4 = llvm.mlir.constant(1 : i64) : i64
   %5 = llvm.alloca %4 x i32 {bindc_name = "dvc", in_type = i32, operand_segment_sizes = array<i32: 0, 0>, uniq_name = "_QFomp_target_enter_exitEdvc"} : (i64) -> !llvm.ptr<i32>
   %6 = llvm.mlir.constant(1 : i64) : i64
@@ -92,25 +90,27 @@ llvm.func @_QPomp_target_enter_exit(%1 : !llvm.ptr<array<1024 x i32>>, %3 : !llv
   %11 = llvm.mlir.constant(10 : i32) : i32
   %12 = llvm.icmp "slt" %10, %11 : i32
   %13 = llvm.load %5 : !llvm.ptr<i32>
-  omp.target_enter_data   if(%12 : i1) device(%13 : i32) map((to -> %1 : !llvm.ptr<array<1024 x i32>>), (alloc -> %3 : !llvm.ptr<array<1024 x i32>>))
+  omp.target_enter_data   if(%12 : i1) device(%13 : i32) map((to -> %1 : !llvm.ptr<array<1024 x i32>>), (alloc -> %3 : !llvm.ptr<array<512 x i32>>))
   %14 = llvm.load %7 : !llvm.ptr<i32>
   %15 = llvm.mlir.constant(10 : i32) : i32
   %16 = llvm.icmp "sgt" %14, %15 : i32
   %17 = llvm.load %5 : !llvm.ptr<i32>
-  omp.target_exit_data   if(%16 : i1) device(%17 : i32) map((from -> %1 : !llvm.ptr<array<1024 x i32>>), (release -> %3 : !llvm.ptr<array<1024 x i32>>))
+  omp.target_exit_data   if(%16 : i1) device(%17 : i32) map((from -> %1 : !llvm.ptr<array<1024 x i32>>), (release -> %3 : !llvm.ptr<array<512 x i32>>))
   llvm.return
 }
 
+// CHECK:         @.offload_sizes = private unnamed_addr constant [2 x i64] [i64 4096, i64 2048]
 // CHECK:         @.offload_maptypes = private unnamed_addr constant [2 x i64] [i64 1, i64 0]
-// CHECK:         @.offload_maptypes.1 = private unnamed_addr constant [2 x i64] [i64 2, i64 0]
+// CHECK:         @.offload_sizes.1 = private unnamed_addr constant [2 x i64] [i64 4096, i64 2048]
+// CHECK:         @.offload_maptypes.2 = private unnamed_addr constant [2 x i64] [i64 2, i64 0]
 // CHECK-LABEL: define void @_QPomp_target_enter_exit
 // CHECK:         (ptr %[[ARG_0:.*]], ptr %[[ARG_1:.*]]) {
 // CHECK:         %[[VAL_0:.*]] = alloca [2 x ptr], align 8
 // CHECK:         %[[VAL_1:.*]] = alloca [2 x ptr], align 8
-// CHECK:         %[[VAL_2:.*]] = alloca [2 x i64], align 8
+// CHECK:         %[[VAL_2:.*]] = alloca [2 x ptr], align 8
 // CHECK:         %[[VAL_3:.*]] = alloca [2 x ptr], align 8
 // CHECK:         %[[VAL_4:.*]] = alloca [2 x ptr], align 8
-// CHECK:         %[[VAL_5:.*]] = alloca [2 x i64], align 8
+// CHECK:         %[[VAL_5:.*]] = alloca [2 x ptr], align 8
 // CHECK:         %[[VAL_6:.*]] = alloca i32, i64 1, align 4
 // CHECK:         %[[VAL_7:.*]] = alloca i32, i64 1, align 4
 // CHECK:         store i32 5, ptr %[[VAL_7]], align 4
@@ -126,44 +126,45 @@ llvm.func @_QPomp_target_enter_exit(%1 : !llvm.ptr<array<1024 x i32>>, %3 : !llv
 // CHECK:         store ptr %[[VAL_16:.*]], ptr %[[VAL_15]], align 8
 // CHECK:         %[[VAL_17:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_4]], i32 0, i32 0
 // CHECK:         store ptr %[[VAL_16]], ptr %[[VAL_17]], align 8
-// CHECK:         %[[VAL_18:.*]] = getelementptr inbounds [2 x i64], ptr %[[VAL_5]], i32 0, i32 0
-// CHECK:         store i64 ptrtoint (ptr getelementptr (ptr, ptr null, i32 1) to i64), ptr %[[VAL_18]], align 4
+// CHECK:         %[[VAL_18:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_5]], i64 0, i64 0
+// CHECK:         store ptr null, ptr %[[VAL_18]], align 8
 // CHECK:         %[[VAL_19:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_3]], i32 0, i32 1
 // CHECK:         store ptr %[[VAL_20:.*]], ptr %[[VAL_19]], align 8
 // CHECK:         %[[VAL_21:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_4]], i32 0, i32 1
 // CHECK:         store ptr %[[VAL_20]], ptr %[[VAL_21]], align 8
-// CHECK:         %[[VAL_22:.*]] = getelementptr inbounds [2 x i64], ptr %[[VAL_5]], i32 0, i32 1
-// CHECK:         store i64 ptrtoint (ptr getelementptr (ptr, ptr null, i32 1) to i64), ptr %[[VAL_22]], align 4
+// CHECK:         %[[VAL_22:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_5]], i64 0, i64 1
+// CHECK:         store ptr null, ptr %[[VAL_22]], align 8
 // CHECK:         %[[VAL_23:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_3]], i32 0, i32 0
 // CHECK:         %[[VAL_24:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_4]], i32 0, i32 0
-// CHECK:         %[[VAL_25:.*]] = getelementptr inbounds [2 x i64], ptr %[[VAL_5]], i32 0, i32 0
-// CHECK:         call void @__tgt_target_data_begin_mapper(ptr @3, i64 -1, i32 2, ptr %[[VAL_23]], ptr %[[VAL_24]], ptr %[[VAL_25]], ptr @.offload_maptypes, ptr @.offload_mapnames, ptr null)
-// CHECK:         br label %[[VAL_14]]
-// CHECK:       omp_if.end:                                       ; preds = %[[VAL_11]], %[[VAL_13]]
+// CHECK:         call void @__tgt_target_data_begin_mapper(ptr @3, i64 -1, i32 2, ptr %[[VAL_23]], ptr %[[VAL_24]], ptr @.offload_sizes, ptr @.offload_maptypes, ptr null, ptr null)
+// CHECK:         br label %[[VAL_25:.*]]
+// CHECK:       omp_if.else:                                      ; preds = %[[VAL_11]]
+// CHECK:         br label %[[VAL_25]]
+// CHECK:       omp_if.end:                                       ; preds = %[[VAL_14]], %[[VAL_13]]
 // CHECK:         %[[VAL_26:.*]] = load i32, ptr %[[VAL_7]], align 4
 // CHECK:         %[[VAL_27:.*]] = icmp sgt i32 %[[VAL_26]], 10
 // CHECK:         %[[VAL_28:.*]] = load i32, ptr %[[VAL_6]], align 4
 // CHECK:         br i1 %[[VAL_27]], label %[[VAL_29:.*]], label %[[VAL_30:.*]]
-// CHECK:       omp_if.then4:                                     ; preds = %[[VAL_14]]
+// CHECK:       omp_if.then1:                                     ; preds = %[[VAL_25]]
 // CHECK:         %[[VAL_31:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_0]], i32 0, i32 0
 // CHECK:         store ptr %[[VAL_16]], ptr %[[VAL_31]], align 8
 // CHECK:         %[[VAL_32:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_1]], i32 0, i32 0
 // CHECK:         store ptr %[[VAL_16]], ptr %[[VAL_32]], align 8
-// CHECK:         %[[VAL_33:.*]] = getelementptr inbounds [2 x i64], ptr %[[VAL_2]], i32 0, i32 0
-// CHECK:         store i64 ptrtoint (ptr getelementptr (ptr, ptr null, i32 1) to i64), ptr %[[VAL_33]], align 4
+// CHECK:         %[[VAL_33:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_2]], i64 0, i64 0
+// CHECK:         store ptr null, ptr %[[VAL_33]], align 8
 // CHECK:         %[[VAL_34:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_0]], i32 0, i32 1
 // CHECK:         store ptr %[[VAL_20]], ptr %[[VAL_34]], align 8
 // CHECK:         %[[VAL_35:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_1]], i32 0, i32 1
 // CHECK:         store ptr %[[VAL_20]], ptr %[[VAL_35]], align 8
-// CHECK:         %[[VAL_36:.*]] = getelementptr inbounds [2 x i64], ptr %[[VAL_2]], i32 0, i32 1
-// CHECK:         store i64 ptrtoint (ptr getelementptr (ptr, ptr null, i32 1) to i64), ptr %[[VAL_36]], align 4
+// CHECK:         %[[VAL_36:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_2]], i64 0, i64 1
+// CHECK:         store ptr null, ptr %[[VAL_36]], align 8
 // CHECK:         %[[VAL_37:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_0]], i32 0, i32 0
 // CHECK:         %[[VAL_38:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_1]], i32 0, i32 0
-// CHECK:         %[[VAL_39:.*]] = getelementptr inbounds [2 x i64], ptr %[[VAL_2]], i32 0, i32 0
-// CHECK:         call void @__tgt_target_data_end_mapper(ptr @3, i64 -1, i32 2, ptr %[[VAL_37]], ptr %[[VAL_38]], ptr %[[VAL_39]], ptr @.offload_maptypes.1, ptr @.offload_mapnames.2, ptr null)
-// CHECK:         br label %[[VAL_30]]
-// CHECK:       omp_if.end5:                                      ; preds = %[[VAL_14]], %[[VAL_29]]
+// CHECK:         call void @__tgt_target_data_end_mapper(ptr @3, i64 -1, i32 2, ptr %[[VAL_37]], ptr %[[VAL_38]], ptr @.offload_sizes.1, ptr @.offload_maptypes.2, ptr null, ptr null)
+// CHECK:         br label %[[VAL_39:.*]]
+// CHECK:       omp_if.else5:                                     ; preds = %[[VAL_25]]
+// CHECK:         br label %[[VAL_39]]
+// CHECK:       omp_if.end6:                                      ; preds = %[[VAL_30]], %[[VAL_29]]
 // CHECK:         ret void
 
 // -----
-

From c26fe199c13871db565e14651c060616143d7a8c Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Mon, 19 Jun 2023 14:40:33 +0200
Subject: [PATCH 014/130] [BasicAA] Add test for PR63266 (NFC)

---
 llvm/test/Analysis/BasicAA/range.ll | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/llvm/test/Analysis/BasicAA/range.ll b/llvm/test/Analysis/BasicAA/range.ll
index f153b58af3c88..06c8103994a8a 100644
--- a/llvm/test/Analysis/BasicAA/range.ll
+++ b/llvm/test/Analysis/BasicAA/range.ll
@@ -239,6 +239,26 @@ define void @benign_overflow(ptr %p, i64 %o) {
   ret void
 }
 
+; FIXME: This is a miscompile
+; CHECK-LABEL: pr63266
+; CHECK: NoAlias:	i8* %gep2, i8* %offset16
+define void @pr63266(i1 %c, ptr %base) {
+entry:
+  %offset16 = getelementptr inbounds i8, ptr %base, i64 16
+  %gep1 = getelementptr i8, ptr %base, i64 -9223372036854775792
+  br i1 %c, label %if, label %join
+
+if:
+  br label %join
+
+join:
+  %phi = phi i64 [ -9223372036854775808, %if ], [ 0, %entry ]
+  %gep2 = getelementptr i8, ptr %gep1, i64 %phi
+  store i8 0, ptr %gep2
+  load i8, ptr %offset16
+  ret void
+}
+
 declare void @llvm.assume(i1)
 
 

From 85232b0ecbf817bc1d70ae602cf44cf6ea03c0e6 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Sun, 20 Nov 2022 08:49:35 -0800
Subject: [PATCH 015/130] HIP: Directly call isfinite builtins

---
 clang/lib/Headers/__clang_hip_math.h    |  4 ++--
 clang/test/Headers/__clang_hip_math.hip | 24 +++++++++---------------
 2 files changed, 11 insertions(+), 17 deletions(-)

diff --git a/clang/lib/Headers/__clang_hip_math.h b/clang/lib/Headers/__clang_hip_math.h
index c19e32bd29364..f7949f30dfbae 100644
--- a/clang/lib/Headers/__clang_hip_math.h
+++ b/clang/lib/Headers/__clang_hip_math.h
@@ -268,7 +268,7 @@ __DEVICE__
 int ilogbf(float __x) { return __ocml_ilogb_f32(__x); }
 
 __DEVICE__
-__RETURN_TYPE __finitef(float __x) { return __ocml_isfinite_f32(__x); }
+__RETURN_TYPE __finitef(float __x) { return __builtin_isfinite(__x); }
 
 __DEVICE__
 __RETURN_TYPE __isinff(float __x) { return __builtin_isinf(__x); }
@@ -817,7 +817,7 @@ __DEVICE__
 int ilogb(double __x) { return __ocml_ilogb_f64(__x); }
 
 __DEVICE__
-__RETURN_TYPE __finite(double __x) { return __ocml_isfinite_f64(__x); }
+__RETURN_TYPE __finite(double __x) { return __builtin_isfinite(__x); }
 
 __DEVICE__
 __RETURN_TYPE __isinf(double __x) { return __builtin_isinf(__x); }
diff --git a/clang/test/Headers/__clang_hip_math.hip b/clang/test/Headers/__clang_hip_math.hip
index 7169614112d18..bb96aeaa1ab9b 100644
--- a/clang/test/Headers/__clang_hip_math.hip
+++ b/clang/test/Headers/__clang_hip_math.hip
@@ -1155,17 +1155,14 @@ extern "C" __device__ int test_ilogb(double x) {
 
 // DEFAULT-LABEL: @test___finitef(
 // DEFAULT-NEXT:  entry:
-// DEFAULT-NEXT:    [[CALL_I:%.*]] = tail call i32 @__ocml_isfinite_f32(float noundef [[X:%.*]]) #[[ATTR14]]
-// DEFAULT-NEXT:    [[TOBOOL_I:%.*]] = icmp ne i32 [[CALL_I]], 0
-// DEFAULT-NEXT:    [[CONV:%.*]] = zext i1 [[TOBOOL_I]] to i32
+// DEFAULT-NEXT:    [[TMP0:%.*]] = tail call contract float @llvm.fabs.f32(float [[X:%.*]]) #[[ATTR17:[0-9]+]]
+// DEFAULT-NEXT:    [[CMPINF_I:%.*]] = fcmp contract one float [[TMP0]], 0x7FF0000000000000
+// DEFAULT-NEXT:    [[CONV:%.*]] = zext i1 [[CMPINF_I]] to i32
 // DEFAULT-NEXT:    ret i32 [[CONV]]
 //
 // FINITEONLY-LABEL: @test___finitef(
 // FINITEONLY-NEXT:  entry:
-// FINITEONLY-NEXT:    [[CALL_I:%.*]] = tail call i32 @__ocml_isfinite_f32(float noundef nofpclass(nan inf) [[X:%.*]]) #[[ATTR14]]
-// FINITEONLY-NEXT:    [[TOBOOL_I:%.*]] = icmp ne i32 [[CALL_I]], 0
-// FINITEONLY-NEXT:    [[CONV:%.*]] = zext i1 [[TOBOOL_I]] to i32
-// FINITEONLY-NEXT:    ret i32 [[CONV]]
+// FINITEONLY-NEXT:    ret i32 1
 //
 extern "C" __device__ BOOL_TYPE test___finitef(float x) {
   return __finitef(x);
@@ -1173,17 +1170,14 @@ extern "C" __device__ BOOL_TYPE test___finitef(float x) {
 
 // DEFAULT-LABEL: @test___finite(
 // DEFAULT-NEXT:  entry:
-// DEFAULT-NEXT:    [[CALL_I:%.*]] = tail call i32 @__ocml_isfinite_f64(double noundef [[X:%.*]]) #[[ATTR14]]
-// DEFAULT-NEXT:    [[TOBOOL_I:%.*]] = icmp ne i32 [[CALL_I]], 0
-// DEFAULT-NEXT:    [[CONV:%.*]] = zext i1 [[TOBOOL_I]] to i32
+// DEFAULT-NEXT:    [[TMP0:%.*]] = tail call contract double @llvm.fabs.f64(double [[X:%.*]]) #[[ATTR17]]
+// DEFAULT-NEXT:    [[CMPINF_I:%.*]] = fcmp contract one double [[TMP0]], 0x7FF0000000000000
+// DEFAULT-NEXT:    [[CONV:%.*]] = zext i1 [[CMPINF_I]] to i32
 // DEFAULT-NEXT:    ret i32 [[CONV]]
 //
 // FINITEONLY-LABEL: @test___finite(
 // FINITEONLY-NEXT:  entry:
-// FINITEONLY-NEXT:    [[CALL_I:%.*]] = tail call i32 @__ocml_isfinite_f64(double noundef nofpclass(nan inf) [[X:%.*]]) #[[ATTR14]]
-// FINITEONLY-NEXT:    [[TOBOOL_I:%.*]] = icmp ne i32 [[CALL_I]], 0
-// FINITEONLY-NEXT:    [[CONV:%.*]] = zext i1 [[TOBOOL_I]] to i32
-// FINITEONLY-NEXT:    ret i32 [[CONV]]
+// FINITEONLY-NEXT:    ret i32 1
 //
 extern "C" __device__ BOOL_TYPE test___finite(double x) {
   return __finite(x);
@@ -1191,7 +1185,7 @@ extern "C" __device__ BOOL_TYPE test___finite(double x) {
 
 // DEFAULT-LABEL: @test___isinff(
 // DEFAULT-NEXT:  entry:
-// DEFAULT-NEXT:    [[TMP0:%.*]] = tail call contract float @llvm.fabs.f32(float [[X:%.*]]) #[[ATTR17:[0-9]+]]
+// DEFAULT-NEXT:    [[TMP0:%.*]] = tail call contract float @llvm.fabs.f32(float [[X:%.*]]) #[[ATTR17]]
 // DEFAULT-NEXT:    [[CMPINF_I:%.*]] = fcmp contract oeq float [[TMP0]], 0x7FF0000000000000
 // DEFAULT-NEXT:    [[CONV:%.*]] = zext i1 [[CMPINF_I]] to i32
 // DEFAULT-NEXT:    ret i32 [[CONV]]

From f407a7399575a6821940973c54754d42e72dd9ce Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Tue, 22 Nov 2022 11:24:09 -0500
Subject: [PATCH 016/130] clang/HIP: Remove __llvm_amdgcn_* wrapper hacks

These are leftover hacks from using asm declaratios to access
intrinsics.
---
 .../Headers/__clang_hip_libdevice_declares.h  | 32 -------------------
 clang/lib/Headers/__clang_hip_math.h          |  2 +-
 2 files changed, 1 insertion(+), 33 deletions(-)

diff --git a/clang/lib/Headers/__clang_hip_libdevice_declares.h b/clang/lib/Headers/__clang_hip_libdevice_declares.h
index be25f4b4a0506..2fc5e17d2f1ea 100644
--- a/clang/lib/Headers/__clang_hip_libdevice_declares.h
+++ b/clang/lib/Headers/__clang_hip_libdevice_declares.h
@@ -137,23 +137,6 @@ __device__ __attribute__((const)) float __ocml_fma_rte_f32(float, float, float);
 __device__ __attribute__((const)) float __ocml_fma_rtn_f32(float, float, float);
 __device__ __attribute__((const)) float __ocml_fma_rtp_f32(float, float, float);
 __device__ __attribute__((const)) float __ocml_fma_rtz_f32(float, float, float);
-
-__device__ inline __attribute__((const)) float
-__llvm_amdgcn_cos_f32(float __x) {
-  return __builtin_amdgcn_cosf(__x);
-}
-__device__ inline __attribute__((const)) float
-__llvm_amdgcn_rcp_f32(float __x) {
-  return __builtin_amdgcn_rcpf(__x);
-}
-__device__ inline __attribute__((const)) float
-__llvm_amdgcn_rsq_f32(float __x) {
-  return __builtin_amdgcn_rsqf(__x);
-}
-__device__ inline __attribute__((const)) float
-__llvm_amdgcn_sin_f32(float __x) {
-  return __builtin_amdgcn_sinf(__x);
-}
 // END INTRINSICS
 // END FLOAT
 
@@ -277,15 +260,6 @@ __device__ __attribute__((const)) double __ocml_fma_rtp_f64(double, double,
 __device__ __attribute__((const)) double __ocml_fma_rtz_f64(double, double,
                                                             double);
 
-__device__ inline __attribute__((const)) double
-__llvm_amdgcn_rcp_f64(double __x) {
-  return __builtin_amdgcn_rcp(__x);
-}
-__device__ inline __attribute__((const)) double
-__llvm_amdgcn_rsq_f64(double __x) {
-  return __builtin_amdgcn_rsq(__x);
-}
-
 __device__ __attribute__((const)) _Float16 __ocml_ceil_f16(_Float16);
 __device__ _Float16 __ocml_cos_f16(_Float16);
 __device__ __attribute__((const)) _Float16 __ocml_cvtrtn_f16_f32(float);
@@ -305,7 +279,6 @@ __device__ __attribute__((const)) int __ocml_isnan_f16(_Float16);
 __device__ __attribute__((pure)) _Float16 __ocml_log_f16(_Float16);
 __device__ __attribute__((pure)) _Float16 __ocml_log10_f16(_Float16);
 __device__ __attribute__((pure)) _Float16 __ocml_log2_f16(_Float16);
-__device__ __attribute__((const)) _Float16 __llvm_amdgcn_rcp_f16(_Float16);
 __device__ __attribute__((const)) _Float16 __ocml_rint_f16(_Float16);
 __device__ __attribute__((const)) _Float16 __ocml_rsqrt_f16(_Float16);
 __device__ _Float16 __ocml_sin_f16(_Float16);
@@ -332,11 +305,6 @@ __device__ __attribute__((const)) __2i16 __ocml_isnan_2f16(__2f16);
 __device__ __attribute__((pure)) __2f16 __ocml_log_2f16(__2f16);
 __device__ __attribute__((pure)) __2f16 __ocml_log10_2f16(__2f16);
 __device__ __attribute__((pure)) __2f16 __ocml_log2_2f16(__2f16);
-__device__ inline __2f16
-__llvm_amdgcn_rcp_2f16(__2f16 __x) // Not currently exposed by ROCDL.
-{
-  return (__2f16)(__llvm_amdgcn_rcp_f16(__x.x), __llvm_amdgcn_rcp_f16(__x.y));
-}
 __device__ __attribute__((const)) __2f16 __ocml_rint_2f16(__2f16);
 __device__ __attribute__((const)) __2f16 __ocml_rsqrt_2f16(__2f16);
 __device__ __2f16 __ocml_sin_2f16(__2f16);
diff --git a/clang/lib/Headers/__clang_hip_math.h b/clang/lib/Headers/__clang_hip_math.h
index f7949f30dfbae..2b33efcd8317a 100644
--- a/clang/lib/Headers/__clang_hip_math.h
+++ b/clang/lib/Headers/__clang_hip_math.h
@@ -647,7 +647,7 @@ float __frcp_rn(float __x) { return 1.0f / __x; }
 #endif
 
 __DEVICE__
-float __frsqrt_rn(float __x) { return __llvm_amdgcn_rsq_f32(__x); }
+float __frsqrt_rn(float __x) { return __builtin_amdgcn_rsqf(__x); }
 
 #if defined OCML_BASIC_ROUNDED_OPERATIONS
 __DEVICE__

From d8655fb3656f289fcd8816c7b11b30dc06661a2f Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Fri, 16 Jun 2023 09:52:23 -0400
Subject: [PATCH 017/130] [libc++][NFC] Rename __lower_bound_impl to
 __lower_bound

For consistency with other algorithms.

Differential Revision: https://reviews.llvm.org/D153141
---
 libcxx/include/__algorithm/equal_range.h          |  2 +-
 libcxx/include/__algorithm/lower_bound.h          |  4 ++--
 libcxx/include/__algorithm/ranges_binary_search.h |  4 ++--
 libcxx/include/__algorithm/ranges_lower_bound.h   |  4 ++--
 libcxx/include/__algorithm/ranges_upper_bound.h   | 12 ++++++------
 5 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/libcxx/include/__algorithm/equal_range.h b/libcxx/include/__algorithm/equal_range.h
index b731dcceb7886..dc1268a6ff110 100644
--- a/libcxx/include/__algorithm/equal_range.h
+++ b/libcxx/include/__algorithm/equal_range.h
@@ -50,7 +50,7 @@ __equal_range(_Iter __first, _Sent __last, const _Tp& __value, _Compare&& __comp
     } else {
       _Iter __mp1 = __mid;
       return pair<_Iter, _Iter>(
-          std::__lower_bound_impl<_AlgPolicy>(__first, __mid, __value, __comp, __proj),
+          std::__lower_bound<_AlgPolicy>(__first, __mid, __value, __comp, __proj),
           std::__upper_bound<_AlgPolicy>(++__mp1, __end, __value, __comp, __proj));
     }
   }
diff --git a/libcxx/include/__algorithm/lower_bound.h b/libcxx/include/__algorithm/lower_bound.h
index 7a7a2995cc300..91c3bdaafd0cf 100644
--- a/libcxx/include/__algorithm/lower_bound.h
+++ b/libcxx/include/__algorithm/lower_bound.h
@@ -29,7 +29,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _AlgPolicy, class _Iter, class _Sent, class _Type, class _Proj, class _Comp>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
-_Iter __lower_bound_impl(_Iter __first, _Sent __last, const _Type& __value, _Comp& __comp, _Proj& __proj) {
+_Iter __lower_bound(_Iter __first, _Sent __last, const _Type& __value, _Comp& __comp, _Proj& __proj) {
   auto __len = _IterOps<_AlgPolicy>::distance(__first, __last);
 
   while (__len != 0) {
@@ -52,7 +52,7 @@ _ForwardIterator lower_bound(_ForwardIterator __first, _ForwardIterator __last,
   static_assert(__is_callable<_Compare, decltype(*__first), const _Tp&>::value,
                 "The comparator has to be callable");
   auto __proj = std::__identity();
-  return std::__lower_bound_impl<_ClassicAlgPolicy>(__first, __last, __value, __comp, __proj);
+  return std::__lower_bound<_ClassicAlgPolicy>(__first, __last, __value, __comp, __proj);
 }
 
 template <class _ForwardIterator, class _Tp>
diff --git a/libcxx/include/__algorithm/ranges_binary_search.h b/libcxx/include/__algorithm/ranges_binary_search.h
index d89597e1be31c..728eb8b68ef76 100644
--- a/libcxx/include/__algorithm/ranges_binary_search.h
+++ b/libcxx/include/__algorithm/ranges_binary_search.h
@@ -35,7 +35,7 @@ struct __fn {
             indirect_strict_weak_order<const _Type*, projected<_Iter, _Proj>> _Comp = ranges::less>
   _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
   bool operator()(_Iter __first, _Sent __last, const _Type& __value, _Comp __comp = {}, _Proj __proj = {}) const {
-    auto __ret = std::__lower_bound_impl<_RangeAlgPolicy>(__first, __last, __value, __comp, __proj);
+    auto __ret = std::__lower_bound<_RangeAlgPolicy>(__first, __last, __value, __comp, __proj);
     return __ret != __last && !std::invoke(__comp, __value, std::invoke(__proj, *__ret));
   }
 
@@ -45,7 +45,7 @@ struct __fn {
   bool operator()(_Range&& __r, const _Type& __value, _Comp __comp = {}, _Proj __proj = {}) const {
     auto __first = ranges::begin(__r);
     auto __last = ranges::end(__r);
-    auto __ret = std::__lower_bound_impl<_RangeAlgPolicy>(__first, __last, __value, __comp, __proj);
+    auto __ret = std::__lower_bound<_RangeAlgPolicy>(__first, __last, __value, __comp, __proj);
     return __ret != __last && !std::invoke(__comp, __value, std::invoke(__proj, *__ret));
   }
 };
diff --git a/libcxx/include/__algorithm/ranges_lower_bound.h b/libcxx/include/__algorithm/ranges_lower_bound.h
index 74356394077b1..3293886fda4db 100644
--- a/libcxx/include/__algorithm/ranges_lower_bound.h
+++ b/libcxx/include/__algorithm/ranges_lower_bound.h
@@ -39,7 +39,7 @@ struct __fn {
             indirect_strict_weak_order<const _Type*, projected<_Iter, _Proj>> _Comp = ranges::less>
   _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
   _Iter operator()(_Iter __first, _Sent __last, const _Type& __value, _Comp __comp = {}, _Proj __proj = {}) const {
-    return std::__lower_bound_impl<_RangeAlgPolicy>(__first, __last, __value, __comp, __proj);
+    return std::__lower_bound<_RangeAlgPolicy>(__first, __last, __value, __comp, __proj);
   }
 
   template <forward_range _Range, class _Type, class _Proj = identity,
@@ -49,7 +49,7 @@ struct __fn {
                                          const _Type& __value,
                                          _Comp __comp = {},
                                          _Proj __proj = {}) const {
-    return std::__lower_bound_impl<_RangeAlgPolicy>(ranges::begin(__r), ranges::end(__r), __value, __comp, __proj);
+    return std::__lower_bound<_RangeAlgPolicy>(ranges::begin(__r), ranges::end(__r), __value, __comp, __proj);
   }
 };
 } // namespace __lower_bound
diff --git a/libcxx/include/__algorithm/ranges_upper_bound.h b/libcxx/include/__algorithm/ranges_upper_bound.h
index 43ce89b898beb..6e6f18dbc510b 100644
--- a/libcxx/include/__algorithm/ranges_upper_bound.h
+++ b/libcxx/include/__algorithm/ranges_upper_bound.h
@@ -40,7 +40,7 @@ struct __fn {
       return !std::invoke(__comp, __rhs, __lhs);
     };
 
-    return std::__lower_bound_impl<_RangeAlgPolicy>(__first, __last, __value, __comp_lhs_rhs_swapped, __proj);
+    return std::__lower_bound<_RangeAlgPolicy>(__first, __last, __value, __comp_lhs_rhs_swapped, __proj);
   }
 
   template <forward_range _Range, class _Type, class _Proj = identity,
@@ -54,11 +54,11 @@ struct __fn {
       return !std::invoke(__comp, __rhs, __lhs);
     };
 
-    return std::__lower_bound_impl<_RangeAlgPolicy>(ranges::begin(__r),
-                                                   ranges::end(__r),
-                                                   __value,
-                                                   __comp_lhs_rhs_swapped,
-                                                   __proj);
+    return std::__lower_bound<_RangeAlgPolicy>(ranges::begin(__r),
+                                               ranges::end(__r),
+                                               __value,
+                                               __comp_lhs_rhs_swapped,
+                                               __proj);
   }
 };
 } // namespace __upper_bound

From dc71a77d33620830020c939bba25ebdfdd1b6194 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Fri, 16 Jun 2023 11:11:56 -0400
Subject: [PATCH 018/130] [libc++][NFC] Move several .fail.cpp tests to
 .verify.cpp

A few tests were also straightforward to translate to SFINAE tests
instead, so in a few cases I did that and removed the .fail.cpp test
entirely.

Differential Revision: https://reviews.llvm.org/D153149
---
 .../fstreams/filebuf/traits_mismatch.fail.cpp | 35 ---------
 .../filebuf/traits_mismatch.verify.cpp        | 30 ++++++++
 ...ch.fail.cpp => traits_mismatch.verify.cpp} | 11 +--
 ...ch.fail.cpp => traits_mismatch.verify.cpp} | 13 +---
 ...ch.fail.cpp => traits_mismatch.verify.cpp} | 13 +---
 .../string.streams/traits_mismatch.fail.cpp   | 34 ---------
 .../string.streams/traits_mismatch.verify.cpp | 29 ++++++++
 ...> thread_safety_missing_unlock.verify.cpp} |  4 +-
 ..._tuple_ref_binding_diagnostics.verify.cpp} |  6 +-
 ...fail.cpp => pair.tuple_element.verify.cpp} | 13 ++--
 .../{get.fail.cpp => get.verify.cpp}          | 19 ++---
 ...ment.fail.cpp => tuple_element.verify.cpp} | 15 ++--
 .../views/views.span/span.cons/array.fail.cpp | 72 -------------------
 .../views/views.span/span.cons/array.pass.cpp | 41 ++++++++++-
 .../views.span/span.cons/default.fail.cpp     | 29 --------
 ...othrow_t.fail.cpp => nothrow_t.verify.cpp} |  7 +-
 .../{types.fail.cpp => types.verify.cpp}      |  7 +-
 ..._arg.fail.cpp => allocator_arg.verify.cpp} |  7 +-
 ...ail.cpp => optional_in_place_t.verify.cpp} | 12 +---
 ...fail.cpp => optional_nullopt_t.verify.cpp} | 18 ++---
 .../{alloc.fail.cpp => alloc.verify.cpp}      |  4 --
 ....fail.cpp => alloc_const_Types.verify.cpp} |  7 --
 ...fail.cpp => alloc_convert_copy.verify.cpp} |  7 --
 ...fail.cpp => alloc_convert_move.verify.cpp} |  7 --
 ..._Types.fail.cpp => const_Types.verify.cpp} |  6 --
 .../{default.fail.cpp => default.verify.cpp}  |  4 --
 ...st_rv.fail.cpp => get_const_rv.verify.cpp} |  7 +-
 ...type.fail.cpp => tuple.by.type.verify.cpp} |  5 +-
 ...ment.fail.cpp => tuple_element.verify.cpp} | 15 ++--
 ...le_size.fail.cpp => tuple_size.verify.cpp} |  5 +-
 ...l.cpp => tuple_size_incomplete.verify.cpp} |  5 +-
 ...ize_v.fail.cpp => tuple_size_v.verify.cpp} |  5 +-
 .../piecewise_construct_t.verify.cpp          |  2 +-
 .../ctor.default.explicit_LWG2510.verify.cpp  |  2 +-
 34 files changed, 148 insertions(+), 348 deletions(-)
 delete mode 100644 libcxx/test/libcxx/input.output/file.streams/fstreams/filebuf/traits_mismatch.fail.cpp
 create mode 100644 libcxx/test/libcxx/input.output/file.streams/fstreams/filebuf/traits_mismatch.verify.cpp
 rename libcxx/test/libcxx/input.output/file.streams/fstreams/{traits_mismatch.fail.cpp => traits_mismatch.verify.cpp} (83%)
 rename libcxx/test/libcxx/input.output/iostream.format/input.streams/{traits_mismatch.fail.cpp => traits_mismatch.verify.cpp} (69%)
 rename libcxx/test/libcxx/input.output/iostream.format/output.streams/{traits_mismatch.fail.cpp => traits_mismatch.verify.cpp} (69%)
 delete mode 100644 libcxx/test/libcxx/input.output/string.streams/traits_mismatch.fail.cpp
 create mode 100644 libcxx/test/libcxx/input.output/string.streams/traits_mismatch.verify.cpp
 rename libcxx/test/libcxx/thread/thread.mutex/{thread_safety_missing_unlock.fail.cpp => thread_safety_missing_unlock.verify.cpp} (95%)
 rename libcxx/test/libcxx/utilities/tuple/tuple.tuple/tuple.cnstr/{PR20855_tuple_ref_binding_diagnostics.fail.cpp => PR20855_tuple_ref_binding_diagnostics.verify.cpp} (97%)
 rename libcxx/test/libcxx/utilities/utility/pairs/pairs.pair/{pair.tuple_element.fail.cpp => pair.tuple_element.verify.cpp} (58%)
 rename libcxx/test/std/containers/sequences/array/array.tuple/{get.fail.cpp => get.verify.cpp} (59%)
 rename libcxx/test/std/containers/sequences/array/array.tuple/{tuple_element.fail.cpp => tuple_element.verify.cpp} (54%)
 delete mode 100644 libcxx/test/std/containers/views/views.span/span.cons/array.fail.cpp
 delete mode 100644 libcxx/test/std/containers/views/views.span/span.cons/default.fail.cpp
 rename libcxx/test/std/language.support/support.dynamic/{nothrow_t.fail.cpp => nothrow_t.verify.cpp} (88%)
 rename libcxx/test/std/thread/thread.mutex/thread.lock/{types.fail.cpp => types.verify.cpp} (92%)
 rename libcxx/test/std/utilities/memory/allocator.tag/{allocator_arg.fail.cpp => allocator_arg.verify.cpp} (89%)
 rename libcxx/test/std/utilities/optional/optional.syn/{optional_in_place_t.fail.cpp => optional_in_place_t.verify.cpp} (78%)
 rename libcxx/test/std/utilities/optional/optional.syn/{optional_nullopt_t.fail.cpp => optional_nullopt_t.verify.cpp} (62%)
 rename libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/{alloc.fail.cpp => alloc.verify.cpp} (95%)
 rename libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/{alloc_const_Types.fail.cpp => alloc_const_Types.verify.cpp} (91%)
 rename libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/{alloc_convert_copy.fail.cpp => alloc_convert_copy.verify.cpp} (96%)
 rename libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/{alloc_convert_move.fail.cpp => alloc_convert_move.verify.cpp} (95%)
 rename libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/{const_Types.fail.cpp => const_Types.verify.cpp} (96%)
 rename libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/{default.fail.cpp => default.verify.cpp} (98%)
 rename libcxx/test/std/utilities/tuple/tuple.tuple/tuple.elem/{get_const_rv.fail.cpp => get_const_rv.verify.cpp} (87%)
 rename libcxx/test/std/utilities/tuple/tuple.tuple/tuple.elem/{tuple.by.type.fail.cpp => tuple.by.type.verify.cpp} (97%)
 rename libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/{tuple_element.fail.cpp => tuple_element.verify.cpp} (59%)
 rename libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/{tuple_size.fail.cpp => tuple_size.verify.cpp} (96%)
 rename libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/{tuple_size_incomplete.fail.cpp => tuple_size_incomplete.verify.cpp} (97%)
 rename libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/{tuple_size_v.fail.cpp => tuple_size_v.verify.cpp} (96%)

diff --git a/libcxx/test/libcxx/input.output/file.streams/fstreams/filebuf/traits_mismatch.fail.cpp b/libcxx/test/libcxx/input.output/file.streams/fstreams/filebuf/traits_mismatch.fail.cpp
deleted file mode 100644
index 27871b0c7bea5..0000000000000
--- a/libcxx/test/libcxx/input.output/file.streams/fstreams/filebuf/traits_mismatch.fail.cpp
+++ /dev/null
@@ -1,35 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// <fstream>
-
-// template<class charT, class traits = char_traits<charT>>
-//   class basic_filebuf;
-//
-// The char type of the stream and the char_type of the traits have to match
-
-// UNSUPPORTED: no-wide-characters
-
-#include <fstream>
-
-int main(int, char**)
-{
-  std::basic_filebuf<char, std::char_traits<wchar_t> > f;
-  // expected-error-re@streambuf:* {{{{(static_assert|static assertion)}} failed{{.*}}traits_type::char_type must be the same type as CharT}}
-  // expected-error@fstream:* {{only virtual member functions can be marked 'override'}}
-  // expected-error@fstream:* {{only virtual member functions can be marked 'override'}}
-  // expected-error@fstream:* {{only virtual member functions can be marked 'override'}}
-  // expected-error@fstream:* {{only virtual member functions can be marked 'override'}}
-  // expected-error@fstream:* {{only virtual member functions can be marked 'override'}}
-  // expected-error@fstream:* {{only virtual member functions can be marked 'override'}}
-  // expected-error@fstream:* {{only virtual member functions can be marked 'override'}}
-  // expected-error@fstream:* {{only virtual member functions can be marked 'override'}}
-  // expected-error@fstream:* {{only virtual member functions can be marked 'override'}}
-
-  return 0;
-}
diff --git a/libcxx/test/libcxx/input.output/file.streams/fstreams/filebuf/traits_mismatch.verify.cpp b/libcxx/test/libcxx/input.output/file.streams/fstreams/filebuf/traits_mismatch.verify.cpp
new file mode 100644
index 0000000000000..e6f651ca41d32
--- /dev/null
+++ b/libcxx/test/libcxx/input.output/file.streams/fstreams/filebuf/traits_mismatch.verify.cpp
@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <fstream>
+
+// template<class charT, class traits = char_traits<charT>>
+//   class basic_filebuf;
+//
+// The char type of the stream and the char_type of the traits have to match
+
+// UNSUPPORTED: no-wide-characters
+
+#include <fstream>
+
+std::basic_filebuf<char, std::char_traits<wchar_t> > f;
+// expected-error-re@streambuf:* {{{{(static_assert|static assertion)}} failed{{.*}}traits_type::char_type must be the same type as CharT}}
+// expected-error@fstream:* {{only virtual member functions can be marked 'override'}}
+// expected-error@fstream:* {{only virtual member functions can be marked 'override'}}
+// expected-error@fstream:* {{only virtual member functions can be marked 'override'}}
+// expected-error@fstream:* {{only virtual member functions can be marked 'override'}}
+// expected-error@fstream:* {{only virtual member functions can be marked 'override'}}
+// expected-error@fstream:* {{only virtual member functions can be marked 'override'}}
+// expected-error@fstream:* {{only virtual member functions can be marked 'override'}}
+// expected-error@fstream:* {{only virtual member functions can be marked 'override'}}
+// expected-error@fstream:* {{only virtual member functions can be marked 'override'}}
diff --git a/libcxx/test/libcxx/input.output/file.streams/fstreams/traits_mismatch.fail.cpp b/libcxx/test/libcxx/input.output/file.streams/fstreams/traits_mismatch.verify.cpp
similarity index 83%
rename from libcxx/test/libcxx/input.output/file.streams/fstreams/traits_mismatch.fail.cpp
rename to libcxx/test/libcxx/input.output/file.streams/fstreams/traits_mismatch.verify.cpp
index ce22fee618732..94299fe0af473 100644
--- a/libcxx/test/libcxx/input.output/file.streams/fstreams/traits_mismatch.fail.cpp
+++ b/libcxx/test/libcxx/input.output/file.streams/fstreams/traits_mismatch.verify.cpp
@@ -17,11 +17,9 @@
 
 #include <fstream>
 
-int main(int, char**)
-{
-  std::basic_fstream<char, std::char_traits<wchar_t> > f;
-//  expected-error-re@ios:* {{{{(static_assert|static assertion)}} failed{{.*}}traits_type::char_type must be the same type as CharT}}
-//  expected-error-re@streambuf:* {{{{(static_assert|static assertion)}} failed{{.*}}traits_type::char_type must be the same type as CharT}}
+std::basic_fstream<char, std::char_traits<wchar_t> > f;
+// expected-error-re@ios:* {{{{(static_assert|static assertion)}} failed{{.*}}traits_type::char_type must be the same type as CharT}}
+// expected-error-re@streambuf:* {{{{(static_assert|static assertion)}} failed{{.*}}traits_type::char_type must be the same type as CharT}}
 
 // expected-error@fstream:* {{only virtual member functions can be marked 'override'}}
 // expected-error@fstream:* {{only virtual member functions can be marked 'override'}}
@@ -39,6 +37,3 @@ int main(int, char**)
 // exception specifications for types which are already invalid for one reason or another.
 // For now we tolerate this diagnostic.
 // expected-error@ostream:* 0-1 {{exception specification of overriding function is more lax than base version}}
-
-  return 0;
-}
diff --git a/libcxx/test/libcxx/input.output/iostream.format/input.streams/traits_mismatch.fail.cpp b/libcxx/test/libcxx/input.output/iostream.format/input.streams/traits_mismatch.verify.cpp
similarity index 69%
rename from libcxx/test/libcxx/input.output/iostream.format/input.streams/traits_mismatch.fail.cpp
rename to libcxx/test/libcxx/input.output/iostream.format/input.streams/traits_mismatch.verify.cpp
index 273dd0fe4c857..5789e8e2b07b1 100644
--- a/libcxx/test/libcxx/input.output/iostream.format/input.streams/traits_mismatch.fail.cpp
+++ b/libcxx/test/libcxx/input.output/iostream.format/input.streams/traits_mismatch.verify.cpp
@@ -16,17 +16,10 @@
 // UNSUPPORTED: no-wide-characters
 
 #include <istream>
-#include <type_traits>
-#include <cassert>
+#include <string>
 
 struct test_istream
     : public std::basic_istream<char, std::char_traits<wchar_t> > {};
 
-
-int main(int, char**)
-{
-//  expected-error-re@ios:* {{{{(static_assert|static assertion)}} failed{{.*}}traits_type::char_type must be the same type as CharT}}
-//  expected-error@istream:* {{only virtual member functions can be marked 'override'}}
-
-  return 0;
-}
+// expected-error-re@ios:* {{{{(static_assert|static assertion)}} failed{{.*}}traits_type::char_type must be the same type as CharT}}
+// expected-error@istream:* {{only virtual member functions can be marked 'override'}}
diff --git a/libcxx/test/libcxx/input.output/iostream.format/output.streams/traits_mismatch.fail.cpp b/libcxx/test/libcxx/input.output/iostream.format/output.streams/traits_mismatch.verify.cpp
similarity index 69%
rename from libcxx/test/libcxx/input.output/iostream.format/output.streams/traits_mismatch.fail.cpp
rename to libcxx/test/libcxx/input.output/iostream.format/output.streams/traits_mismatch.verify.cpp
index 9afcde0d8c68c..6f8ba7baaa1e2 100644
--- a/libcxx/test/libcxx/input.output/iostream.format/output.streams/traits_mismatch.fail.cpp
+++ b/libcxx/test/libcxx/input.output/iostream.format/output.streams/traits_mismatch.verify.cpp
@@ -16,17 +16,10 @@
 // UNSUPPORTED: no-wide-characters
 
 #include <ostream>
-#include <type_traits>
-#include <cassert>
+#include <string>
 
 struct test_ostream
     : public std::basic_ostream<char, std::char_traits<wchar_t> > {};
 
-
-int main(int, char**)
-{
-//  expected-error-re@ios:* {{{{(static_assert|static assertion)}} failed{{.*}}traits_type::char_type must be the same type as CharT}}
-//  expected-error@ostream:* {{only virtual member functions can be marked 'override'}}
-
-  return 0;
-}
+// expected-error-re@ios:* {{{{(static_assert|static assertion)}} failed{{.*}}traits_type::char_type must be the same type as CharT}}
+// expected-error@ostream:* {{only virtual member functions can be marked 'override'}}
diff --git a/libcxx/test/libcxx/input.output/string.streams/traits_mismatch.fail.cpp b/libcxx/test/libcxx/input.output/string.streams/traits_mismatch.fail.cpp
deleted file mode 100644
index 23ab5f0c88a0b..0000000000000
--- a/libcxx/test/libcxx/input.output/string.streams/traits_mismatch.fail.cpp
+++ /dev/null
@@ -1,34 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// <sstream>
-
-//  template<class charT, class traits = char_traits<charT>,
-//           class Allocator = allocator<charT>>
-//    class basic_stringbuf;
-//
-// The char type of the stream and the char_type of the traits have to match
-
-// UNSUPPORTED: no-wide-characters
-
-#include <sstream>
-
-int main(int, char**)
-{
-  std::basic_stringbuf<char, std::char_traits<wchar_t> > sb;
-//  expected-error-re@streambuf:* {{{{(static_assert|static assertion)}} failed{{.*}}traits_type::char_type must be the same type as CharT}}
-//  expected-error-re@string:* {{{{(static_assert|static assertion)}} failed{{.*}}traits_type::char_type must be the same type as CharT}}
-
-  // expected-error@sstream:* {{only virtual member functions can be marked 'override'}}
-  // expected-error@sstream:* {{only virtual member functions can be marked 'override'}}
-  // expected-error@sstream:* {{only virtual member functions can be marked 'override'}}
-  // expected-error@sstream:* {{only virtual member functions can be marked 'override'}}
-  // expected-error@sstream:* {{only virtual member functions can be marked 'override'}}
-
-  return 0;
-}
diff --git a/libcxx/test/libcxx/input.output/string.streams/traits_mismatch.verify.cpp b/libcxx/test/libcxx/input.output/string.streams/traits_mismatch.verify.cpp
new file mode 100644
index 0000000000000..36c4b23eb2978
--- /dev/null
+++ b/libcxx/test/libcxx/input.output/string.streams/traits_mismatch.verify.cpp
@@ -0,0 +1,29 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <sstream>
+
+//  template<class charT, class traits = char_traits<charT>,
+//           class Allocator = allocator<charT>>
+//    class basic_stringbuf;
+//
+// The char type of the stream and the char_type of the traits have to match
+
+// UNSUPPORTED: no-wide-characters
+
+#include <sstream>
+
+std::basic_stringbuf<char, std::char_traits<wchar_t> > sb;
+// expected-error-re@streambuf:* {{{{(static_assert|static assertion)}} failed{{.*}}traits_type::char_type must be the same type as CharT}}
+// expected-error-re@string:* {{{{(static_assert|static assertion)}} failed{{.*}}traits_type::char_type must be the same type as CharT}}
+
+// expected-error@sstream:* {{only virtual member functions can be marked 'override'}}
+// expected-error@sstream:* {{only virtual member functions can be marked 'override'}}
+// expected-error@sstream:* {{only virtual member functions can be marked 'override'}}
+// expected-error@sstream:* {{only virtual member functions can be marked 'override'}}
+// expected-error@sstream:* {{only virtual member functions can be marked 'override'}}
diff --git a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_missing_unlock.fail.cpp b/libcxx/test/libcxx/thread/thread.mutex/thread_safety_missing_unlock.verify.cpp
similarity index 95%
rename from libcxx/test/libcxx/thread/thread.mutex/thread_safety_missing_unlock.fail.cpp
rename to libcxx/test/libcxx/thread/thread.mutex/thread_safety_missing_unlock.verify.cpp
index 07a51b5db90cf..bab686e7b4ca2 100644
--- a/libcxx/test/libcxx/thread/thread.mutex/thread_safety_missing_unlock.fail.cpp
+++ b/libcxx/test/libcxx/thread/thread.mutex/thread_safety_missing_unlock.verify.cpp
@@ -21,8 +21,6 @@
 
 std::mutex m;
 
-int main(int, char**) {
+void f() {
   m.lock();
-
-  return 0;
 } // expected-error {{mutex 'm' is still held at the end of function}}
diff --git a/libcxx/test/libcxx/utilities/tuple/tuple.tuple/tuple.cnstr/PR20855_tuple_ref_binding_diagnostics.fail.cpp b/libcxx/test/libcxx/utilities/tuple/tuple.tuple/tuple.cnstr/PR20855_tuple_ref_binding_diagnostics.verify.cpp
similarity index 97%
rename from libcxx/test/libcxx/utilities/tuple/tuple.tuple/tuple.cnstr/PR20855_tuple_ref_binding_diagnostics.fail.cpp
rename to libcxx/test/libcxx/utilities/tuple/tuple.tuple/tuple.cnstr/PR20855_tuple_ref_binding_diagnostics.verify.cpp
index 2586c32ca80ec..4a6e3095c1019 100644
--- a/libcxx/test/libcxx/utilities/tuple/tuple.tuple/tuple.cnstr/PR20855_tuple_ref_binding_diagnostics.fail.cpp
+++ b/libcxx/test/libcxx/utilities/tuple/tuple.tuple/tuple.cnstr/PR20855_tuple_ref_binding_diagnostics.verify.cpp
@@ -10,7 +10,7 @@
 
 // <tuple>
 
-// See llvm.org/PR20855
+// See https://llvm.org/PR20855
 
 #include <tuple>
 #include <string>
@@ -39,7 +39,7 @@ template <class ...Args>
 void F(typename CannotDeduce<std::tuple<Args...>>::type const&) {}
 
 
-int main(int, char**) {
+void f() {
 #if TEST_HAS_BUILTIN_IDENTIFIER(__reference_binds_to_temporary)
   // Test that we emit our diagnostic from the library.
   // expected-error@tuple:* 8 {{Attempted construction of reference element binds to a temporary whose lifetime has ended}}
@@ -77,6 +77,4 @@ int main(int, char**) {
 #error force failure
 // expected-error@-1 {{force failure}}
 #endif
-
-  return 0;
 }
diff --git a/libcxx/test/libcxx/utilities/utility/pairs/pairs.pair/pair.tuple_element.fail.cpp b/libcxx/test/libcxx/utilities/utility/pairs/pairs.pair/pair.tuple_element.verify.cpp
similarity index 58%
rename from libcxx/test/libcxx/utilities/utility/pairs/pairs.pair/pair.tuple_element.fail.cpp
rename to libcxx/test/libcxx/utilities/utility/pairs/pairs.pair/pair.tuple_element.verify.cpp
index 40e57df508959..c21afe5caae25 100644
--- a/libcxx/test/libcxx/utilities/utility/pairs/pairs.pair/pair.tuple_element.fail.cpp
+++ b/libcxx/test/libcxx/utilities/utility/pairs/pairs.pair/pair.tuple_element.verify.cpp
@@ -14,13 +14,8 @@
 
 #include <utility>
 
-int main(int, char**)
-{
-    {
-    typedef std::pair<int, double> P;
-    std::tuple_element<2, P>::type foo; // expected-note {{requested here}}
-        // expected-error-re@*:* {{{{(static_assert|static assertion)}} failed{{( due to requirement '2U[L]{0,2} < 2')?}}{{.*}}Index out of bounds in std::tuple_element<std::pair<T1, T2>>}}
-    }
-
-  return 0;
+void f() {
+  typedef std::pair<int, double> P;
+  std::tuple_element<2, P>::type foo; // expected-note {{requested here}}
+    // expected-error-re@*:* {{{{(static_assert|static assertion)}} failed{{( due to requirement '2U[L]{0,2} < 2')?}}{{.*}}Index out of bounds in std::tuple_element<std::pair<T1, T2>>}}
 }
diff --git a/libcxx/test/std/containers/sequences/array/array.tuple/get.fail.cpp b/libcxx/test/std/containers/sequences/array/array.tuple/get.verify.cpp
similarity index 59%
rename from libcxx/test/std/containers/sequences/array/array.tuple/get.fail.cpp
rename to libcxx/test/std/containers/sequences/array/array.tuple/get.verify.cpp
index 2e1071550a51c..a447a1113d4af 100644
--- a/libcxx/test/std/containers/sequences/array/array.tuple/get.fail.cpp
+++ b/libcxx/test/std/containers/sequences/array/array.tuple/get.verify.cpp
@@ -16,17 +16,10 @@
 #include <array>
 #include <cassert>
 
-#include "test_macros.h"
-
-int main(int, char**)
-{
-    {
-        typedef double T;
-        typedef std::array<T, 3> C;
-        C c = {1, 2, 3.5};
-        std::get<3>(c) = 5.5; // expected-note {{requested here}}
-        // expected-error-re@array:* {{{{(static_assert|static assertion)}} failed{{( due to requirement '3U[L]{0,2} < 3U[L]{0,2}')?}}{{.*}}Index out of bounds in std::get<> (std::array)}}
-    }
-
-  return 0;
+void f() {
+  typedef double T;
+  typedef std::array<T, 3> C;
+  C c = {1, 2, 3.5};
+  std::get<3>(c) = 5.5; // expected-note {{requested here}}
+  // expected-error-re@array:* {{{{(static_assert|static assertion)}} failed{{( due to requirement '3U[L]{0,2} < 3U[L]{0,2}')?}}{{.*}}Index out of bounds in std::get<> (std::array)}}
 }
diff --git a/libcxx/test/std/containers/sequences/array/array.tuple/tuple_element.fail.cpp b/libcxx/test/std/containers/sequences/array/array.tuple/tuple_element.verify.cpp
similarity index 54%
rename from libcxx/test/std/containers/sequences/array/array.tuple/tuple_element.fail.cpp
rename to libcxx/test/std/containers/sequences/array/array.tuple/tuple_element.verify.cpp
index 6010c893d6995..e0ae7f10fbb2f 100644
--- a/libcxx/test/std/containers/sequences/array/array.tuple/tuple_element.fail.cpp
+++ b/libcxx/test/std/containers/sequences/array/array.tuple/tuple_element.verify.cpp
@@ -13,14 +13,7 @@
 #include <array>
 #include <cassert>
 
-int main(int, char**)
-{
-    {
-        typedef double T;
-        typedef std::array<T, 3> C;
-        std::tuple_element<3, C> foo; // expected-note {{requested here}}
-        // expected-error-re@array:* {{{{(static_assert|static assertion)}} failed{{( due to requirement '3U[L]{0,2} < 3U[L]{0,2}')?}}{{.*}}Index out of bounds in std::tuple_element<> (std::array)}}
-    }
-
-  return 0;
-}
+typedef double T;
+typedef std::array<T, 3> C;
+std::tuple_element<3, C> foo; // expected-note {{requested here}}
+// expected-error-re@array:* {{{{(static_assert|static assertion)}} failed{{( due to requirement '3U[L]{0,2} < 3U[L]{0,2}')?}}{{.*}}Index out of bounds in std::tuple_element<> (std::array)}}
diff --git a/libcxx/test/std/containers/views/views.span/span.cons/array.fail.cpp b/libcxx/test/std/containers/views/views.span/span.cons/array.fail.cpp
deleted file mode 100644
index 74682167c6d27..0000000000000
--- a/libcxx/test/std/containers/views/views.span/span.cons/array.fail.cpp
+++ /dev/null
@@ -1,72 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// <span>
-
-// template<size_t N>
-//     constexpr span(element_type (&arr)[N]) noexcept;
-// template<size_t N>
-//     constexpr span(array<value_type, N>& arr) noexcept;
-// template<size_t N>
-//     constexpr span(const array<value_type, N>& arr) noexcept;
-//
-// Remarks: These constructors shall not participate in overload resolution unless:
-//   - extent == dynamic_extent || N == extent is true, and
-//   - remove_pointer_t<decltype(data(arr))>(*)[] is convertible to ElementType(*)[].
-//
-
-
-#include <span>
-#include <cassert>
-#include <string>
-
-#include "test_macros.h"
-
-               int   arr[] = {1,2,3};
-const          int  carr[] = {4,5,6};
-      volatile int  varr[] = {7,8,9};
-const volatile int cvarr[] = {1,3,5};
-
-int main(int, char**)
-{
-//  Size wrong
-    {
-    std::span<int, 2>   s1(arr); // expected-error {{no matching constructor for initialization of 'std::span<int, 2>'}}
-    }
-
-//  Type wrong
-    {
-    std::span<float>    s1(arr);   // expected-error {{no matching constructor for initialization of 'std::span<float>'}}
-    std::span<float, 3> s2(arr);   // expected-error {{no matching constructor for initialization of 'std::span<float, 3>'}}
-    }
-
-//  CV wrong (dynamically sized)
-    {
-    std::span<               int> s1{ carr};    // expected-error {{no matching constructor for initialization of 'std::span<int>'}}
-    std::span<               int> s2{ varr};    // expected-error {{no matching constructor for initialization of 'std::span<int>'}}
-    std::span<               int> s3{cvarr};    // expected-error {{no matching constructor for initialization of 'std::span<int>'}}
-    std::span<const          int> s4{ varr};    // expected-error {{no matching constructor for initialization of 'std::span<const int>'}}
-    std::span<const          int> s5{cvarr};    // expected-error {{no matching constructor for initialization of 'std::span<const int>'}}
-    std::span<      volatile int> s6{ carr};    // expected-error {{no matching constructor for initialization of 'std::span<volatile int>'}}
-    std::span<      volatile int> s7{cvarr};    // expected-error {{no matching constructor for initialization of 'std::span<volatile int>'}}
-    }
-
-//  CV wrong (statically sized)
-    {
-    std::span<               int,3> s1{ carr};  // expected-error {{no matching constructor for initialization of 'std::span<int, 3>'}}
-    std::span<               int,3> s2{ varr};  // expected-error {{no matching constructor for initialization of 'std::span<int, 3>'}}
-    std::span<               int,3> s3{cvarr};  // expected-error {{no matching constructor for initialization of 'std::span<int, 3>'}}
-    std::span<const          int,3> s4{ varr};  // expected-error {{no matching constructor for initialization of 'std::span<const int, 3>'}}
-    std::span<const          int,3> s5{cvarr};  // expected-error {{no matching constructor for initialization of 'std::span<const int, 3>'}}
-    std::span<      volatile int,3> s6{ carr};  // expected-error {{no matching constructor for initialization of 'std::span<volatile int, 3>'}}
-    std::span<      volatile int,3> s7{cvarr};  // expected-error {{no matching constructor for initialization of 'std::span<volatile int, 3>'}}
-    }
-
-  return 0;
-}
diff --git a/libcxx/test/std/containers/views/views.span/span.cons/array.pass.cpp b/libcxx/test/std/containers/views/views.span/span.cons/array.pass.cpp
index ce03c4fa92475..8fa7692c3b637 100644
--- a/libcxx/test/std/containers/views/views.span/span.cons/array.pass.cpp
+++ b/libcxx/test/std/containers/views/views.span/span.cons/array.pass.cpp
@@ -17,10 +17,10 @@
 //   - remove_pointer_t<decltype(data(arr))>(*)[] is convertible to ElementType(*)[].
 //
 
-
-#include <span>
 #include <cassert>
+#include <span>
 #include <string>
+#include <type_traits>
 
 #include "test_macros.h"
 
@@ -117,5 +117,42 @@ int main(int, char**)
 
     checkCV();
 
+    // Size wrong
+    {
+        static_assert(!std::is_constructible<std::span<int, 2>, int (&)[3]>::value, "");
+    }
+
+    // Type wrong
+    {
+        static_assert(!std::is_constructible<std::span<float>,    int (&)[3]>::value, "");
+        static_assert(!std::is_constructible<std::span<float, 3>, int (&)[3]>::value, "");
+    }
+
+    // CV wrong (dynamically sized)
+    {
+        static_assert(!std::is_constructible<std::span<int>, const int (&)[3]>::value, "");
+        static_assert(!std::is_constructible<std::span<int>, volatile int (&)[3]>::value, "");
+        static_assert(!std::is_constructible<std::span<int>, const volatile int (&)[3]>::value, "");
+
+        static_assert(!std::is_constructible<std::span<const int>, volatile int (&)[3]>::value, "");
+        static_assert(!std::is_constructible<std::span<const int>, const volatile int (&)[3]>::value, "");
+
+        static_assert(!std::is_constructible<std::span<volatile int>, const int (&)[3]>::value, "");
+        static_assert(!std::is_constructible<std::span<volatile int>, const volatile int (&)[3]>::value, "");
+    }
+
+    // CV wrong (statically sized)
+    {
+        static_assert(!std::is_constructible<std::span<int, 3>, const int (&)[3]>::value, "");
+        static_assert(!std::is_constructible<std::span<int, 3>, volatile int (&)[3]>::value, "");
+        static_assert(!std::is_constructible<std::span<int, 3>, const volatile int (&)[3]>::value, "");
+
+        static_assert(!std::is_constructible<std::span<const int, 3>, volatile int (&)[3]>::value, "");
+        static_assert(!std::is_constructible<std::span<const int, 3>, const volatile int (&)[3]>::value, "");
+
+        static_assert(!std::is_constructible<std::span<volatile int, 3>, const int (&)[3]>::value, "");
+        static_assert(!std::is_constructible<std::span<volatile int, 3>, const volatile int (&)[3]>::value, "");
+    }
+
     return 0;
 }
diff --git a/libcxx/test/std/containers/views/views.span/span.cons/default.fail.cpp b/libcxx/test/std/containers/views/views.span/span.cons/default.fail.cpp
deleted file mode 100644
index 2c62278aac867..0000000000000
--- a/libcxx/test/std/containers/views/views.span/span.cons/default.fail.cpp
+++ /dev/null
@@ -1,29 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// <span>
-
-// constexpr span() noexcept;
-//
-//  Remarks: This constructor shall not participate in overload resolution
-//          unless Extent == 0 || Extent == dynamic_extent is true.
-
-
-#include <span>
-#include <cassert>
-#include <string>
-
-#include "test_macros.h"
-
-int main(int, char**)
-{
-  std::span<int, 2> s; // expected-error {{no matching constructor for initialization of 'std::span<int, 2>'}}
-
-  return 0;
-}
diff --git a/libcxx/test/std/language.support/support.dynamic/nothrow_t.fail.cpp b/libcxx/test/std/language.support/support.dynamic/nothrow_t.verify.cpp
similarity index 88%
rename from libcxx/test/std/language.support/support.dynamic/nothrow_t.fail.cpp
rename to libcxx/test/std/language.support/support.dynamic/nothrow_t.verify.cpp
index 5ec2696a3c9c2..50dd63a6350f0 100644
--- a/libcxx/test/std/language.support/support.dynamic/nothrow_t.fail.cpp
+++ b/libcxx/test/std/language.support/support.dynamic/nothrow_t.verify.cpp
@@ -14,13 +14,8 @@
 // };
 // extern const nothrow_t nothrow;
 
-// This test checks for LWG 2510.
+// This test checks for https://wg21.link/LWG2510.
 
 #include <new>
 
-
 std::nothrow_t f() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
-
-int main(int, char**) {
-    return 0;
-}
diff --git a/libcxx/test/std/thread/thread.mutex/thread.lock/types.fail.cpp b/libcxx/test/std/thread/thread.mutex/thread.lock/types.verify.cpp
similarity index 92%
rename from libcxx/test/std/thread/thread.mutex/thread.lock/types.fail.cpp
rename to libcxx/test/std/thread/thread.mutex/thread.lock/types.verify.cpp
index b4acac85b7e30..623e5c3d22e75 100644
--- a/libcxx/test/std/thread/thread.mutex/thread.lock/types.fail.cpp
+++ b/libcxx/test/std/thread/thread.mutex/thread.lock/types.verify.cpp
@@ -16,15 +16,10 @@
 // struct try_to_lock_t { explicit try_to_lock_t() = default; };
 // struct adopt_lock_t { explicit adopt_lock_t() = default; };
 
-// This test checks for LWG 2510.
+// This test checks for https://wg21.link/LWG2510.
 
 #include <mutex>
 
-
 std::defer_lock_t f1() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
 std::try_to_lock_t f2() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
 std::adopt_lock_t f3() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
-
-int main(int, char**) {
-    return 0;
-}
diff --git a/libcxx/test/std/utilities/memory/allocator.tag/allocator_arg.fail.cpp b/libcxx/test/std/utilities/memory/allocator.tag/allocator_arg.verify.cpp
similarity index 89%
rename from libcxx/test/std/utilities/memory/allocator.tag/allocator_arg.fail.cpp
rename to libcxx/test/std/utilities/memory/allocator.tag/allocator_arg.verify.cpp
index 94752af96fa27..db5a85aa752bd 100644
--- a/libcxx/test/std/utilities/memory/allocator.tag/allocator_arg.fail.cpp
+++ b/libcxx/test/std/utilities/memory/allocator.tag/allocator_arg.verify.cpp
@@ -13,13 +13,8 @@
 // struct allocator_arg_t { explicit allocator_arg_t() = default; };
 // const allocator_arg_t allocator_arg = allocator_arg_t();
 
-// This test checks for LWG 2510.
+// This test checks for https://wg21.link/LWG2510.
 
 #include <memory>
 
-
 std::allocator_arg_t f() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
-
-int main(int, char**) {
-    return 0;
-}
diff --git a/libcxx/test/std/utilities/optional/optional.syn/optional_in_place_t.fail.cpp b/libcxx/test/std/utilities/optional/optional.syn/optional_in_place_t.verify.cpp
similarity index 78%
rename from libcxx/test/std/utilities/optional/optional.syn/optional_in_place_t.fail.cpp
rename to libcxx/test/std/utilities/optional/optional.syn/optional_in_place_t.verify.cpp
index b95cb5606b5f2..20efef515c362 100644
--- a/libcxx/test/std/utilities/optional/optional.syn/optional_in_place_t.fail.cpp
+++ b/libcxx/test/std/utilities/optional/optional.syn/optional_in_place_t.verify.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++03, c++11, c++14
+
 // <optional>
 
 // A program that necessitates the instantiation of template optional for
@@ -14,14 +15,7 @@
 
 #include <optional>
 
-int main(int, char**)
-{
-    using std::optional;
-    using std::in_place_t;
-    using std::in_place;
-
-    optional<in_place_t> opt; // expected-note {{requested here}}
+void f() {
+    std::optional<std::in_place_t> opt; // expected-note {{requested here}}
     // expected-error@optional:* {{instantiation of optional with in_place_t is ill-formed}}
-
-  return 0;
 }
diff --git a/libcxx/test/std/utilities/optional/optional.syn/optional_nullopt_t.fail.cpp b/libcxx/test/std/utilities/optional/optional.syn/optional_nullopt_t.verify.cpp
similarity index 62%
rename from libcxx/test/std/utilities/optional/optional.syn/optional_nullopt_t.fail.cpp
rename to libcxx/test/std/utilities/optional/optional.syn/optional_nullopt_t.verify.cpp
index 044c4e66572aa..f807da7470ae9 100644
--- a/libcxx/test/std/utilities/optional/optional.syn/optional_nullopt_t.fail.cpp
+++ b/libcxx/test/std/utilities/optional/optional.syn/optional_nullopt_t.verify.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++03, c++11, c++14
+
 // <optional>
 
 // A program that necessitates the instantiation of template optional for
@@ -14,17 +15,10 @@
 
 #include <optional>
 
-int main(int, char**)
-{
-    using std::optional;
-    using std::nullopt_t;
-    using std::nullopt;
-
-    optional<nullopt_t> opt; // expected-note 1 {{requested here}}
-    optional<const nullopt_t> opt1; // expected-note 1 {{requested here}}
-    optional<nullopt_t &> opt2; // expected-note 1 {{requested here}}
-    optional<nullopt_t &&> opt3; // expected-note 1 {{requested here}}
+void f() {
+    std::optional<std::nullopt_t> opt; // expected-note 1 {{requested here}}
+    std::optional<const std::nullopt_t> opt1; // expected-note 1 {{requested here}}
+    std::optional<std::nullopt_t &> opt2; // expected-note 1 {{requested here}}
+    std::optional<std::nullopt_t &&> opt3; // expected-note 1 {{requested here}}
     // expected-error@optional:* 4 {{instantiation of optional with nullopt_t is ill-formed}}
-
-  return 0;
 }
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc.fail.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc.verify.cpp
similarity index 95%
rename from libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc.fail.cpp
rename to libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc.verify.cpp
index 2a89996937220..0ce92da28cc42 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc.fail.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc.verify.cpp
@@ -27,7 +27,3 @@ struct ExplicitDefault { explicit ExplicitDefault() { } };
 std::tuple<ExplicitDefault> explicit_default_test() {
     return {std::allocator_arg, std::allocator<int>()}; // expected-error {{chosen constructor is explicit in copy-initialization}}
 }
-
-int main(int, char**) {
-    return 0;
-}
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc_const_Types.fail.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc_const_Types.verify.cpp
similarity index 91%
rename from libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc_const_Types.fail.cpp
rename to libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc_const_Types.verify.cpp
index 1a1e8d2f9d54c..21ec4e07053ca 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc_const_Types.fail.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc_const_Types.verify.cpp
@@ -35,10 +35,3 @@ std::tuple<ExplicitCopy> non_const_explicity_copy_test() {
     return {std::allocator_arg, std::allocator<int>{}, e};
     // expected-error@-1 {{chosen constructor is explicit in copy-initialization}}
 }
-int main(int, char**)
-{
-    const_explicit_copy_test();
-    non_const_explicity_copy_test();
-
-  return 0;
-}
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc_convert_copy.fail.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc_convert_copy.verify.cpp
similarity index 96%
rename from libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc_convert_copy.fail.cpp
rename to libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc_convert_copy.verify.cpp
index b5b7de75344aa..ec8490787294c 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc_convert_copy.fail.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc_convert_copy.verify.cpp
@@ -35,10 +35,3 @@ std::tuple<ExplicitCopy> non_const_explicit_copy_test() {
     return {std::allocator_arg, std::allocator<int>{}, t1};
     // expected-error@-1 {{chosen constructor is explicit in copy-initialization}}
 }
-
-int main(int, char**)
-{
-
-
-  return 0;
-}
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc_convert_move.fail.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc_convert_move.verify.cpp
similarity index 95%
rename from libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc_convert_move.fail.cpp
rename to libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc_convert_move.verify.cpp
index 9202086cdae11..0fa60d54a23b0 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc_convert_move.fail.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/alloc_convert_move.verify.cpp
@@ -28,10 +28,3 @@ std::tuple<ExplicitCopy> explicit_move_test() {
     return {std::allocator_arg, std::allocator<int>{}, std::move(t1)};
     // expected-error@-1 {{chosen constructor is explicit in copy-initialization}}
 }
-
-int main(int, char**)
-{
-
-
-  return 0;
-}
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/const_Types.fail.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/const_Types.verify.cpp
similarity index 96%
rename from libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/const_Types.fail.cpp
rename to libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/const_Types.verify.cpp
index 7b5773e183ae0..ab35585a02993 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/const_Types.fail.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/const_Types.verify.cpp
@@ -41,9 +41,3 @@ std::tuple<ExplicitCopy> const_explicit_copy_no_brace() {
     return e;
     // expected-error@-1 {{no viable conversion}}
 }
-
-int main(int, char**)
-{
-
-  return 0;
-}
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/default.fail.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/default.verify.cpp
similarity index 98%
rename from libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/default.fail.cpp
rename to libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/default.verify.cpp
index ed49c739d2baf..61951d80dad26 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/default.fail.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.cnstr/default.verify.cpp
@@ -43,7 +43,3 @@ std::tuple<Explicit, Implicit, Implicit> test12() { return {}; } // expected-err
 std::tuple<Explicit, Implicit, Explicit> test13() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
 std::tuple<Explicit, Explicit, Implicit> test14() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
 std::tuple<Explicit, Explicit, Explicit> test15() { return {}; } // expected-error 1 {{chosen constructor is explicit in copy-initialization}}
-
-int main(int, char**) {
-    return 0;
-}
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.elem/get_const_rv.fail.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.elem/get_const_rv.verify.cpp
similarity index 87%
rename from libcxx/test/std/utilities/tuple/tuple.tuple/tuple.elem/get_const_rv.fail.cpp
rename to libcxx/test/std/utilities/tuple/tuple.tuple/tuple.elem/get_const_rv.verify.cpp
index ff1e37db8e6ad..2595ea871d5b1 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.elem/get_const_rv.fail.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.elem/get_const_rv.verify.cpp
@@ -23,13 +23,10 @@ template <class T> void cref(T const&&) = delete;
 
 std::tuple<int> const tup4() { return std::make_tuple(4); }
 
-int main(int, char**)
-{
+void f() {
     // LWG2485: tuple should not open a hole in the type system, get() should
     // imitate [expr.ref]'s rules for accessing data members
     {
-        cref(std::get<0>(tup4()));  // expected-error {{call to deleted function 'cref'}}
+        cref(std::get<0>(tup4())); // expected-error {{call to deleted function 'cref'}}
     }
-
-  return 0;
 }
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.elem/tuple.by.type.fail.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.elem/tuple.by.type.verify.cpp
similarity index 97%
rename from libcxx/test/std/utilities/tuple/tuple.tuple/tuple.elem/tuple.by.type.fail.cpp
rename to libcxx/test/std/utilities/tuple/tuple.tuple/tuple.elem/tuple.by.type.verify.cpp
index 4fef6b99f5305..1d05eb5fe76e9 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.elem/tuple.by.type.fail.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.elem/tuple.by.type.verify.cpp
@@ -33,10 +33,7 @@ void test_bad_return_type() {
     upint p = std::get<upint>(t); // expected-error{{deleted copy constructor}}
 }
 
-int main(int, char**)
-{
+void f() {
     test_bad_index();
     test_bad_return_type();
-
-  return 0;
 }
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_element.fail.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_element.verify.cpp
similarity index 59%
rename from libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_element.fail.cpp
rename to libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_element.verify.cpp
index 21b69683b680a..03eb8f40280a4 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_element.fail.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_element.verify.cpp
@@ -19,15 +19,8 @@
 // UNSUPPORTED: c++03
 
 #include <tuple>
-#include <type_traits>
 
-int main(int, char**)
-{
-    using T =  std::tuple<int, long, void*>;
-    using E1 = typename std::tuple_element<1, T &>::type; // expected-error{{undefined template}}
-    using E2 = typename std::tuple_element<3, T>::type;
-    using E3 = typename std::tuple_element<4, T const>::type;
-        // expected-error-re@*:* 2 {{{{(static_assert|static assertion)}} failed}}
-
-  return 0;
-}
+using T =  std::tuple<int, long, void*>;
+using E1 = typename std::tuple_element<1, T &>::type; // expected-error{{undefined template}}
+using E2 = typename std::tuple_element<3, T>::type;
+using E3 = typename std::tuple_element<4, T const>::type; // expected-error-re@*:* 2 {{{{(static_assert|static assertion)}} failed}}
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size.fail.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size.verify.cpp
similarity index 96%
rename from libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size.fail.cpp
rename to libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size.verify.cpp
index 2a4374570eced..27a821ad3bf2e 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size.fail.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size.verify.cpp
@@ -18,11 +18,8 @@
 
 #include <tuple>
 
-int main(int, char**)
-{
+void f() {
     (void)std::tuple_size<std::tuple<> &>::value; // expected-error {{implicit instantiation of undefined template}}
     (void)std::tuple_size<int>::value; // expected-error {{implicit instantiation of undefined template}}
     (void)std::tuple_size<std::tuple<>*>::value; // expected-error {{implicit instantiation of undefined template}}
-
-  return 0;
 }
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size_incomplete.fail.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size_incomplete.verify.cpp
similarity index 97%
rename from libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size_incomplete.fail.cpp
rename to libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size_incomplete.verify.cpp
index a2a65e0ebb8b5..532ccd730ffc1 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size_incomplete.fail.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size_incomplete.verify.cpp
@@ -39,8 +39,7 @@ struct std::tuple_size<Dummy2> {
 template <>
 struct std::tuple_size<Dummy3> {};
 
-int main(int, char**)
-{
+void f() {
   // Test that tuple_size<const T> is not incomplete when tuple_size<T>::value
   // is well-formed but not a constant expression.
   {
@@ -59,6 +58,4 @@ int main(int, char**)
     // expected-error@*:* 1 {{no member named 'value'}}
     (void)std::tuple_size<const Dummy3>::value; // expected-note {{here}}
   }
-
-  return 0;
 }
diff --git a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size_v.fail.cpp b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size_v.verify.cpp
similarity index 96%
rename from libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size_v.fail.cpp
rename to libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size_v.verify.cpp
index 6b1ee6884b39b..7570230b4b074 100644
--- a/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size_v.fail.cpp
+++ b/libcxx/test/std/utilities/tuple/tuple.tuple/tuple.helper/tuple_size_v.verify.cpp
@@ -16,12 +16,9 @@
 
 #include <tuple>
 
-int main(int, char**)
-{
+void f() {
     (void)std::tuple_size_v<std::tuple<> &>; // expected-note {{requested here}}
     (void)std::tuple_size_v<int>; // expected-note {{requested here}}
     (void)std::tuple_size_v<std::tuple<>*>; // expected-note {{requested here}}
     // expected-error@tuple:* 3 {{implicit instantiation of undefined template}}
-
-  return 0;
 }
diff --git a/libcxx/test/std/utilities/utility/pairs/pair.piecewise/piecewise_construct_t.verify.cpp b/libcxx/test/std/utilities/utility/pairs/pair.piecewise/piecewise_construct_t.verify.cpp
index a72a99da4e9dd..fff5044866d91 100644
--- a/libcxx/test/std/utilities/utility/pairs/pair.piecewise/piecewise_construct_t.verify.cpp
+++ b/libcxx/test/std/utilities/utility/pairs/pair.piecewise/piecewise_construct_t.verify.cpp
@@ -13,7 +13,7 @@
 // struct piecewise_construct_t { explicit piecewise_construct_t() = default; };
 // constexpr piecewise_construct_t piecewise_construct = piecewise_construct_t();
 
-// This test checks for LWG 2510.
+// This test checks for https://wg21.link/LWG2510.
 
 #include <utility>
 
diff --git a/libcxx/test/std/utilities/utility/pairs/pairs.pair/ctor.default.explicit_LWG2510.verify.cpp b/libcxx/test/std/utilities/utility/pairs/pairs.pair/ctor.default.explicit_LWG2510.verify.cpp
index 50a4eb74800a0..18b9ab32887aa 100644
--- a/libcxx/test/std/utilities/utility/pairs/pairs.pair/ctor.default.explicit_LWG2510.verify.cpp
+++ b/libcxx/test/std/utilities/utility/pairs/pairs.pair/ctor.default.explicit_LWG2510.verify.cpp
@@ -15,7 +15,7 @@
 // explicit(see-below) constexpr pair();
 
 // This test checks the conditional explicitness of std::pair's default
-// constructor as introduced by the resolution of LWG 2510.
+// constructor as introduced by the resolution of https://wg21.link/LWG2510.
 
 #include <utility>
 

From c7d3c84449f403716a8866e50491a1860a935b30 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Mon, 12 Jun 2023 10:43:55 -0700
Subject: [PATCH 019/130] [libc++] Split sources for <filesystem>

The operations.cpp file contained the implementation of a ton of
functionality unrelated to just the filesystem operations, and
filesystem_common.h contained a lot of unrelated functionality as well.

Splitting this up into more files will make it possible in the future
to support parts of <filesystem> (e.g. path) on systems where there is
no notion of a filesystem.

Differential Revision: https://reviews.llvm.org/D152377
---
 libcxx/src/CMakeLists.txt                     |   10 +-
 libcxx/src/filesystem/directory_entry.cpp     |   74 ++
 libcxx/src/filesystem/directory_iterator.cpp  |   12 +-
 libcxx/src/filesystem/error.h                 |  246 ++++
 libcxx/src/filesystem/file_descriptor.h       |  283 ++++
 libcxx/src/filesystem/filesystem_clock.cpp    |   54 +
 libcxx/src/filesystem/filesystem_error.cpp    |   37 +
 libcxx/src/filesystem/format_string.h         |   89 ++
 libcxx/src/filesystem/operations.cpp          | 1145 +----------------
 libcxx/src/filesystem/path.cpp                |  460 +++++++
 libcxx/src/filesystem/path_parser.h           |  379 ++++++
 libcxx/src/filesystem/posix_compat.h          |   50 +-
 .../{filesystem_common.h => time_utils.h}     |  330 +----
 .../last_write_time.pass.cpp                  |    8 +-
 .../filesystems/convert_file_time.pass.cpp    |    4 +-
 libcxx/utils/data/ignore_format.txt           |    9 +-
 16 files changed, 1732 insertions(+), 1458 deletions(-)
 create mode 100644 libcxx/src/filesystem/directory_entry.cpp
 create mode 100644 libcxx/src/filesystem/error.h
 create mode 100644 libcxx/src/filesystem/file_descriptor.h
 create mode 100644 libcxx/src/filesystem/filesystem_clock.cpp
 create mode 100644 libcxx/src/filesystem/filesystem_error.cpp
 create mode 100644 libcxx/src/filesystem/format_string.h
 create mode 100644 libcxx/src/filesystem/path.cpp
 create mode 100644 libcxx/src/filesystem/path_parser.h
 rename libcxx/src/filesystem/{filesystem_common.h => time_utils.h} (55%)

diff --git a/libcxx/src/CMakeLists.txt b/libcxx/src/CMakeLists.txt
index 358a61e4a0351..dede38652e2f9 100644
--- a/libcxx/src/CMakeLists.txt
+++ b/libcxx/src/CMakeLists.txt
@@ -110,10 +110,16 @@ endif()
 
 if (LIBCXX_ENABLE_FILESYSTEM)
   list(APPEND LIBCXX_SOURCES
-    filesystem/filesystem_common.h
-    filesystem/operations.cpp
+    filesystem/directory_entry.cpp
     filesystem/directory_iterator.cpp
+    filesystem/file_descriptor.h
+    filesystem/filesystem_clock.cpp
+    filesystem/filesystem_error.cpp
+    filesystem/operations.cpp
+    filesystem/path_parser.h
+    filesystem/path.cpp
     filesystem/posix_compat.h
+    filesystem/time_utils.h
     )
   # Filesystem uses __int128_t, which requires a definition of __muloi4 when
   # compiled with UBSAN. This definition is not provided by libgcc_s, but is
diff --git a/libcxx/src/filesystem/directory_entry.cpp b/libcxx/src/filesystem/directory_entry.cpp
new file mode 100644
index 0000000000000..4910d390d1ea7
--- /dev/null
+++ b/libcxx/src/filesystem/directory_entry.cpp
@@ -0,0 +1,74 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <__config>
+#include <cstdint>
+#include <filesystem>
+#include <system_error>
+
+#include "file_descriptor.h"
+#include "posix_compat.h"
+#include "time_utils.h"
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+error_code directory_entry::__do_refresh() noexcept {
+  __data_.__reset();
+  error_code failure_ec;
+
+  detail::StatT full_st;
+  file_status st = detail::posix_lstat(__p_, full_st, &failure_ec);
+  if (!status_known(st)) {
+    __data_.__reset();
+    return failure_ec;
+  }
+
+  if (!_VSTD_FS::exists(st) || !_VSTD_FS::is_symlink(st)) {
+    __data_.__cache_type_ = directory_entry::_RefreshNonSymlink;
+    __data_.__type_ = st.type();
+    __data_.__non_sym_perms_ = st.permissions();
+  } else { // we have a symlink
+    __data_.__sym_perms_ = st.permissions();
+    // Get the information about the linked entity.
+    // Ignore errors from stat, since we don't want errors regarding symlink
+    // resolution to be reported to the user.
+    error_code ignored_ec;
+    st = detail::posix_stat(__p_, full_st, &ignored_ec);
+
+    __data_.__type_ = st.type();
+    __data_.__non_sym_perms_ = st.permissions();
+
+    // If we failed to resolve the link, then only partially populate the
+    // cache.
+    if (!status_known(st)) {
+      __data_.__cache_type_ = directory_entry::_RefreshSymlinkUnresolved;
+      return error_code{};
+    }
+    // Otherwise, we resolved the link, potentially as not existing.
+    // That's OK.
+    __data_.__cache_type_ = directory_entry::_RefreshSymlink;
+  }
+
+  if (_VSTD_FS::is_regular_file(st))
+    __data_.__size_ = static_cast<uintmax_t>(full_st.st_size);
+
+  if (_VSTD_FS::exists(st)) {
+    __data_.__nlink_ = static_cast<uintmax_t>(full_st.st_nlink);
+
+    // Attempt to extract the mtime, and fail if it's not representable using
+    // file_time_type. For now we ignore the error, as we'll report it when
+    // the value is actually used.
+    error_code ignored_ec;
+    __data_.__write_time_ =
+        detail::__extract_last_write_time(__p_, full_st, &ignored_ec);
+  }
+
+  return failure_ec;
+}
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
diff --git a/libcxx/src/filesystem/directory_iterator.cpp b/libcxx/src/filesystem/directory_iterator.cpp
index 8d069cc080d89..4a7e01b09655f 100644
--- a/libcxx/src/filesystem/directory_iterator.cpp
+++ b/libcxx/src/filesystem/directory_iterator.cpp
@@ -11,8 +11,18 @@
 #include <errno.h>
 #include <filesystem>
 #include <stack>
+#include <utility>
 
-#include "filesystem_common.h"
+#include "error.h"
+#include "file_descriptor.h"
+
+#if defined(_LIBCPP_WIN32API)
+# define WIN32_LEAN_AND_MEAN
+# define NOMINMAX
+# include <windows.h>
+#else
+# include <dirent.h>   // for DIR & friends
+#endif
 
 _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
 
diff --git a/libcxx/src/filesystem/error.h b/libcxx/src/filesystem/error.h
new file mode 100644
index 0000000000000..72696007ce0e5
--- /dev/null
+++ b/libcxx/src/filesystem/error.h
@@ -0,0 +1,246 @@
+//===----------------------------------------------------------------------===////
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===////
+
+#ifndef FILESYSTEM_ERROR_H
+#define FILESYSTEM_ERROR_H
+
+#include <__assert>
+#include <__config>
+#include <cerrno>
+#include <cstdarg>
+#include <cstddef>
+#include <cstdint>
+#include <filesystem>
+#include <string>
+#include <system_error>
+#include <utility> // __libcpp_unreachable
+
+#include "format_string.h"
+
+#if defined(_LIBCPP_WIN32API)
+# define WIN32_LEAN_AND_MEAN
+# define NOMINMAX
+# include <windows.h> // ERROR_* macros
+#endif
+
+// TODO: Check whether these functions actually need internal linkage, or if they can be made normal header functions
+_LIBCPP_DIAGNOSTIC_PUSH
+_LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wunused-function")
+_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wunused-function")
+_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wunused-template")
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+namespace detail {
+namespace {
+
+#if defined(_LIBCPP_WIN32API)
+
+errc __win_err_to_errc(int err) {
+  constexpr struct {
+    DWORD win;
+    errc errc;
+  } win_error_mapping[] = {
+      {ERROR_ACCESS_DENIED, errc::permission_denied},
+      {ERROR_ALREADY_EXISTS, errc::file_exists},
+      {ERROR_BAD_NETPATH, errc::no_such_file_or_directory},
+      {ERROR_BAD_PATHNAME, errc::no_such_file_or_directory},
+      {ERROR_BAD_UNIT, errc::no_such_device},
+      {ERROR_BROKEN_PIPE, errc::broken_pipe},
+      {ERROR_BUFFER_OVERFLOW, errc::filename_too_long},
+      {ERROR_BUSY, errc::device_or_resource_busy},
+      {ERROR_BUSY_DRIVE, errc::device_or_resource_busy},
+      {ERROR_CANNOT_MAKE, errc::permission_denied},
+      {ERROR_CANTOPEN, errc::io_error},
+      {ERROR_CANTREAD, errc::io_error},
+      {ERROR_CANTWRITE, errc::io_error},
+      {ERROR_CURRENT_DIRECTORY, errc::permission_denied},
+      {ERROR_DEV_NOT_EXIST, errc::no_such_device},
+      {ERROR_DEVICE_IN_USE, errc::device_or_resource_busy},
+      {ERROR_DIR_NOT_EMPTY, errc::directory_not_empty},
+      {ERROR_DIRECTORY, errc::invalid_argument},
+      {ERROR_DISK_FULL, errc::no_space_on_device},
+      {ERROR_FILE_EXISTS, errc::file_exists},
+      {ERROR_FILE_NOT_FOUND, errc::no_such_file_or_directory},
+      {ERROR_HANDLE_DISK_FULL, errc::no_space_on_device},
+      {ERROR_INVALID_ACCESS, errc::permission_denied},
+      {ERROR_INVALID_DRIVE, errc::no_such_device},
+      {ERROR_INVALID_FUNCTION, errc::function_not_supported},
+      {ERROR_INVALID_HANDLE, errc::invalid_argument},
+      {ERROR_INVALID_NAME, errc::no_such_file_or_directory},
+      {ERROR_INVALID_PARAMETER, errc::invalid_argument},
+      {ERROR_LOCK_VIOLATION, errc::no_lock_available},
+      {ERROR_LOCKED, errc::no_lock_available},
+      {ERROR_NEGATIVE_SEEK, errc::invalid_argument},
+      {ERROR_NOACCESS, errc::permission_denied},
+      {ERROR_NOT_ENOUGH_MEMORY, errc::not_enough_memory},
+      {ERROR_NOT_READY, errc::resource_unavailable_try_again},
+      {ERROR_NOT_SAME_DEVICE, errc::cross_device_link},
+      {ERROR_NOT_SUPPORTED, errc::not_supported},
+      {ERROR_OPEN_FAILED, errc::io_error},
+      {ERROR_OPEN_FILES, errc::device_or_resource_busy},
+      {ERROR_OPERATION_ABORTED, errc::operation_canceled},
+      {ERROR_OUTOFMEMORY, errc::not_enough_memory},
+      {ERROR_PATH_NOT_FOUND, errc::no_such_file_or_directory},
+      {ERROR_READ_FAULT, errc::io_error},
+      {ERROR_REPARSE_TAG_INVALID, errc::invalid_argument},
+      {ERROR_RETRY, errc::resource_unavailable_try_again},
+      {ERROR_SEEK, errc::io_error},
+      {ERROR_SHARING_VIOLATION, errc::permission_denied},
+      {ERROR_TOO_MANY_OPEN_FILES, errc::too_many_files_open},
+      {ERROR_WRITE_FAULT, errc::io_error},
+      {ERROR_WRITE_PROTECT, errc::permission_denied},
+  };
+
+  for (const auto &pair : win_error_mapping)
+    if (pair.win == static_cast<DWORD>(err))
+      return pair.errc;
+  return errc::invalid_argument;
+}
+
+#endif // _LIBCPP_WIN32API
+
+error_code capture_errno() {
+  _LIBCPP_ASSERT(errno != 0, "Expected errno to be non-zero");
+  return error_code(errno, generic_category());
+}
+
+#if defined(_LIBCPP_WIN32API)
+error_code make_windows_error(int err) {
+  return make_error_code(__win_err_to_errc(err));
+}
+#endif
+
+template <class T>
+T error_value();
+template <>
+_LIBCPP_CONSTEXPR_SINCE_CXX14 void error_value<void>() {}
+template <>
+bool error_value<bool>() {
+  return false;
+}
+#if __SIZEOF_SIZE_T__ != __SIZEOF_LONG_LONG__
+template <>
+size_t error_value<size_t>() {
+  return size_t(-1);
+}
+#endif
+template <>
+uintmax_t error_value<uintmax_t>() {
+  return uintmax_t(-1);
+}
+template <>
+_LIBCPP_CONSTEXPR_SINCE_CXX14 file_time_type error_value<file_time_type>() {
+  return file_time_type::min();
+}
+template <>
+path error_value<path>() {
+  return {};
+}
+
+template <class T>
+struct ErrorHandler {
+  const char* func_name_;
+  error_code* ec_ = nullptr;
+  const path* p1_ = nullptr;
+  const path* p2_ = nullptr;
+
+  ErrorHandler(const char* fname, error_code* ec, const path* p1 = nullptr,
+               const path* p2 = nullptr)
+      : func_name_(fname), ec_(ec), p1_(p1), p2_(p2) {
+    if (ec_)
+      ec_->clear();
+  }
+
+  T report(const error_code& ec) const {
+    if (ec_) {
+      *ec_ = ec;
+      return error_value<T>();
+    }
+    string what = string("in ") + func_name_;
+    switch (bool(p1_) + bool(p2_)) {
+    case 0:
+      __throw_filesystem_error(what, ec);
+    case 1:
+      __throw_filesystem_error(what, *p1_, ec);
+    case 2:
+      __throw_filesystem_error(what, *p1_, *p2_, ec);
+    }
+    __libcpp_unreachable();
+  }
+
+  _LIBCPP_ATTRIBUTE_FORMAT(__printf__, 3, 0)
+  void report_impl(const error_code& ec, const char* msg, va_list ap) const {
+    if (ec_) {
+      *ec_ = ec;
+      return;
+    }
+    string what =
+        string("in ") + func_name_ + ": " + detail::vformat_string(msg, ap);
+    switch (bool(p1_) + bool(p2_)) {
+    case 0:
+      __throw_filesystem_error(what, ec);
+    case 1:
+      __throw_filesystem_error(what, *p1_, ec);
+    case 2:
+      __throw_filesystem_error(what, *p1_, *p2_, ec);
+    }
+    __libcpp_unreachable();
+  }
+
+  _LIBCPP_ATTRIBUTE_FORMAT(__printf__, 3, 4)
+  T report(const error_code& ec, const char* msg, ...) const {
+    va_list ap;
+    va_start(ap, msg);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      report_impl(ec, msg, ap);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      va_end(ap);
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    va_end(ap);
+    return error_value<T>();
+  }
+
+  T report(errc const& err) const {
+    return report(make_error_code(err));
+  }
+
+  _LIBCPP_ATTRIBUTE_FORMAT(__printf__, 3, 4)
+  T report(errc const& err, const char* msg, ...) const {
+    va_list ap;
+    va_start(ap, msg);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+      report_impl(make_error_code(err), msg, ap);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+    } catch (...) {
+      va_end(ap);
+      throw;
+    }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    va_end(ap);
+    return error_value<T>();
+  }
+
+private:
+  ErrorHandler(ErrorHandler const&) = delete;
+  ErrorHandler& operator=(ErrorHandler const&) = delete;
+};
+
+} // end anonymous namespace
+} // end namespace detail
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+#endif // FILESYSTEM_ERROR_H
diff --git a/libcxx/src/filesystem/file_descriptor.h b/libcxx/src/filesystem/file_descriptor.h
new file mode 100644
index 0000000000000..e9520424d301f
--- /dev/null
+++ b/libcxx/src/filesystem/file_descriptor.h
@@ -0,0 +1,283 @@
+//===----------------------------------------------------------------------===////
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===////
+
+#ifndef FILESYSTEM_FILE_DESCRIPTOR_H
+#define FILESYSTEM_FILE_DESCRIPTOR_H
+
+#include <__config>
+#include <cstdint>
+#include <filesystem>
+#include <string_view>
+#include <system_error>
+#include <utility>
+
+#include "error.h"
+#include "posix_compat.h"
+
+#if defined(_LIBCPP_WIN32API)
+# define WIN32_LEAN_AND_MEAN
+# define NOMINMAX
+# include <windows.h>
+#else
+# include <dirent.h>   // for DIR & friends
+# include <fcntl.h>    // values for fchmodat
+# include <sys/stat.h>
+# include <sys/statvfs.h>
+# include <unistd.h>
+#endif // defined(_LIBCPP_WIN32API)
+
+// TODO: Check whether these functions actually need internal linkage, or if they can be made normal header functions
+_LIBCPP_DIAGNOSTIC_PUSH
+_LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wunused-function")
+_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wunused-function")
+_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wunused-template")
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+namespace detail {
+namespace {
+
+#if !defined(_LIBCPP_WIN32API)
+
+#if defined(DT_BLK)
+template <class DirEntT, class = decltype(DirEntT::d_type)>
+static file_type get_file_type(DirEntT* ent, int) {
+  switch (ent->d_type) {
+  case DT_BLK:
+    return file_type::block;
+  case DT_CHR:
+    return file_type::character;
+  case DT_DIR:
+    return file_type::directory;
+  case DT_FIFO:
+    return file_type::fifo;
+  case DT_LNK:
+    return file_type::symlink;
+  case DT_REG:
+    return file_type::regular;
+  case DT_SOCK:
+    return file_type::socket;
+  // Unlike in lstat, hitting "unknown" here simply means that the underlying
+  // filesystem doesn't support d_type. Report is as 'none' so we correctly
+  // set the cache to empty.
+  case DT_UNKNOWN:
+    break;
+  }
+  return file_type::none;
+}
+#endif // defined(DT_BLK)
+
+template <class DirEntT>
+static file_type get_file_type(DirEntT*, long) {
+  return file_type::none;
+}
+
+static pair<string_view, file_type> posix_readdir(DIR* dir_stream,
+                                                  error_code& ec) {
+  struct dirent* dir_entry_ptr = nullptr;
+  errno = 0; // zero errno in order to detect errors
+  ec.clear();
+  if ((dir_entry_ptr = ::readdir(dir_stream)) == nullptr) {
+    if (errno)
+      ec = capture_errno();
+    return {};
+  } else {
+    return {dir_entry_ptr->d_name, get_file_type(dir_entry_ptr, 0)};
+  }
+}
+
+#else // _LIBCPP_WIN32API
+
+static file_type get_file_type(const WIN32_FIND_DATAW& data) {
+  if (data.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT &&
+      data.dwReserved0 == IO_REPARSE_TAG_SYMLINK)
+    return file_type::symlink;
+  if (data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
+    return file_type::directory;
+  return file_type::regular;
+}
+static uintmax_t get_file_size(const WIN32_FIND_DATAW& data) {
+  return (static_cast<uint64_t>(data.nFileSizeHigh) << 32) + data.nFileSizeLow;
+}
+static file_time_type get_write_time(const WIN32_FIND_DATAW& data) {
+  ULARGE_INTEGER tmp;
+  const FILETIME& time = data.ftLastWriteTime;
+  tmp.u.LowPart = time.dwLowDateTime;
+  tmp.u.HighPart = time.dwHighDateTime;
+  return file_time_type(file_time_type::duration(tmp.QuadPart));
+}
+
+#endif // !_LIBCPP_WIN32API
+
+//                       POSIX HELPERS
+
+using value_type = path::value_type;
+using string_type = path::string_type;
+
+struct FileDescriptor {
+  const path& name;
+  int fd = -1;
+  StatT m_stat;
+  file_status m_status;
+
+  template <class... Args>
+  static FileDescriptor create(const path* p, error_code& ec, Args... args) {
+    ec.clear();
+    int fd;
+    if ((fd = detail::open(p->c_str(), args...)) == -1) {
+      ec = capture_errno();
+      return FileDescriptor{p};
+    }
+    return FileDescriptor(p, fd);
+  }
+
+  template <class... Args>
+  static FileDescriptor create_with_status(const path* p, error_code& ec,
+                                           Args... args) {
+    FileDescriptor fd = create(p, ec, args...);
+    if (!ec)
+      fd.refresh_status(ec);
+
+    return fd;
+  }
+
+  file_status get_status() const { return m_status; }
+  StatT const& get_stat() const { return m_stat; }
+
+  bool status_known() const { return _VSTD_FS::status_known(m_status); }
+
+  file_status refresh_status(error_code& ec);
+
+  void close() noexcept {
+    if (fd != -1)
+      detail::close(fd);
+    fd = -1;
+  }
+
+  FileDescriptor(FileDescriptor&& other)
+      : name(other.name), fd(other.fd), m_stat(other.m_stat),
+        m_status(other.m_status) {
+    other.fd = -1;
+    other.m_status = file_status{};
+  }
+
+  ~FileDescriptor() { close(); }
+
+  FileDescriptor(FileDescriptor const&) = delete;
+  FileDescriptor& operator=(FileDescriptor const&) = delete;
+
+private:
+  explicit FileDescriptor(const path* p, int descriptor = -1) : name(*p), fd(descriptor) {}
+};
+
+perms posix_get_perms(const StatT& st) noexcept {
+  return static_cast<perms>(st.st_mode) & perms::mask;
+}
+
+file_status create_file_status(error_code& m_ec, path const& p,
+                               const StatT& path_stat, error_code* ec) {
+  if (ec)
+    *ec = m_ec;
+  if (m_ec && (m_ec.value() == ENOENT || m_ec.value() == ENOTDIR)) {
+    return file_status(file_type::not_found);
+  } else if (m_ec) {
+    ErrorHandler<void> err("posix_stat", ec, &p);
+    err.report(m_ec, "failed to determine attributes for the specified path");
+    return file_status(file_type::none);
+  }
+  // else
+
+  file_status fs_tmp;
+  auto const mode = path_stat.st_mode;
+  if (S_ISLNK(mode))
+    fs_tmp.type(file_type::symlink);
+  else if (S_ISREG(mode))
+    fs_tmp.type(file_type::regular);
+  else if (S_ISDIR(mode))
+    fs_tmp.type(file_type::directory);
+  else if (S_ISBLK(mode))
+    fs_tmp.type(file_type::block);
+  else if (S_ISCHR(mode))
+    fs_tmp.type(file_type::character);
+  else if (S_ISFIFO(mode))
+    fs_tmp.type(file_type::fifo);
+  else if (S_ISSOCK(mode))
+    fs_tmp.type(file_type::socket);
+  else
+    fs_tmp.type(file_type::unknown);
+
+  fs_tmp.permissions(detail::posix_get_perms(path_stat));
+  return fs_tmp;
+}
+
+file_status posix_stat(path const& p, StatT& path_stat, error_code* ec) {
+  error_code m_ec;
+  if (detail::stat(p.c_str(), &path_stat) == -1)
+    m_ec = detail::capture_errno();
+  return create_file_status(m_ec, p, path_stat, ec);
+}
+
+file_status posix_stat(path const& p, error_code* ec) {
+  StatT path_stat;
+  return posix_stat(p, path_stat, ec);
+}
+
+file_status posix_lstat(path const& p, StatT& path_stat, error_code* ec) {
+  error_code m_ec;
+  if (detail::lstat(p.c_str(), &path_stat) == -1)
+    m_ec = detail::capture_errno();
+  return create_file_status(m_ec, p, path_stat, ec);
+}
+
+file_status posix_lstat(path const& p, error_code* ec) {
+  StatT path_stat;
+  return posix_lstat(p, path_stat, ec);
+}
+
+// http://pubs.opengroup.org/onlinepubs/9699919799/functions/ftruncate.html
+bool posix_ftruncate(const FileDescriptor& fd, off_t to_size, error_code& ec) {
+  if (detail::ftruncate(fd.fd, to_size) == -1) {
+    ec = capture_errno();
+    return true;
+  }
+  ec.clear();
+  return false;
+}
+
+bool posix_fchmod(const FileDescriptor& fd, const StatT& st, error_code& ec) {
+  if (detail::fchmod(fd.fd, st.st_mode) == -1) {
+    ec = capture_errno();
+    return true;
+  }
+  ec.clear();
+  return false;
+}
+
+bool stat_equivalent(const StatT& st1, const StatT& st2) {
+  return (st1.st_dev == st2.st_dev && st1.st_ino == st2.st_ino);
+}
+
+file_status FileDescriptor::refresh_status(error_code& ec) {
+  // FD must be open and good.
+  m_status = file_status{};
+  m_stat = {};
+  error_code m_ec;
+  if (detail::fstat(fd, &m_stat) == -1)
+    m_ec = capture_errno();
+  m_status = create_file_status(m_ec, name, m_stat, &ec);
+  return m_status;
+}
+
+} // namespace
+} // end namespace detail
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+_LIBCPP_DIAGNOSTIC_POP
+
+#endif // FILESYSTEM_FILE_DESCRIPTOR_H
diff --git a/libcxx/src/filesystem/filesystem_clock.cpp b/libcxx/src/filesystem/filesystem_clock.cpp
new file mode 100644
index 0000000000000..c6b3c893ff8f7
--- /dev/null
+++ b/libcxx/src/filesystem/filesystem_clock.cpp
@@ -0,0 +1,54 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <__config>
+#include <chrono>
+#include <filesystem>
+#include <time.h>
+
+#include "posix_compat.h" // TimeSpec
+
+#if defined(_LIBCPP_WIN32API)
+# define WIN32_LEAN_AND_MEAN
+# define NOMINMAX
+# include <windows.h>
+#endif
+
+#if !defined(CLOCK_REALTIME) && !defined(_LIBCPP_WIN32API)
+# include <sys/time.h> // for gettimeofday and timeval
+#endif
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+const bool _FilesystemClock::is_steady;
+
+_FilesystemClock::time_point _FilesystemClock::now() noexcept {
+  typedef chrono::duration<rep> __secs;
+#if defined(_LIBCPP_WIN32API)
+  typedef chrono::duration<rep, nano> __nsecs;
+  FILETIME time;
+  GetSystemTimeAsFileTime(&time);
+  detail::TimeSpec tp = detail::filetime_to_timespec(time);
+  return time_point(__secs(tp.tv_sec) +
+                    chrono::duration_cast<duration>(__nsecs(tp.tv_nsec)));
+#elif defined(CLOCK_REALTIME)
+  typedef chrono::duration<rep, nano> __nsecs;
+  struct timespec tp;
+  if (0 != clock_gettime(CLOCK_REALTIME, &tp))
+    __throw_system_error(errno, "clock_gettime(CLOCK_REALTIME) failed");
+  return time_point(__secs(tp.tv_sec) +
+                    chrono::duration_cast<duration>(__nsecs(tp.tv_nsec)));
+#else
+  typedef chrono::duration<rep, micro> __microsecs;
+  timeval tv;
+  gettimeofday(&tv, 0);
+  return time_point(__secs(tv.tv_sec) + __microsecs(tv.tv_usec));
+#endif // CLOCK_REALTIME
+}
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
diff --git a/libcxx/src/filesystem/filesystem_error.cpp b/libcxx/src/filesystem/filesystem_error.cpp
new file mode 100644
index 0000000000000..5faed3b89efa6
--- /dev/null
+++ b/libcxx/src/filesystem/filesystem_error.cpp
@@ -0,0 +1,37 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <__config>
+#include <__utility/unreachable.h>
+#include <filesystem>
+#include <system_error>
+
+#include "format_string.h"
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+filesystem_error::~filesystem_error() {}
+
+void filesystem_error::__create_what(int __num_paths) {
+  const char* derived_what = system_error::what();
+  __storage_->__what_ = [&]() -> string {
+    switch (__num_paths) {
+    case 0:
+      return detail::format_string("filesystem error: %s", derived_what);
+    case 1:
+      return detail::format_string("filesystem error: %s [" PATH_CSTR_FMT "]",
+                                   derived_what, path1().c_str());
+    case 2:
+      return detail::format_string("filesystem error: %s [" PATH_CSTR_FMT "] [" PATH_CSTR_FMT "]",
+                                   derived_what, path1().c_str(), path2().c_str());
+    }
+    __libcpp_unreachable();
+  }();
+}
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
diff --git a/libcxx/src/filesystem/format_string.h b/libcxx/src/filesystem/format_string.h
new file mode 100644
index 0000000000000..d95d9bd3f55ec
--- /dev/null
+++ b/libcxx/src/filesystem/format_string.h
@@ -0,0 +1,89 @@
+//===----------------------------------------------------------------------===////
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===////
+
+#ifndef FILESYSTEM_FORMAT_STRING_H
+#define FILESYSTEM_FORMAT_STRING_H
+
+#include <__assert>
+#include <__config>
+#include <array>
+#include <cstdarg>
+#include <cstddef>
+#include <cstdio>
+#include <string>
+
+#if defined(_LIBCPP_WIN32API)
+#  define PATHSTR(x) (L##x)
+#  define PATH_CSTR_FMT "\"%ls\""
+#else
+#  define PATHSTR(x) (x)
+#  define PATH_CSTR_FMT "\"%s\""
+#endif
+
+// TODO: Check whether these functions actually need internal linkage, or if they can be made normal header functions
+_LIBCPP_DIAGNOSTIC_PUSH
+_LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wunused-function")
+_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wunused-function")
+_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wunused-template")
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+namespace detail {
+namespace {
+
+_LIBCPP_ATTRIBUTE_FORMAT(__printf__, 1, 0)
+string vformat_string(const char* msg, va_list ap) {
+  array<char, 256> buf;
+
+  va_list apcopy;
+  va_copy(apcopy, ap);
+  int ret = ::vsnprintf(buf.data(), buf.size(), msg, apcopy);
+  va_end(apcopy);
+
+  string result;
+  if (static_cast<size_t>(ret) < buf.size()) {
+    result.assign(buf.data(), static_cast<size_t>(ret));
+  } else {
+    // we did not provide a long enough buffer on our first attempt. The
+    // return value is the number of bytes (excluding the null byte) that are
+    // needed for formatting.
+    size_t size_with_null = static_cast<size_t>(ret) + 1;
+    result.__resize_default_init(size_with_null - 1);
+    ret = ::vsnprintf(&result[0], size_with_null, msg, ap);
+    _LIBCPP_ASSERT(static_cast<size_t>(ret) == (size_with_null - 1), "TODO");
+  }
+  return result;
+}
+
+_LIBCPP_ATTRIBUTE_FORMAT(__printf__, 1, 2)
+string format_string(const char* msg, ...) {
+  string ret;
+  va_list ap;
+  va_start(ap, msg);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  try {
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+    ret = detail::vformat_string(msg, ap);
+#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  } catch (...) {
+    va_end(ap);
+    throw;
+  }
+#endif // _LIBCPP_HAS_NO_EXCEPTIONS
+  va_end(ap);
+  return ret;
+}
+
+} // end anonymous namespace
+} // end namespace detail
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+_LIBCPP_DIAGNOSTIC_POP
+
+#endif // FILESYSTEM_FORMAT_STRING_H
diff --git a/libcxx/src/filesystem/operations.cpp b/libcxx/src/filesystem/operations.cpp
index af31fe049b3e4..f6c3ff13080dd 100644
--- a/libcxx/src/filesystem/operations.cpp
+++ b/libcxx/src/filesystem/operations.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <__assert>
+#include <__config>
 #include <__utility/unreachable.h>
 #include <array>
 #include <climits>
@@ -17,9 +18,11 @@
 #include <type_traits>
 #include <vector>
 
-#include "filesystem_common.h"
-
+#include "error.h"
+#include "file_descriptor.h"
+#include "path_parser.h"
 #include "posix_compat.h"
+#include "time_utils.h"
 
 #if defined(_LIBCPP_WIN32API)
 # define WIN32_LEAN_AND_MEAN
@@ -45,595 +48,12 @@
 # define _LIBCPP_FILESYSTEM_USE_FSTREAM
 #endif
 
-#if !defined(CLOCK_REALTIME) && !defined(_LIBCPP_WIN32API)
-# include <sys/time.h> // for gettimeofday and timeval
-#endif
-
 #if defined(__ELF__) && defined(_LIBCPP_LINK_RT_LIB)
 # pragma comment(lib, "rt")
 #endif
 
 _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
 
-namespace {
-
-bool isSeparator(path::value_type C) {
-  if (C == '/')
-    return true;
-#if defined(_LIBCPP_WIN32API)
-  if (C == '\\')
-    return true;
-#endif
-  return false;
-}
-
-bool isDriveLetter(path::value_type C) {
-  return (C >= 'a' && C <= 'z') || (C >= 'A' && C <= 'Z');
-}
-
-namespace parser {
-
-using string_view_t = path::__string_view;
-using string_view_pair = pair<string_view_t, string_view_t>;
-using PosPtr = path::value_type const*;
-
-struct PathParser {
-  enum ParserState : unsigned char {
-    // Zero is a special sentinel value used by default constructed iterators.
-    PS_BeforeBegin = path::iterator::_BeforeBegin,
-    PS_InRootName = path::iterator::_InRootName,
-    PS_InRootDir = path::iterator::_InRootDir,
-    PS_InFilenames = path::iterator::_InFilenames,
-    PS_InTrailingSep = path::iterator::_InTrailingSep,
-    PS_AtEnd = path::iterator::_AtEnd
-  };
-
-  const string_view_t Path;
-  string_view_t RawEntry;
-  ParserState State;
-
-private:
-  PathParser(string_view_t P, ParserState State) noexcept : Path(P),
-                                                            State(State) {}
-
-public:
-  PathParser(string_view_t P, string_view_t E, unsigned char S)
-      : Path(P), RawEntry(E), State(static_cast<ParserState>(S)) {
-    // S cannot be '0' or PS_BeforeBegin.
-  }
-
-  static PathParser CreateBegin(string_view_t P) noexcept {
-    PathParser PP(P, PS_BeforeBegin);
-    PP.increment();
-    return PP;
-  }
-
-  static PathParser CreateEnd(string_view_t P) noexcept {
-    PathParser PP(P, PS_AtEnd);
-    return PP;
-  }
-
-  PosPtr peek() const noexcept {
-    auto TkEnd = getNextTokenStartPos();
-    auto End = getAfterBack();
-    return TkEnd == End ? nullptr : TkEnd;
-  }
-
-  void increment() noexcept {
-    const PosPtr End = getAfterBack();
-    const PosPtr Start = getNextTokenStartPos();
-    if (Start == End)
-      return makeState(PS_AtEnd);
-
-    switch (State) {
-    case PS_BeforeBegin: {
-      PosPtr TkEnd = consumeRootName(Start, End);
-      if (TkEnd)
-        return makeState(PS_InRootName, Start, TkEnd);
-    }
-      _LIBCPP_FALLTHROUGH();
-    case PS_InRootName: {
-      PosPtr TkEnd = consumeAllSeparators(Start, End);
-      if (TkEnd)
-        return makeState(PS_InRootDir, Start, TkEnd);
-      else
-        return makeState(PS_InFilenames, Start, consumeName(Start, End));
-    }
-    case PS_InRootDir:
-      return makeState(PS_InFilenames, Start, consumeName(Start, End));
-
-    case PS_InFilenames: {
-      PosPtr SepEnd = consumeAllSeparators(Start, End);
-      if (SepEnd != End) {
-        PosPtr TkEnd = consumeName(SepEnd, End);
-        if (TkEnd)
-          return makeState(PS_InFilenames, SepEnd, TkEnd);
-      }
-      return makeState(PS_InTrailingSep, Start, SepEnd);
-    }
-
-    case PS_InTrailingSep:
-      return makeState(PS_AtEnd);
-
-    case PS_AtEnd:
-      __libcpp_unreachable();
-    }
-  }
-
-  void decrement() noexcept {
-    const PosPtr REnd = getBeforeFront();
-    const PosPtr RStart = getCurrentTokenStartPos() - 1;
-    if (RStart == REnd) // we're decrementing the begin
-      return makeState(PS_BeforeBegin);
-
-    switch (State) {
-    case PS_AtEnd: {
-      // Try to consume a trailing separator or root directory first.
-      if (PosPtr SepEnd = consumeAllSeparators(RStart, REnd)) {
-        if (SepEnd == REnd)
-          return makeState(PS_InRootDir, Path.data(), RStart + 1);
-        PosPtr TkStart = consumeRootName(SepEnd, REnd);
-        if (TkStart == REnd)
-          return makeState(PS_InRootDir, RStart, RStart + 1);
-        return makeState(PS_InTrailingSep, SepEnd + 1, RStart + 1);
-      } else {
-        PosPtr TkStart = consumeRootName(RStart, REnd);
-        if (TkStart == REnd)
-          return makeState(PS_InRootName, TkStart + 1, RStart + 1);
-        TkStart = consumeName(RStart, REnd);
-        return makeState(PS_InFilenames, TkStart + 1, RStart + 1);
-      }
-    }
-    case PS_InTrailingSep:
-      return makeState(PS_InFilenames, consumeName(RStart, REnd) + 1,
-                       RStart + 1);
-    case PS_InFilenames: {
-      PosPtr SepEnd = consumeAllSeparators(RStart, REnd);
-      if (SepEnd == REnd)
-        return makeState(PS_InRootDir, Path.data(), RStart + 1);
-      PosPtr TkStart = consumeRootName(SepEnd ? SepEnd : RStart, REnd);
-      if (TkStart == REnd) {
-        if (SepEnd)
-          return makeState(PS_InRootDir, SepEnd + 1, RStart + 1);
-        return makeState(PS_InRootName, TkStart + 1, RStart + 1);
-      }
-      TkStart = consumeName(SepEnd, REnd);
-      return makeState(PS_InFilenames, TkStart + 1, SepEnd + 1);
-    }
-    case PS_InRootDir:
-      return makeState(PS_InRootName, Path.data(), RStart + 1);
-    case PS_InRootName:
-    case PS_BeforeBegin:
-      __libcpp_unreachable();
-    }
-  }
-
-  /// \brief Return a view with the "preferred representation" of the current
-  ///   element. For example trailing separators are represented as a '.'
-  string_view_t operator*() const noexcept {
-    switch (State) {
-    case PS_BeforeBegin:
-    case PS_AtEnd:
-      return PATHSTR("");
-    case PS_InRootDir:
-      if (RawEntry[0] == '\\')
-        return PATHSTR("\\");
-      else
-        return PATHSTR("/");
-    case PS_InTrailingSep:
-      return PATHSTR("");
-    case PS_InRootName:
-    case PS_InFilenames:
-      return RawEntry;
-    }
-    __libcpp_unreachable();
-  }
-
-  explicit operator bool() const noexcept {
-    return State != PS_BeforeBegin && State != PS_AtEnd;
-  }
-
-  PathParser& operator++() noexcept {
-    increment();
-    return *this;
-  }
-
-  PathParser& operator--() noexcept {
-    decrement();
-    return *this;
-  }
-
-  bool atEnd() const noexcept {
-    return State == PS_AtEnd;
-  }
-
-  bool inRootDir() const noexcept {
-    return State == PS_InRootDir;
-  }
-
-  bool inRootName() const noexcept {
-    return State == PS_InRootName;
-  }
-
-  bool inRootPath() const noexcept {
-    return inRootName() || inRootDir();
-  }
-
-private:
-  void makeState(ParserState NewState, PosPtr Start, PosPtr End) noexcept {
-    State = NewState;
-    RawEntry = string_view_t(Start, End - Start);
-  }
-  void makeState(ParserState NewState) noexcept {
-    State = NewState;
-    RawEntry = {};
-  }
-
-  PosPtr getAfterBack() const noexcept { return Path.data() + Path.size(); }
-
-  PosPtr getBeforeFront() const noexcept { return Path.data() - 1; }
-
-  /// \brief Return a pointer to the first character after the currently
-  ///   lexed element.
-  PosPtr getNextTokenStartPos() const noexcept {
-    switch (State) {
-    case PS_BeforeBegin:
-      return Path.data();
-    case PS_InRootName:
-    case PS_InRootDir:
-    case PS_InFilenames:
-      return &RawEntry.back() + 1;
-    case PS_InTrailingSep:
-    case PS_AtEnd:
-      return getAfterBack();
-    }
-    __libcpp_unreachable();
-  }
-
-  /// \brief Return a pointer to the first character in the currently lexed
-  ///   element.
-  PosPtr getCurrentTokenStartPos() const noexcept {
-    switch (State) {
-    case PS_BeforeBegin:
-    case PS_InRootName:
-      return &Path.front();
-    case PS_InRootDir:
-    case PS_InFilenames:
-    case PS_InTrailingSep:
-      return &RawEntry.front();
-    case PS_AtEnd:
-      return &Path.back() + 1;
-    }
-    __libcpp_unreachable();
-  }
-
-  // Consume all consecutive separators.
-  PosPtr consumeAllSeparators(PosPtr P, PosPtr End) const noexcept {
-    if (P == nullptr || P == End || !isSeparator(*P))
-      return nullptr;
-    const int Inc = P < End ? 1 : -1;
-    P += Inc;
-    while (P != End && isSeparator(*P))
-      P += Inc;
-    return P;
-  }
-
-  // Consume exactly N separators, or return nullptr.
-  PosPtr consumeNSeparators(PosPtr P, PosPtr End, int N) const noexcept {
-    PosPtr Ret = consumeAllSeparators(P, End);
-    if (Ret == nullptr)
-      return nullptr;
-    if (P < End) {
-      if (Ret == P + N)
-        return Ret;
-    } else {
-      if (Ret == P - N)
-        return Ret;
-    }
-    return nullptr;
-  }
-
-  PosPtr consumeName(PosPtr P, PosPtr End) const noexcept {
-    PosPtr Start = P;
-    if (P == nullptr || P == End || isSeparator(*P))
-      return nullptr;
-    const int Inc = P < End ? 1 : -1;
-    P += Inc;
-    while (P != End && !isSeparator(*P))
-      P += Inc;
-    if (P == End && Inc < 0) {
-      // Iterating backwards and consumed all the rest of the input.
-      // Check if the start of the string would have been considered
-      // a root name.
-      PosPtr RootEnd = consumeRootName(End + 1, Start);
-      if (RootEnd)
-        return RootEnd - 1;
-    }
-    return P;
-  }
-
-  PosPtr consumeDriveLetter(PosPtr P, PosPtr End) const noexcept {
-    if (P == End)
-      return nullptr;
-    if (P < End) {
-      if (P + 1 == End || !isDriveLetter(P[0]) || P[1] != ':')
-        return nullptr;
-      return P + 2;
-    } else {
-      if (P - 1 == End || !isDriveLetter(P[-1]) || P[0] != ':')
-        return nullptr;
-      return P - 2;
-    }
-  }
-
-  PosPtr consumeNetworkRoot(PosPtr P, PosPtr End) const noexcept {
-    if (P == End)
-      return nullptr;
-    if (P < End)
-      return consumeName(consumeNSeparators(P, End, 2), End);
-    else
-      return consumeNSeparators(consumeName(P, End), End, 2);
-  }
-
-  PosPtr consumeRootName(PosPtr P, PosPtr End) const noexcept {
-#if defined(_LIBCPP_WIN32API)
-    if (PosPtr Ret = consumeDriveLetter(P, End))
-      return Ret;
-    if (PosPtr Ret = consumeNetworkRoot(P, End))
-      return Ret;
-#endif
-    return nullptr;
-  }
-};
-
-string_view_pair separate_filename(string_view_t const& s) {
-  if (s == PATHSTR(".") || s == PATHSTR("..") || s.empty())
-    return string_view_pair{s, PATHSTR("")};
-  auto pos = s.find_last_of('.');
-  if (pos == string_view_t::npos || pos == 0)
-    return string_view_pair{s, string_view_t{}};
-  return string_view_pair{s.substr(0, pos), s.substr(pos)};
-}
-
-string_view_t createView(PosPtr S, PosPtr E) noexcept {
-  return {S, static_cast<size_t>(E - S) + 1};
-}
-
-} // namespace parser
-} // namespace
-
-//                       POSIX HELPERS
-
-#if defined(_LIBCPP_WIN32API)
-namespace detail {
-
-errc __win_err_to_errc(int err) {
-  constexpr struct {
-    DWORD win;
-    errc errc;
-  } win_error_mapping[] = {
-      {ERROR_ACCESS_DENIED, errc::permission_denied},
-      {ERROR_ALREADY_EXISTS, errc::file_exists},
-      {ERROR_BAD_NETPATH, errc::no_such_file_or_directory},
-      {ERROR_BAD_PATHNAME, errc::no_such_file_or_directory},
-      {ERROR_BAD_UNIT, errc::no_such_device},
-      {ERROR_BROKEN_PIPE, errc::broken_pipe},
-      {ERROR_BUFFER_OVERFLOW, errc::filename_too_long},
-      {ERROR_BUSY, errc::device_or_resource_busy},
-      {ERROR_BUSY_DRIVE, errc::device_or_resource_busy},
-      {ERROR_CANNOT_MAKE, errc::permission_denied},
-      {ERROR_CANTOPEN, errc::io_error},
-      {ERROR_CANTREAD, errc::io_error},
-      {ERROR_CANTWRITE, errc::io_error},
-      {ERROR_CURRENT_DIRECTORY, errc::permission_denied},
-      {ERROR_DEV_NOT_EXIST, errc::no_such_device},
-      {ERROR_DEVICE_IN_USE, errc::device_or_resource_busy},
-      {ERROR_DIR_NOT_EMPTY, errc::directory_not_empty},
-      {ERROR_DIRECTORY, errc::invalid_argument},
-      {ERROR_DISK_FULL, errc::no_space_on_device},
-      {ERROR_FILE_EXISTS, errc::file_exists},
-      {ERROR_FILE_NOT_FOUND, errc::no_such_file_or_directory},
-      {ERROR_HANDLE_DISK_FULL, errc::no_space_on_device},
-      {ERROR_INVALID_ACCESS, errc::permission_denied},
-      {ERROR_INVALID_DRIVE, errc::no_such_device},
-      {ERROR_INVALID_FUNCTION, errc::function_not_supported},
-      {ERROR_INVALID_HANDLE, errc::invalid_argument},
-      {ERROR_INVALID_NAME, errc::no_such_file_or_directory},
-      {ERROR_INVALID_PARAMETER, errc::invalid_argument},
-      {ERROR_LOCK_VIOLATION, errc::no_lock_available},
-      {ERROR_LOCKED, errc::no_lock_available},
-      {ERROR_NEGATIVE_SEEK, errc::invalid_argument},
-      {ERROR_NOACCESS, errc::permission_denied},
-      {ERROR_NOT_ENOUGH_MEMORY, errc::not_enough_memory},
-      {ERROR_NOT_READY, errc::resource_unavailable_try_again},
-      {ERROR_NOT_SAME_DEVICE, errc::cross_device_link},
-      {ERROR_NOT_SUPPORTED, errc::not_supported},
-      {ERROR_OPEN_FAILED, errc::io_error},
-      {ERROR_OPEN_FILES, errc::device_or_resource_busy},
-      {ERROR_OPERATION_ABORTED, errc::operation_canceled},
-      {ERROR_OUTOFMEMORY, errc::not_enough_memory},
-      {ERROR_PATH_NOT_FOUND, errc::no_such_file_or_directory},
-      {ERROR_READ_FAULT, errc::io_error},
-      {ERROR_REPARSE_TAG_INVALID, errc::invalid_argument},
-      {ERROR_RETRY, errc::resource_unavailable_try_again},
-      {ERROR_SEEK, errc::io_error},
-      {ERROR_SHARING_VIOLATION, errc::permission_denied},
-      {ERROR_TOO_MANY_OPEN_FILES, errc::too_many_files_open},
-      {ERROR_WRITE_FAULT, errc::io_error},
-      {ERROR_WRITE_PROTECT, errc::permission_denied},
-  };
-
-  for (const auto &pair : win_error_mapping)
-    if (pair.win == static_cast<DWORD>(err))
-      return pair.errc;
-  return errc::invalid_argument;
-}
-
-} // namespace detail
-#endif
-
-namespace detail {
-namespace {
-
-using value_type = path::value_type;
-using string_type = path::string_type;
-
-struct FileDescriptor {
-  const path& name;
-  int fd = -1;
-  StatT m_stat;
-  file_status m_status;
-
-  template <class... Args>
-  static FileDescriptor create(const path* p, error_code& ec, Args... args) {
-    ec.clear();
-    int fd;
-    if ((fd = detail::open(p->c_str(), args...)) == -1) {
-      ec = capture_errno();
-      return FileDescriptor{p};
-    }
-    return FileDescriptor(p, fd);
-  }
-
-  template <class... Args>
-  static FileDescriptor create_with_status(const path* p, error_code& ec,
-                                           Args... args) {
-    FileDescriptor fd = create(p, ec, args...);
-    if (!ec)
-      fd.refresh_status(ec);
-
-    return fd;
-  }
-
-  file_status get_status() const { return m_status; }
-  StatT const& get_stat() const { return m_stat; }
-
-  bool status_known() const { return _VSTD_FS::status_known(m_status); }
-
-  file_status refresh_status(error_code& ec);
-
-  void close() noexcept {
-    if (fd != -1)
-      detail::close(fd);
-    fd = -1;
-  }
-
-  FileDescriptor(FileDescriptor&& other)
-      : name(other.name), fd(other.fd), m_stat(other.m_stat),
-        m_status(other.m_status) {
-    other.fd = -1;
-    other.m_status = file_status{};
-  }
-
-  ~FileDescriptor() { close(); }
-
-  FileDescriptor(FileDescriptor const&) = delete;
-  FileDescriptor& operator=(FileDescriptor const&) = delete;
-
-private:
-  explicit FileDescriptor(const path* p, int fd = -1) : name(*p), fd(fd) {}
-};
-
-perms posix_get_perms(const StatT& st) noexcept {
-  return static_cast<perms>(st.st_mode) & perms::mask;
-}
-
-file_status create_file_status(error_code& m_ec, path const& p,
-                               const StatT& path_stat, error_code* ec) {
-  if (ec)
-    *ec = m_ec;
-  if (m_ec && (m_ec.value() == ENOENT || m_ec.value() == ENOTDIR)) {
-    return file_status(file_type::not_found);
-  } else if (m_ec) {
-    ErrorHandler<void> err("posix_stat", ec, &p);
-    err.report(m_ec, "failed to determine attributes for the specified path");
-    return file_status(file_type::none);
-  }
-  // else
-
-  file_status fs_tmp;
-  auto const mode = path_stat.st_mode;
-  if (S_ISLNK(mode))
-    fs_tmp.type(file_type::symlink);
-  else if (S_ISREG(mode))
-    fs_tmp.type(file_type::regular);
-  else if (S_ISDIR(mode))
-    fs_tmp.type(file_type::directory);
-  else if (S_ISBLK(mode))
-    fs_tmp.type(file_type::block);
-  else if (S_ISCHR(mode))
-    fs_tmp.type(file_type::character);
-  else if (S_ISFIFO(mode))
-    fs_tmp.type(file_type::fifo);
-  else if (S_ISSOCK(mode))
-    fs_tmp.type(file_type::socket);
-  else
-    fs_tmp.type(file_type::unknown);
-
-  fs_tmp.permissions(detail::posix_get_perms(path_stat));
-  return fs_tmp;
-}
-
-file_status posix_stat(path const& p, StatT& path_stat, error_code* ec) {
-  error_code m_ec;
-  if (detail::stat(p.c_str(), &path_stat) == -1)
-    m_ec = detail::capture_errno();
-  return create_file_status(m_ec, p, path_stat, ec);
-}
-
-file_status posix_stat(path const& p, error_code* ec) {
-  StatT path_stat;
-  return posix_stat(p, path_stat, ec);
-}
-
-file_status posix_lstat(path const& p, StatT& path_stat, error_code* ec) {
-  error_code m_ec;
-  if (detail::lstat(p.c_str(), &path_stat) == -1)
-    m_ec = detail::capture_errno();
-  return create_file_status(m_ec, p, path_stat, ec);
-}
-
-file_status posix_lstat(path const& p, error_code* ec) {
-  StatT path_stat;
-  return posix_lstat(p, path_stat, ec);
-}
-
-// http://pubs.opengroup.org/onlinepubs/9699919799/functions/ftruncate.html
-bool posix_ftruncate(const FileDescriptor& fd, off_t to_size, error_code& ec) {
-  if (detail::ftruncate(fd.fd, to_size) == -1) {
-    ec = capture_errno();
-    return true;
-  }
-  ec.clear();
-  return false;
-}
-
-bool posix_fchmod(const FileDescriptor& fd, const StatT& st, error_code& ec) {
-  if (detail::fchmod(fd.fd, st.st_mode) == -1) {
-    ec = capture_errno();
-    return true;
-  }
-  ec.clear();
-  return false;
-}
-
-bool stat_equivalent(const StatT& st1, const StatT& st2) {
-  return (st1.st_dev == st2.st_dev && st1.st_ino == st2.st_ino);
-}
-
-file_status FileDescriptor::refresh_status(error_code& ec) {
-  // FD must be open and good.
-  m_status = file_status{};
-  m_stat = {};
-  error_code m_ec;
-  if (detail::fstat(fd, &m_stat) == -1)
-    m_ec = capture_errno();
-  m_status = create_file_status(m_ec, name, m_stat, &ec);
-  return m_status;
-}
-} // namespace
-} // end namespace detail
-
 using detail::capture_errno;
 using detail::ErrorHandler;
 using detail::StatT;
@@ -642,51 +62,6 @@ using parser::createView;
 using parser::PathParser;
 using parser::string_view_t;
 
-const bool _FilesystemClock::is_steady;
-
-_FilesystemClock::time_point _FilesystemClock::now() noexcept {
-  typedef chrono::duration<rep> __secs;
-#if defined(_LIBCPP_WIN32API)
-  typedef chrono::duration<rep, nano> __nsecs;
-  FILETIME time;
-  GetSystemTimeAsFileTime(&time);
-  TimeSpec tp = detail::filetime_to_timespec(time);
-  return time_point(__secs(tp.tv_sec) +
-                    chrono::duration_cast<duration>(__nsecs(tp.tv_nsec)));
-#elif defined(CLOCK_REALTIME)
-  typedef chrono::duration<rep, nano> __nsecs;
-  struct timespec tp;
-  if (0 != clock_gettime(CLOCK_REALTIME, &tp))
-    __throw_system_error(errno, "clock_gettime(CLOCK_REALTIME) failed");
-  return time_point(__secs(tp.tv_sec) +
-                    chrono::duration_cast<duration>(__nsecs(tp.tv_nsec)));
-#else
-  typedef chrono::duration<rep, micro> __microsecs;
-  timeval tv;
-  gettimeofday(&tv, 0);
-  return time_point(__secs(tv.tv_sec) + __microsecs(tv.tv_usec));
-#endif // CLOCK_REALTIME
-}
-
-filesystem_error::~filesystem_error() {}
-
-void filesystem_error::__create_what(int __num_paths) {
-  const char* derived_what = system_error::what();
-  __storage_->__what_ = [&]() -> string {
-    switch (__num_paths) {
-    case 0:
-      return detail::format_string("filesystem error: %s", derived_what);
-    case 1:
-      return detail::format_string("filesystem error: %s [" PATH_CSTR_FMT "]",
-                                   derived_what, path1().c_str());
-    case 2:
-      return detail::format_string("filesystem error: %s [" PATH_CSTR_FMT "] [" PATH_CSTR_FMT "]",
-                                   derived_what, path1().c_str(), path2().c_str());
-    }
-    __libcpp_unreachable();
-  }();
-}
-
 static path __do_absolute(const path& p, path* cwd, error_code* ec) {
   if (ec)
     ec->clear();
@@ -1193,18 +568,6 @@ bool __fs_is_empty(const path& p, error_code* ec) {
   __libcpp_unreachable();
 }
 
-static file_time_type __extract_last_write_time(const path& p, const StatT& st,
-                                                error_code* ec) {
-  using detail::fs_time;
-  ErrorHandler<file_time_type> err("last_write_time", ec, &p);
-
-  auto ts = detail::extract_mtime(st);
-  if (!fs_time::is_representable(ts))
-    return err.report(errc::value_too_large);
-
-  return fs_time::convert_from_timespec(ts);
-}
-
 file_time_type __last_write_time(const path& p, error_code* ec) {
   using namespace chrono;
   ErrorHandler<file_time_type> err("last_write_time", ec, &p);
@@ -1214,7 +577,7 @@ file_time_type __last_write_time(const path& p, error_code* ec) {
   detail::posix_stat(p, st, &m_ec);
   if (m_ec)
     return err.report(m_ec);
-  return __extract_last_write_time(p, st, ec);
+  return detail::__extract_last_write_time(p, st, ec);
 }
 
 void __last_write_time(const path& p, file_time_type new_time, error_code* ec) {
@@ -1609,500 +972,4 @@ path __weakly_canonical(const path& p, error_code* ec) {
   return result.lexically_normal();
 }
 
-///////////////////////////////////////////////////////////////////////////////
-//                            path definitions
-///////////////////////////////////////////////////////////////////////////////
-
-constexpr path::value_type path::preferred_separator;
-
-path& path::replace_extension(path const& replacement) {
-  path p = extension();
-  if (not p.empty()) {
-    __pn_.erase(__pn_.size() - p.native().size());
-  }
-  if (!replacement.empty()) {
-    if (replacement.native()[0] != '.') {
-      __pn_ += PATHSTR(".");
-    }
-    __pn_.append(replacement.__pn_);
-  }
-  return *this;
-}
-
-///////////////////////////////////////////////////////////////////////////////
-// path.decompose
-
-string_view_t path::__root_name() const {
-  auto PP = PathParser::CreateBegin(__pn_);
-  if (PP.State == PathParser::PS_InRootName)
-    return *PP;
-  return {};
-}
-
-string_view_t path::__root_directory() const {
-  auto PP = PathParser::CreateBegin(__pn_);
-  if (PP.State == PathParser::PS_InRootName)
-    ++PP;
-  if (PP.State == PathParser::PS_InRootDir)
-    return *PP;
-  return {};
-}
-
-string_view_t path::__root_path_raw() const {
-  auto PP = PathParser::CreateBegin(__pn_);
-  if (PP.State == PathParser::PS_InRootName) {
-    auto NextCh = PP.peek();
-    if (NextCh && isSeparator(*NextCh)) {
-      ++PP;
-      return createView(__pn_.data(), &PP.RawEntry.back());
-    }
-    return PP.RawEntry;
-  }
-  if (PP.State == PathParser::PS_InRootDir)
-    return *PP;
-  return {};
-}
-
-static bool ConsumeRootName(PathParser *PP) {
-  static_assert(PathParser::PS_BeforeBegin == 1 &&
-      PathParser::PS_InRootName == 2,
-      "Values for enums are incorrect");
-  while (PP->State <= PathParser::PS_InRootName)
-    ++(*PP);
-  return PP->State == PathParser::PS_AtEnd;
-}
-
-static bool ConsumeRootDir(PathParser* PP) {
-  static_assert(PathParser::PS_BeforeBegin == 1 &&
-                PathParser::PS_InRootName == 2 &&
-                PathParser::PS_InRootDir == 3, "Values for enums are incorrect");
-  while (PP->State <= PathParser::PS_InRootDir)
-    ++(*PP);
-  return PP->State == PathParser::PS_AtEnd;
-}
-
-string_view_t path::__relative_path() const {
-  auto PP = PathParser::CreateBegin(__pn_);
-  if (ConsumeRootDir(&PP))
-    return {};
-  return createView(PP.RawEntry.data(), &__pn_.back());
-}
-
-string_view_t path::__parent_path() const {
-  if (empty())
-    return {};
-  // Determine if we have a root path but not a relative path. In that case
-  // return *this.
-  {
-    auto PP = PathParser::CreateBegin(__pn_);
-    if (ConsumeRootDir(&PP))
-      return __pn_;
-  }
-  // Otherwise remove a single element from the end of the path, and return
-  // a string representing that path
-  {
-    auto PP = PathParser::CreateEnd(__pn_);
-    --PP;
-    if (PP.RawEntry.data() == __pn_.data())
-      return {};
-    --PP;
-    return createView(__pn_.data(), &PP.RawEntry.back());
-  }
-}
-
-string_view_t path::__filename() const {
-  if (empty())
-    return {};
-  {
-    PathParser PP = PathParser::CreateBegin(__pn_);
-    if (ConsumeRootDir(&PP))
-      return {};
-  }
-  return *(--PathParser::CreateEnd(__pn_));
-}
-
-string_view_t path::__stem() const {
-  return parser::separate_filename(__filename()).first;
-}
-
-string_view_t path::__extension() const {
-  return parser::separate_filename(__filename()).second;
-}
-
-////////////////////////////////////////////////////////////////////////////
-// path.gen
-
-enum PathPartKind : unsigned char {
-  PK_None,
-  PK_RootSep,
-  PK_Filename,
-  PK_Dot,
-  PK_DotDot,
-  PK_TrailingSep
-};
-
-static PathPartKind ClassifyPathPart(string_view_t Part) {
-  if (Part.empty())
-    return PK_TrailingSep;
-  if (Part == PATHSTR("."))
-    return PK_Dot;
-  if (Part == PATHSTR(".."))
-    return PK_DotDot;
-  if (Part == PATHSTR("/"))
-    return PK_RootSep;
-#if defined(_LIBCPP_WIN32API)
-  if (Part == PATHSTR("\\"))
-    return PK_RootSep;
-#endif
-  return PK_Filename;
-}
-
-path path::lexically_normal() const {
-  if (__pn_.empty())
-    return *this;
-
-  using PartKindPair = pair<string_view_t, PathPartKind>;
-  vector<PartKindPair> Parts;
-  // Guess as to how many elements the path has to avoid reallocating.
-  Parts.reserve(32);
-
-  // Track the total size of the parts as we collect them. This allows the
-  // resulting path to reserve the correct amount of memory.
-  size_t NewPathSize = 0;
-  auto AddPart = [&](PathPartKind K, string_view_t P) {
-    NewPathSize += P.size();
-    Parts.emplace_back(P, K);
-  };
-  auto LastPartKind = [&]() {
-    if (Parts.empty())
-      return PK_None;
-    return Parts.back().second;
-  };
-
-  bool MaybeNeedTrailingSep = false;
-  // Build a stack containing the remaining elements of the path, popping off
-  // elements which occur before a '..' entry.
-  for (auto PP = PathParser::CreateBegin(__pn_); PP; ++PP) {
-    auto Part = *PP;
-    PathPartKind Kind = ClassifyPathPart(Part);
-    switch (Kind) {
-    case PK_Filename:
-    case PK_RootSep: {
-      // Add all non-dot and non-dot-dot elements to the stack of elements.
-      AddPart(Kind, Part);
-      MaybeNeedTrailingSep = false;
-      break;
-    }
-    case PK_DotDot: {
-      // Only push a ".." element if there are no elements preceding the "..",
-      // or if the preceding element is itself "..".
-      auto LastKind = LastPartKind();
-      if (LastKind == PK_Filename) {
-        NewPathSize -= Parts.back().first.size();
-        Parts.pop_back();
-      } else if (LastKind != PK_RootSep)
-        AddPart(PK_DotDot, PATHSTR(".."));
-      MaybeNeedTrailingSep = LastKind == PK_Filename;
-      break;
-    }
-    case PK_Dot:
-    case PK_TrailingSep: {
-      MaybeNeedTrailingSep = true;
-      break;
-    }
-    case PK_None:
-      __libcpp_unreachable();
-    }
-  }
-  // [fs.path.generic]p6.8: If the path is empty, add a dot.
-  if (Parts.empty())
-    return PATHSTR(".");
-
-  // [fs.path.generic]p6.7: If the last filename is dot-dot, remove any
-  // trailing directory-separator.
-  bool NeedTrailingSep = MaybeNeedTrailingSep && LastPartKind() == PK_Filename;
-
-  path Result;
-  Result.__pn_.reserve(Parts.size() + NewPathSize + NeedTrailingSep);
-  for (auto& PK : Parts)
-    Result /= PK.first;
-
-  if (NeedTrailingSep)
-    Result /= PATHSTR("");
-
-  Result.make_preferred();
-  return Result;
-}
-
-static int DetermineLexicalElementCount(PathParser PP) {
-  int Count = 0;
-  for (; PP; ++PP) {
-    auto Elem = *PP;
-    if (Elem == PATHSTR(".."))
-      --Count;
-    else if (Elem != PATHSTR(".") && Elem != PATHSTR(""))
-      ++Count;
-  }
-  return Count;
-}
-
-path path::lexically_relative(const path& base) const {
-  { // perform root-name/root-directory mismatch checks
-    auto PP = PathParser::CreateBegin(__pn_);
-    auto PPBase = PathParser::CreateBegin(base.__pn_);
-    auto CheckIterMismatchAtBase = [&]() {
-      return PP.State != PPBase.State &&
-             (PP.inRootPath() || PPBase.inRootPath());
-    };
-    if (PP.inRootName() && PPBase.inRootName()) {
-      if (*PP != *PPBase)
-        return {};
-    } else if (CheckIterMismatchAtBase())
-      return {};
-
-    if (PP.inRootPath())
-      ++PP;
-    if (PPBase.inRootPath())
-      ++PPBase;
-    if (CheckIterMismatchAtBase())
-      return {};
-  }
-
-  // Find the first mismatching element
-  auto PP = PathParser::CreateBegin(__pn_);
-  auto PPBase = PathParser::CreateBegin(base.__pn_);
-  while (PP && PPBase && PP.State == PPBase.State && *PP == *PPBase) {
-    ++PP;
-    ++PPBase;
-  }
-
-  // If there is no mismatch, return ".".
-  if (!PP && !PPBase)
-    return ".";
-
-  // Otherwise, determine the number of elements, 'n', which are not dot or
-  // dot-dot minus the number of dot-dot elements.
-  int ElemCount = DetermineLexicalElementCount(PPBase);
-  if (ElemCount < 0)
-    return {};
-
-  // if n == 0 and (a == end() || a->empty()), returns path("."); otherwise
-  if (ElemCount == 0 && (PP.atEnd() || *PP == PATHSTR("")))
-    return PATHSTR(".");
-
-  // return a path constructed with 'n' dot-dot elements, followed by the
-  // elements of '*this' after the mismatch.
-  path Result;
-  // FIXME: Reserve enough room in Result that it won't have to re-allocate.
-  while (ElemCount--)
-    Result /= PATHSTR("..");
-  for (; PP; ++PP)
-    Result /= *PP;
-  return Result;
-}
-
-////////////////////////////////////////////////////////////////////////////
-// path.comparisons
-static int CompareRootName(PathParser *LHS, PathParser *RHS) {
-  if (!LHS->inRootName() && !RHS->inRootName())
-    return 0;
-
-  auto GetRootName = [](PathParser *Parser) -> string_view_t {
-    return Parser->inRootName() ? **Parser : PATHSTR("");
-  };
-  int res = GetRootName(LHS).compare(GetRootName(RHS));
-  ConsumeRootName(LHS);
-  ConsumeRootName(RHS);
-  return res;
-}
-
-static int CompareRootDir(PathParser *LHS, PathParser *RHS) {
-  if (!LHS->inRootDir() && RHS->inRootDir())
-    return -1;
-  else if (LHS->inRootDir() && !RHS->inRootDir())
-    return 1;
-  else {
-    ConsumeRootDir(LHS);
-    ConsumeRootDir(RHS);
-    return 0;
-  }
-}
-
-static int CompareRelative(PathParser *LHSPtr, PathParser *RHSPtr) {
-  auto &LHS = *LHSPtr;
-  auto &RHS = *RHSPtr;
-
-  int res;
-  while (LHS && RHS) {
-    if ((res = (*LHS).compare(*RHS)) != 0)
-      return res;
-    ++LHS;
-    ++RHS;
-  }
-  return 0;
-}
-
-static int CompareEndState(PathParser *LHS, PathParser *RHS) {
-  if (LHS->atEnd() && !RHS->atEnd())
-    return -1;
-  else if (!LHS->atEnd() && RHS->atEnd())
-    return 1;
-  return 0;
-}
-
-int path::__compare(string_view_t __s) const {
-  auto LHS = PathParser::CreateBegin(__pn_);
-  auto RHS = PathParser::CreateBegin(__s);
-  int res;
-
-  if ((res = CompareRootName(&LHS, &RHS)) != 0)
-    return res;
-
-  if ((res = CompareRootDir(&LHS, &RHS)) != 0)
-    return res;
-
-  if ((res = CompareRelative(&LHS, &RHS)) != 0)
-    return res;
-
-  return CompareEndState(&LHS, &RHS);
-}
-
-////////////////////////////////////////////////////////////////////////////
-// path.nonmembers
-size_t hash_value(const path& __p) noexcept {
-  auto PP = PathParser::CreateBegin(__p.native());
-  size_t hash_value = 0;
-  hash<string_view_t> hasher;
-  while (PP) {
-    hash_value = __hash_combine(hash_value, hasher(*PP));
-    ++PP;
-  }
-  return hash_value;
-}
-
-////////////////////////////////////////////////////////////////////////////
-// path.itr
-path::iterator path::begin() const {
-  auto PP = PathParser::CreateBegin(__pn_);
-  iterator it;
-  it.__path_ptr_ = this;
-  it.__state_ = static_cast<path::iterator::_ParserState>(PP.State);
-  it.__entry_ = PP.RawEntry;
-  it.__stashed_elem_.__assign_view(*PP);
-  return it;
-}
-
-path::iterator path::end() const {
-  iterator it{};
-  it.__state_ = path::iterator::_AtEnd;
-  it.__path_ptr_ = this;
-  return it;
-}
-
-path::iterator& path::iterator::__increment() {
-  PathParser PP(__path_ptr_->native(), __entry_, __state_);
-  ++PP;
-  __state_ = static_cast<_ParserState>(PP.State);
-  __entry_ = PP.RawEntry;
-  __stashed_elem_.__assign_view(*PP);
-  return *this;
-}
-
-path::iterator& path::iterator::__decrement() {
-  PathParser PP(__path_ptr_->native(), __entry_, __state_);
-  --PP;
-  __state_ = static_cast<_ParserState>(PP.State);
-  __entry_ = PP.RawEntry;
-  __stashed_elem_.__assign_view(*PP);
-  return *this;
-}
-
-#if defined(_LIBCPP_WIN32API)
-////////////////////////////////////////////////////////////////////////////
-// Windows path conversions
-size_t __wide_to_char(const wstring &str, char *out, size_t outlen) {
-  if (str.empty())
-    return 0;
-  ErrorHandler<size_t> err("__wide_to_char", nullptr);
-  UINT codepage = AreFileApisANSI() ? CP_ACP : CP_OEMCP;
-  BOOL used_default = FALSE;
-  int ret = WideCharToMultiByte(codepage, 0, str.data(), str.size(), out,
-                                outlen, nullptr, &used_default);
-  if (ret <= 0 || used_default)
-    return err.report(errc::illegal_byte_sequence);
-  return ret;
-}
-
-size_t __char_to_wide(const string &str, wchar_t *out, size_t outlen) {
-  if (str.empty())
-    return 0;
-  ErrorHandler<size_t> err("__char_to_wide", nullptr);
-  UINT codepage = AreFileApisANSI() ? CP_ACP : CP_OEMCP;
-  int ret = MultiByteToWideChar(codepage, MB_ERR_INVALID_CHARS, str.data(),
-                                str.size(), out, outlen);
-  if (ret <= 0)
-    return err.report(errc::illegal_byte_sequence);
-  return ret;
-}
-#endif
-
-
-///////////////////////////////////////////////////////////////////////////////
-//                           directory entry definitions
-///////////////////////////////////////////////////////////////////////////////
-
-error_code directory_entry::__do_refresh() noexcept {
-  __data_.__reset();
-  error_code failure_ec;
-
-  StatT full_st;
-  file_status st = detail::posix_lstat(__p_, full_st, &failure_ec);
-  if (!status_known(st)) {
-    __data_.__reset();
-    return failure_ec;
-  }
-
-  if (!_VSTD_FS::exists(st) || !_VSTD_FS::is_symlink(st)) {
-    __data_.__cache_type_ = directory_entry::_RefreshNonSymlink;
-    __data_.__type_ = st.type();
-    __data_.__non_sym_perms_ = st.permissions();
-  } else { // we have a symlink
-    __data_.__sym_perms_ = st.permissions();
-    // Get the information about the linked entity.
-    // Ignore errors from stat, since we don't want errors regarding symlink
-    // resolution to be reported to the user.
-    error_code ignored_ec;
-    st = detail::posix_stat(__p_, full_st, &ignored_ec);
-
-    __data_.__type_ = st.type();
-    __data_.__non_sym_perms_ = st.permissions();
-
-    // If we failed to resolve the link, then only partially populate the
-    // cache.
-    if (!status_known(st)) {
-      __data_.__cache_type_ = directory_entry::_RefreshSymlinkUnresolved;
-      return error_code{};
-    }
-    // Otherwise, we resolved the link, potentially as not existing.
-    // That's OK.
-    __data_.__cache_type_ = directory_entry::_RefreshSymlink;
-  }
-
-  if (_VSTD_FS::is_regular_file(st))
-    __data_.__size_ = static_cast<uintmax_t>(full_st.st_size);
-
-  if (_VSTD_FS::exists(st)) {
-    __data_.__nlink_ = static_cast<uintmax_t>(full_st.st_nlink);
-
-    // Attempt to extract the mtime, and fail if it's not representable using
-    // file_time_type. For now we ignore the error, as we'll report it when
-    // the value is actually used.
-    error_code ignored_ec;
-    __data_.__write_time_ =
-        __extract_last_write_time(__p_, full_st, &ignored_ec);
-  }
-
-  return failure_ec;
-}
-
 _LIBCPP_END_NAMESPACE_FILESYSTEM
diff --git a/libcxx/src/filesystem/path.cpp b/libcxx/src/filesystem/path.cpp
new file mode 100644
index 0000000000000..82f1ba7ba69da
--- /dev/null
+++ b/libcxx/src/filesystem/path.cpp
@@ -0,0 +1,460 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <__config>
+#include <filesystem>
+#include <vector>
+
+#include "error.h"
+#include "path_parser.h"
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+using detail::ErrorHandler;
+using parser::createView;
+using parser::PathParser;
+using parser::string_view_t;
+
+///////////////////////////////////////////////////////////////////////////////
+//                            path definitions
+///////////////////////////////////////////////////////////////////////////////
+
+constexpr path::value_type path::preferred_separator;
+
+path& path::replace_extension(path const& replacement) {
+  path p = extension();
+  if (not p.empty()) {
+    __pn_.erase(__pn_.size() - p.native().size());
+  }
+  if (!replacement.empty()) {
+    if (replacement.native()[0] != '.') {
+      __pn_ += PATHSTR(".");
+    }
+    __pn_.append(replacement.__pn_);
+  }
+  return *this;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// path.decompose
+
+string_view_t path::__root_name() const {
+  auto PP = PathParser::CreateBegin(__pn_);
+  if (PP.State == PathParser::PS_InRootName)
+    return *PP;
+  return {};
+}
+
+string_view_t path::__root_directory() const {
+  auto PP = PathParser::CreateBegin(__pn_);
+  if (PP.State == PathParser::PS_InRootName)
+    ++PP;
+  if (PP.State == PathParser::PS_InRootDir)
+    return *PP;
+  return {};
+}
+
+string_view_t path::__root_path_raw() const {
+  auto PP = PathParser::CreateBegin(__pn_);
+  if (PP.State == PathParser::PS_InRootName) {
+    auto NextCh = PP.peek();
+    if (NextCh && isSeparator(*NextCh)) {
+      ++PP;
+      return createView(__pn_.data(), &PP.RawEntry.back());
+    }
+    return PP.RawEntry;
+  }
+  if (PP.State == PathParser::PS_InRootDir)
+    return *PP;
+  return {};
+}
+
+static bool ConsumeRootName(PathParser *PP) {
+  static_assert(PathParser::PS_BeforeBegin == 1 &&
+      PathParser::PS_InRootName == 2,
+      "Values for enums are incorrect");
+  while (PP->State <= PathParser::PS_InRootName)
+    ++(*PP);
+  return PP->State == PathParser::PS_AtEnd;
+}
+
+static bool ConsumeRootDir(PathParser* PP) {
+  static_assert(PathParser::PS_BeforeBegin == 1 &&
+                PathParser::PS_InRootName == 2 &&
+                PathParser::PS_InRootDir == 3, "Values for enums are incorrect");
+  while (PP->State <= PathParser::PS_InRootDir)
+    ++(*PP);
+  return PP->State == PathParser::PS_AtEnd;
+}
+
+string_view_t path::__relative_path() const {
+  auto PP = PathParser::CreateBegin(__pn_);
+  if (ConsumeRootDir(&PP))
+    return {};
+  return createView(PP.RawEntry.data(), &__pn_.back());
+}
+
+string_view_t path::__parent_path() const {
+  if (empty())
+    return {};
+  // Determine if we have a root path but not a relative path. In that case
+  // return *this.
+  {
+    auto PP = PathParser::CreateBegin(__pn_);
+    if (ConsumeRootDir(&PP))
+      return __pn_;
+  }
+  // Otherwise remove a single element from the end of the path, and return
+  // a string representing that path
+  {
+    auto PP = PathParser::CreateEnd(__pn_);
+    --PP;
+    if (PP.RawEntry.data() == __pn_.data())
+      return {};
+    --PP;
+    return createView(__pn_.data(), &PP.RawEntry.back());
+  }
+}
+
+string_view_t path::__filename() const {
+  if (empty())
+    return {};
+  {
+    PathParser PP = PathParser::CreateBegin(__pn_);
+    if (ConsumeRootDir(&PP))
+      return {};
+  }
+  return *(--PathParser::CreateEnd(__pn_));
+}
+
+string_view_t path::__stem() const {
+  return parser::separate_filename(__filename()).first;
+}
+
+string_view_t path::__extension() const {
+  return parser::separate_filename(__filename()).second;
+}
+
+////////////////////////////////////////////////////////////////////////////
+// path.gen
+
+enum PathPartKind : unsigned char {
+  PK_None,
+  PK_RootSep,
+  PK_Filename,
+  PK_Dot,
+  PK_DotDot,
+  PK_TrailingSep
+};
+
+static PathPartKind ClassifyPathPart(string_view_t Part) {
+  if (Part.empty())
+    return PK_TrailingSep;
+  if (Part == PATHSTR("."))
+    return PK_Dot;
+  if (Part == PATHSTR(".."))
+    return PK_DotDot;
+  if (Part == PATHSTR("/"))
+    return PK_RootSep;
+#if defined(_LIBCPP_WIN32API)
+  if (Part == PATHSTR("\\"))
+    return PK_RootSep;
+#endif
+  return PK_Filename;
+}
+
+path path::lexically_normal() const {
+  if (__pn_.empty())
+    return *this;
+
+  using PartKindPair = pair<string_view_t, PathPartKind>;
+  vector<PartKindPair> Parts;
+  // Guess as to how many elements the path has to avoid reallocating.
+  Parts.reserve(32);
+
+  // Track the total size of the parts as we collect them. This allows the
+  // resulting path to reserve the correct amount of memory.
+  size_t NewPathSize = 0;
+  auto AddPart = [&](PathPartKind K, string_view_t P) {
+    NewPathSize += P.size();
+    Parts.emplace_back(P, K);
+  };
+  auto LastPartKind = [&]() {
+    if (Parts.empty())
+      return PK_None;
+    return Parts.back().second;
+  };
+
+  bool MaybeNeedTrailingSep = false;
+  // Build a stack containing the remaining elements of the path, popping off
+  // elements which occur before a '..' entry.
+  for (auto PP = PathParser::CreateBegin(__pn_); PP; ++PP) {
+    auto Part = *PP;
+    PathPartKind Kind = ClassifyPathPart(Part);
+    switch (Kind) {
+    case PK_Filename:
+    case PK_RootSep: {
+      // Add all non-dot and non-dot-dot elements to the stack of elements.
+      AddPart(Kind, Part);
+      MaybeNeedTrailingSep = false;
+      break;
+    }
+    case PK_DotDot: {
+      // Only push a ".." element if there are no elements preceding the "..",
+      // or if the preceding element is itself "..".
+      auto LastKind = LastPartKind();
+      if (LastKind == PK_Filename) {
+        NewPathSize -= Parts.back().first.size();
+        Parts.pop_back();
+      } else if (LastKind != PK_RootSep)
+        AddPart(PK_DotDot, PATHSTR(".."));
+      MaybeNeedTrailingSep = LastKind == PK_Filename;
+      break;
+    }
+    case PK_Dot:
+    case PK_TrailingSep: {
+      MaybeNeedTrailingSep = true;
+      break;
+    }
+    case PK_None:
+      __libcpp_unreachable();
+    }
+  }
+  // [fs.path.generic]p6.8: If the path is empty, add a dot.
+  if (Parts.empty())
+    return PATHSTR(".");
+
+  // [fs.path.generic]p6.7: If the last filename is dot-dot, remove any
+  // trailing directory-separator.
+  bool NeedTrailingSep = MaybeNeedTrailingSep && LastPartKind() == PK_Filename;
+
+  path Result;
+  Result.__pn_.reserve(Parts.size() + NewPathSize + NeedTrailingSep);
+  for (auto& PK : Parts)
+    Result /= PK.first;
+
+  if (NeedTrailingSep)
+    Result /= PATHSTR("");
+
+  Result.make_preferred();
+  return Result;
+}
+
+static int DetermineLexicalElementCount(PathParser PP) {
+  int Count = 0;
+  for (; PP; ++PP) {
+    auto Elem = *PP;
+    if (Elem == PATHSTR(".."))
+      --Count;
+    else if (Elem != PATHSTR(".") && Elem != PATHSTR(""))
+      ++Count;
+  }
+  return Count;
+}
+
+path path::lexically_relative(const path& base) const {
+  { // perform root-name/root-directory mismatch checks
+    auto PP = PathParser::CreateBegin(__pn_);
+    auto PPBase = PathParser::CreateBegin(base.__pn_);
+    auto CheckIterMismatchAtBase = [&]() {
+      return PP.State != PPBase.State &&
+             (PP.inRootPath() || PPBase.inRootPath());
+    };
+    if (PP.inRootName() && PPBase.inRootName()) {
+      if (*PP != *PPBase)
+        return {};
+    } else if (CheckIterMismatchAtBase())
+      return {};
+
+    if (PP.inRootPath())
+      ++PP;
+    if (PPBase.inRootPath())
+      ++PPBase;
+    if (CheckIterMismatchAtBase())
+      return {};
+  }
+
+  // Find the first mismatching element
+  auto PP = PathParser::CreateBegin(__pn_);
+  auto PPBase = PathParser::CreateBegin(base.__pn_);
+  while (PP && PPBase && PP.State == PPBase.State && *PP == *PPBase) {
+    ++PP;
+    ++PPBase;
+  }
+
+  // If there is no mismatch, return ".".
+  if (!PP && !PPBase)
+    return ".";
+
+  // Otherwise, determine the number of elements, 'n', which are not dot or
+  // dot-dot minus the number of dot-dot elements.
+  int ElemCount = DetermineLexicalElementCount(PPBase);
+  if (ElemCount < 0)
+    return {};
+
+  // if n == 0 and (a == end() || a->empty()), returns path("."); otherwise
+  if (ElemCount == 0 && (PP.atEnd() || *PP == PATHSTR("")))
+    return PATHSTR(".");
+
+  // return a path constructed with 'n' dot-dot elements, followed by the
+  // elements of '*this' after the mismatch.
+  path Result;
+  // FIXME: Reserve enough room in Result that it won't have to re-allocate.
+  while (ElemCount--)
+    Result /= PATHSTR("..");
+  for (; PP; ++PP)
+    Result /= *PP;
+  return Result;
+}
+
+////////////////////////////////////////////////////////////////////////////
+// path.comparisons
+static int CompareRootName(PathParser *LHS, PathParser *RHS) {
+  if (!LHS->inRootName() && !RHS->inRootName())
+    return 0;
+
+  auto GetRootName = [](PathParser *Parser) -> string_view_t {
+    return Parser->inRootName() ? **Parser : PATHSTR("");
+  };
+  int res = GetRootName(LHS).compare(GetRootName(RHS));
+  ConsumeRootName(LHS);
+  ConsumeRootName(RHS);
+  return res;
+}
+
+static int CompareRootDir(PathParser *LHS, PathParser *RHS) {
+  if (!LHS->inRootDir() && RHS->inRootDir())
+    return -1;
+  else if (LHS->inRootDir() && !RHS->inRootDir())
+    return 1;
+  else {
+    ConsumeRootDir(LHS);
+    ConsumeRootDir(RHS);
+    return 0;
+  }
+}
+
+static int CompareRelative(PathParser *LHSPtr, PathParser *RHSPtr) {
+  auto &LHS = *LHSPtr;
+  auto &RHS = *RHSPtr;
+
+  int res;
+  while (LHS && RHS) {
+    if ((res = (*LHS).compare(*RHS)) != 0)
+      return res;
+    ++LHS;
+    ++RHS;
+  }
+  return 0;
+}
+
+static int CompareEndState(PathParser *LHS, PathParser *RHS) {
+  if (LHS->atEnd() && !RHS->atEnd())
+    return -1;
+  else if (!LHS->atEnd() && RHS->atEnd())
+    return 1;
+  return 0;
+}
+
+int path::__compare(string_view_t __s) const {
+  auto LHS = PathParser::CreateBegin(__pn_);
+  auto RHS = PathParser::CreateBegin(__s);
+  int res;
+
+  if ((res = CompareRootName(&LHS, &RHS)) != 0)
+    return res;
+
+  if ((res = CompareRootDir(&LHS, &RHS)) != 0)
+    return res;
+
+  if ((res = CompareRelative(&LHS, &RHS)) != 0)
+    return res;
+
+  return CompareEndState(&LHS, &RHS);
+}
+
+////////////////////////////////////////////////////////////////////////////
+// path.nonmembers
+size_t hash_value(const path& __p) noexcept {
+  auto PP = PathParser::CreateBegin(__p.native());
+  size_t hash_value = 0;
+  hash<string_view_t> hasher;
+  while (PP) {
+    hash_value = __hash_combine(hash_value, hasher(*PP));
+    ++PP;
+  }
+  return hash_value;
+}
+
+////////////////////////////////////////////////////////////////////////////
+// path.itr
+path::iterator path::begin() const {
+  auto PP = PathParser::CreateBegin(__pn_);
+  iterator it;
+  it.__path_ptr_ = this;
+  it.__state_ = static_cast<path::iterator::_ParserState>(PP.State);
+  it.__entry_ = PP.RawEntry;
+  it.__stashed_elem_.__assign_view(*PP);
+  return it;
+}
+
+path::iterator path::end() const {
+  iterator it{};
+  it.__state_ = path::iterator::_AtEnd;
+  it.__path_ptr_ = this;
+  return it;
+}
+
+path::iterator& path::iterator::__increment() {
+  PathParser PP(__path_ptr_->native(), __entry_, __state_);
+  ++PP;
+  __state_ = static_cast<_ParserState>(PP.State);
+  __entry_ = PP.RawEntry;
+  __stashed_elem_.__assign_view(*PP);
+  return *this;
+}
+
+path::iterator& path::iterator::__decrement() {
+  PathParser PP(__path_ptr_->native(), __entry_, __state_);
+  --PP;
+  __state_ = static_cast<_ParserState>(PP.State);
+  __entry_ = PP.RawEntry;
+  __stashed_elem_.__assign_view(*PP);
+  return *this;
+}
+
+#if defined(_LIBCPP_WIN32API)
+////////////////////////////////////////////////////////////////////////////
+// Windows path conversions
+size_t __wide_to_char(const wstring &str, char *out, size_t outlen) {
+  if (str.empty())
+    return 0;
+  ErrorHandler<size_t> err("__wide_to_char", nullptr);
+  UINT codepage = AreFileApisANSI() ? CP_ACP : CP_OEMCP;
+  BOOL used_default = FALSE;
+  int ret = WideCharToMultiByte(codepage, 0, str.data(), str.size(), out,
+                                outlen, nullptr, &used_default);
+  if (ret <= 0 || used_default)
+    return err.report(errc::illegal_byte_sequence);
+  return ret;
+}
+
+size_t __char_to_wide(const string &str, wchar_t *out, size_t outlen) {
+  if (str.empty())
+    return 0;
+  ErrorHandler<size_t> err("__char_to_wide", nullptr);
+  UINT codepage = AreFileApisANSI() ? CP_ACP : CP_OEMCP;
+  int ret = MultiByteToWideChar(codepage, MB_ERR_INVALID_CHARS, str.data(),
+                                str.size(), out, outlen);
+  if (ret <= 0)
+    return err.report(errc::illegal_byte_sequence);
+  return ret;
+}
+#endif
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
diff --git a/libcxx/src/filesystem/path_parser.h b/libcxx/src/filesystem/path_parser.h
new file mode 100644
index 0000000000000..4cc75a50b9640
--- /dev/null
+++ b/libcxx/src/filesystem/path_parser.h
@@ -0,0 +1,379 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef PATH_PARSER_H
+#define PATH_PARSER_H
+
+#include <__config>
+#include <__utility/unreachable.h>
+#include <cstddef>
+#include <filesystem>
+#include <utility>
+
+#include "format_string.h"
+
+// TODO: Check whether these functions actually need internal linkage, or if they can be made normal header functions
+_LIBCPP_DIAGNOSTIC_PUSH
+_LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wunused-function")
+_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wunused-function")
+_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wunused-template")
+
+_LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
+
+namespace {
+
+bool isSeparator(path::value_type C) {
+  if (C == '/')
+    return true;
+#if defined(_LIBCPP_WIN32API)
+  if (C == '\\')
+    return true;
+#endif
+  return false;
+}
+
+bool isDriveLetter(path::value_type C) {
+  return (C >= 'a' && C <= 'z') || (C >= 'A' && C <= 'Z');
+}
+
+namespace parser {
+
+using string_view_t = path::__string_view;
+using string_view_pair = pair<string_view_t, string_view_t>;
+using PosPtr = path::value_type const*;
+
+struct PathParser {
+  enum ParserState : unsigned char {
+    // Zero is a special sentinel value used by default constructed iterators.
+    PS_BeforeBegin = path::iterator::_BeforeBegin,
+    PS_InRootName = path::iterator::_InRootName,
+    PS_InRootDir = path::iterator::_InRootDir,
+    PS_InFilenames = path::iterator::_InFilenames,
+    PS_InTrailingSep = path::iterator::_InTrailingSep,
+    PS_AtEnd = path::iterator::_AtEnd
+  };
+
+  const string_view_t Path;
+  string_view_t RawEntry;
+  ParserState State;
+
+private:
+  PathParser(string_view_t P, ParserState State) noexcept : Path(P),
+                                                            State(State) {}
+
+public:
+  PathParser(string_view_t P, string_view_t E, unsigned char S)
+      : Path(P), RawEntry(E), State(static_cast<ParserState>(S)) {
+    // S cannot be '0' or PS_BeforeBegin.
+  }
+
+  static PathParser CreateBegin(string_view_t P) noexcept {
+    PathParser PP(P, PS_BeforeBegin);
+    PP.increment();
+    return PP;
+  }
+
+  static PathParser CreateEnd(string_view_t P) noexcept {
+    PathParser PP(P, PS_AtEnd);
+    return PP;
+  }
+
+  PosPtr peek() const noexcept {
+    auto TkEnd = getNextTokenStartPos();
+    auto End = getAfterBack();
+    return TkEnd == End ? nullptr : TkEnd;
+  }
+
+  void increment() noexcept {
+    const PosPtr End = getAfterBack();
+    const PosPtr Start = getNextTokenStartPos();
+    if (Start == End)
+      return makeState(PS_AtEnd);
+
+    switch (State) {
+    case PS_BeforeBegin: {
+      PosPtr TkEnd = consumeRootName(Start, End);
+      if (TkEnd)
+        return makeState(PS_InRootName, Start, TkEnd);
+    }
+      _LIBCPP_FALLTHROUGH();
+    case PS_InRootName: {
+      PosPtr TkEnd = consumeAllSeparators(Start, End);
+      if (TkEnd)
+        return makeState(PS_InRootDir, Start, TkEnd);
+      else
+        return makeState(PS_InFilenames, Start, consumeName(Start, End));
+    }
+    case PS_InRootDir:
+      return makeState(PS_InFilenames, Start, consumeName(Start, End));
+
+    case PS_InFilenames: {
+      PosPtr SepEnd = consumeAllSeparators(Start, End);
+      if (SepEnd != End) {
+        PosPtr TkEnd = consumeName(SepEnd, End);
+        if (TkEnd)
+          return makeState(PS_InFilenames, SepEnd, TkEnd);
+      }
+      return makeState(PS_InTrailingSep, Start, SepEnd);
+    }
+
+    case PS_InTrailingSep:
+      return makeState(PS_AtEnd);
+
+    case PS_AtEnd:
+      __libcpp_unreachable();
+    }
+  }
+
+  void decrement() noexcept {
+    const PosPtr REnd = getBeforeFront();
+    const PosPtr RStart = getCurrentTokenStartPos() - 1;
+    if (RStart == REnd) // we're decrementing the begin
+      return makeState(PS_BeforeBegin);
+
+    switch (State) {
+    case PS_AtEnd: {
+      // Try to consume a trailing separator or root directory first.
+      if (PosPtr SepEnd = consumeAllSeparators(RStart, REnd)) {
+        if (SepEnd == REnd)
+          return makeState(PS_InRootDir, Path.data(), RStart + 1);
+        PosPtr TkStart = consumeRootName(SepEnd, REnd);
+        if (TkStart == REnd)
+          return makeState(PS_InRootDir, RStart, RStart + 1);
+        return makeState(PS_InTrailingSep, SepEnd + 1, RStart + 1);
+      } else {
+        PosPtr TkStart = consumeRootName(RStart, REnd);
+        if (TkStart == REnd)
+          return makeState(PS_InRootName, TkStart + 1, RStart + 1);
+        TkStart = consumeName(RStart, REnd);
+        return makeState(PS_InFilenames, TkStart + 1, RStart + 1);
+      }
+    }
+    case PS_InTrailingSep:
+      return makeState(PS_InFilenames, consumeName(RStart, REnd) + 1,
+                       RStart + 1);
+    case PS_InFilenames: {
+      PosPtr SepEnd = consumeAllSeparators(RStart, REnd);
+      if (SepEnd == REnd)
+        return makeState(PS_InRootDir, Path.data(), RStart + 1);
+      PosPtr TkStart = consumeRootName(SepEnd ? SepEnd : RStart, REnd);
+      if (TkStart == REnd) {
+        if (SepEnd)
+          return makeState(PS_InRootDir, SepEnd + 1, RStart + 1);
+        return makeState(PS_InRootName, TkStart + 1, RStart + 1);
+      }
+      TkStart = consumeName(SepEnd, REnd);
+      return makeState(PS_InFilenames, TkStart + 1, SepEnd + 1);
+    }
+    case PS_InRootDir:
+      return makeState(PS_InRootName, Path.data(), RStart + 1);
+    case PS_InRootName:
+    case PS_BeforeBegin:
+      __libcpp_unreachable();
+    }
+  }
+
+  /// \brief Return a view with the "preferred representation" of the current
+  ///   element. For example trailing separators are represented as a '.'
+  string_view_t operator*() const noexcept {
+    switch (State) {
+    case PS_BeforeBegin:
+    case PS_AtEnd:
+      return PATHSTR("");
+    case PS_InRootDir:
+      if (RawEntry[0] == '\\')
+        return PATHSTR("\\");
+      else
+        return PATHSTR("/");
+    case PS_InTrailingSep:
+      return PATHSTR("");
+    case PS_InRootName:
+    case PS_InFilenames:
+      return RawEntry;
+    }
+    __libcpp_unreachable();
+  }
+
+  explicit operator bool() const noexcept {
+    return State != PS_BeforeBegin && State != PS_AtEnd;
+  }
+
+  PathParser& operator++() noexcept {
+    increment();
+    return *this;
+  }
+
+  PathParser& operator--() noexcept {
+    decrement();
+    return *this;
+  }
+
+  bool atEnd() const noexcept {
+    return State == PS_AtEnd;
+  }
+
+  bool inRootDir() const noexcept {
+    return State == PS_InRootDir;
+  }
+
+  bool inRootName() const noexcept {
+    return State == PS_InRootName;
+  }
+
+  bool inRootPath() const noexcept {
+    return inRootName() || inRootDir();
+  }
+
+private:
+  void makeState(ParserState NewState, PosPtr Start, PosPtr End) noexcept {
+    State = NewState;
+    RawEntry = string_view_t(Start, End - Start);
+  }
+  void makeState(ParserState NewState) noexcept {
+    State = NewState;
+    RawEntry = {};
+  }
+
+  PosPtr getAfterBack() const noexcept { return Path.data() + Path.size(); }
+
+  PosPtr getBeforeFront() const noexcept { return Path.data() - 1; }
+
+  /// \brief Return a pointer to the first character after the currently
+  ///   lexed element.
+  PosPtr getNextTokenStartPos() const noexcept {
+    switch (State) {
+    case PS_BeforeBegin:
+      return Path.data();
+    case PS_InRootName:
+    case PS_InRootDir:
+    case PS_InFilenames:
+      return &RawEntry.back() + 1;
+    case PS_InTrailingSep:
+    case PS_AtEnd:
+      return getAfterBack();
+    }
+    __libcpp_unreachable();
+  }
+
+  /// \brief Return a pointer to the first character in the currently lexed
+  ///   element.
+  PosPtr getCurrentTokenStartPos() const noexcept {
+    switch (State) {
+    case PS_BeforeBegin:
+    case PS_InRootName:
+      return &Path.front();
+    case PS_InRootDir:
+    case PS_InFilenames:
+    case PS_InTrailingSep:
+      return &RawEntry.front();
+    case PS_AtEnd:
+      return &Path.back() + 1;
+    }
+    __libcpp_unreachable();
+  }
+
+  // Consume all consecutive separators.
+  PosPtr consumeAllSeparators(PosPtr P, PosPtr End) const noexcept {
+    if (P == nullptr || P == End || !isSeparator(*P))
+      return nullptr;
+    const int Inc = P < End ? 1 : -1;
+    P += Inc;
+    while (P != End && isSeparator(*P))
+      P += Inc;
+    return P;
+  }
+
+  // Consume exactly N separators, or return nullptr.
+  PosPtr consumeNSeparators(PosPtr P, PosPtr End, int N) const noexcept {
+    PosPtr Ret = consumeAllSeparators(P, End);
+    if (Ret == nullptr)
+      return nullptr;
+    if (P < End) {
+      if (Ret == P + N)
+        return Ret;
+    } else {
+      if (Ret == P - N)
+        return Ret;
+    }
+    return nullptr;
+  }
+
+  PosPtr consumeName(PosPtr P, PosPtr End) const noexcept {
+    PosPtr Start = P;
+    if (P == nullptr || P == End || isSeparator(*P))
+      return nullptr;
+    const int Inc = P < End ? 1 : -1;
+    P += Inc;
+    while (P != End && !isSeparator(*P))
+      P += Inc;
+    if (P == End && Inc < 0) {
+      // Iterating backwards and consumed all the rest of the input.
+      // Check if the start of the string would have been considered
+      // a root name.
+      PosPtr RootEnd = consumeRootName(End + 1, Start);
+      if (RootEnd)
+        return RootEnd - 1;
+    }
+    return P;
+  }
+
+  PosPtr consumeDriveLetter(PosPtr P, PosPtr End) const noexcept {
+    if (P == End)
+      return nullptr;
+    if (P < End) {
+      if (P + 1 == End || !isDriveLetter(P[0]) || P[1] != ':')
+        return nullptr;
+      return P + 2;
+    } else {
+      if (P - 1 == End || !isDriveLetter(P[-1]) || P[0] != ':')
+        return nullptr;
+      return P - 2;
+    }
+  }
+
+  PosPtr consumeNetworkRoot(PosPtr P, PosPtr End) const noexcept {
+    if (P == End)
+      return nullptr;
+    if (P < End)
+      return consumeName(consumeNSeparators(P, End, 2), End);
+    else
+      return consumeNSeparators(consumeName(P, End), End, 2);
+  }
+
+  PosPtr consumeRootName(PosPtr P, PosPtr End) const noexcept {
+#if defined(_LIBCPP_WIN32API)
+    if (PosPtr Ret = consumeDriveLetter(P, End))
+      return Ret;
+    if (PosPtr Ret = consumeNetworkRoot(P, End))
+      return Ret;
+#endif
+    return nullptr;
+  }
+};
+
+string_view_pair separate_filename(string_view_t const& s) {
+  if (s == PATHSTR(".") || s == PATHSTR("..") || s.empty())
+    return string_view_pair{s, PATHSTR("")};
+  auto pos = s.find_last_of('.');
+  if (pos == string_view_t::npos || pos == 0)
+    return string_view_pair{s, string_view_t{}};
+  return string_view_pair{s.substr(0, pos), s.substr(pos)};
+}
+
+string_view_t createView(PosPtr S, PosPtr E) noexcept {
+  return {S, static_cast<size_t>(E - S) + 1};
+}
+
+} // namespace parser
+} // namespace
+
+_LIBCPP_END_NAMESPACE_FILESYSTEM
+
+_LIBCPP_DIAGNOSTIC_POP
+
+#endif // PATH_PARSER_H
diff --git a/libcxx/src/filesystem/posix_compat.h b/libcxx/src/filesystem/posix_compat.h
index 36116ec5a3952..bddead75960d8 100644
--- a/libcxx/src/filesystem/posix_compat.h
+++ b/libcxx/src/filesystem/posix_compat.h
@@ -24,9 +24,10 @@
 #define POSIX_COMPAT_H
 
 #include <__assert>
+#include <__config>
 #include <filesystem>
 
-#include "filesystem_common.h"
+#include "error.h"
 
 #if defined(_LIBCPP_WIN32API)
 # define WIN32_LEAN_AND_MEAN
@@ -35,10 +36,12 @@
 # include <io.h>
 # include <winioctl.h>
 #else
+# include <fcntl.h>
 # include <unistd.h>
 # include <sys/stat.h>
 # include <sys/statvfs.h>
 #endif
+#include <stdlib.h>
 #include <time.h>
 
 #if defined(_LIBCPP_WIN32API)
@@ -71,11 +74,52 @@ struct LIBCPP_REPARSE_DATA_BUFFER {
 };
 #endif
 
+// TODO: Check whether these functions actually need internal linkage, or if they can be made normal header functions
+_LIBCPP_DIAGNOSTIC_PUSH
+_LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wunused-function")
+_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wunused-function")
+_LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wunused-template")
+
 _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
 
 namespace detail {
 namespace {
 
+#if defined(_LIBCPP_WIN32API)
+// Various C runtime versions (UCRT, or the legacy msvcrt.dll used by
+// some mingw toolchains) provide different stat function implementations,
+// with a number of limitations with respect to what we want from the
+// stat function. Instead provide our own which does exactly what we want,
+// along with our own stat structure and flag macros.
+
+struct TimeSpec {
+  int64_t tv_sec;
+  int64_t tv_nsec;
+};
+struct StatT {
+  unsigned st_mode;
+  TimeSpec st_atim;
+  TimeSpec st_mtim;
+  uint64_t st_dev; // FILE_ID_INFO::VolumeSerialNumber
+  struct FileIdStruct {
+    unsigned char id[16]; // FILE_ID_INFO::FileId
+    bool operator==(const FileIdStruct &other) const {
+      for (int i = 0; i < 16; i++)
+        if (id[i] != other.id[i])
+          return false;
+      return true;
+    }
+  } st_ino;
+  uint32_t st_nlink;
+  uintmax_t st_size;
+};
+
+#else
+using TimeSpec = struct timespec;
+using TimeVal = struct timeval;
+using StatT = struct stat;
+#endif
+
 #if defined(_LIBCPP_WIN32API)
 
 // Various C runtime header sets provide more or less of these. As we
@@ -400,7 +444,7 @@ int fchmod_handle(HANDLE h, int perms) {
   return 0;
 }
 
-int fchmodat(int fd, const wchar_t *path, int perms, int flag) {
+int fchmodat(int /*fd*/, const wchar_t *path, int perms, int flag) {
   DWORD attributes = GetFileAttributesW(path);
   if (attributes == INVALID_FILE_ATTRIBUTES)
     return set_errno();
@@ -519,4 +563,6 @@ using SSizeT = ::ssize_t;
 
 _LIBCPP_END_NAMESPACE_FILESYSTEM
 
+_LIBCPP_DIAGNOSTIC_POP
+
 #endif // POSIX_COMPAT_H
diff --git a/libcxx/src/filesystem/filesystem_common.h b/libcxx/src/filesystem/time_utils.h
similarity index 55%
rename from libcxx/src/filesystem/filesystem_common.h
rename to libcxx/src/filesystem/time_utils.h
index a1756cf5ef83e..2e299b85a46f1 100644
--- a/libcxx/src/filesystem/filesystem_common.h
+++ b/libcxx/src/filesystem/time_utils.h
@@ -6,34 +6,30 @@
 //
 //===----------------------------------------------------------------------===////
 
-#ifndef FILESYSTEM_COMMON_H
-#define FILESYSTEM_COMMON_H
+#ifndef FILESYSTEM_TIME_UTILS_H
+#define FILESYSTEM_TIME_UTILS_H
 
-#include <__assert>
 #include <__config>
 #include <array>
-#include <cerrno>
 #include <chrono>
-#include <climits>
-#include <cstdarg>
-#include <ctime>
 #include <filesystem>
+#include <limits>
 #include <ratio>
-#include <utility>
 #include <system_error>
+#include <type_traits>
+#include <utility>
+
+#include "error.h"
+#include "format_string.h"
+#include "posix_compat.h"
 
 #if defined(_LIBCPP_WIN32API)
 # define WIN32_LEAN_AND_MEAN
 # define NOMINMAX
 # include <windows.h>
 #else
-# include <dirent.h>   // for DIR & friends
-# include <fcntl.h>    /* values for fchmodat */
-# include <sys/stat.h>
-# include <sys/statvfs.h>
 # include <sys/time.h> // for ::utimes as used in __last_write_time
-# include <unistd.h>
-#endif // defined(_LIBCPP_WIN32API)
+#endif
 
 // We can use the presence of UTIME_OMIT to detect platforms that provide utimensat.
 #if defined(UTIME_OMIT)
@@ -46,240 +42,14 @@ _LIBCPP_GCC_DIAGNOSTIC_IGNORED("-Wunused-function")
 _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wunused-function")
 _LIBCPP_CLANG_DIAGNOSTIC_IGNORED("-Wunused-template")
 
-#if defined(_LIBCPP_WIN32API)
-#  define PATHSTR(x) (L##x)
-#  define PATH_CSTR_FMT "\"%ls\""
-#else
-#  define PATHSTR(x) (x)
-#  define PATH_CSTR_FMT "\"%s\""
-#endif
-
 _LIBCPP_BEGIN_NAMESPACE_FILESYSTEM
 
 namespace detail {
-
-#if defined(_LIBCPP_WIN32API)
-// Non anonymous, to allow access from two translation units.
-errc __win_err_to_errc(int err);
-#endif
-
 namespace {
 
-static _LIBCPP_ATTRIBUTE_FORMAT(__printf__, 1, 0) string
-format_string_impl(const char* msg, va_list ap) {
-  array<char, 256> buf;
-
-  va_list apcopy;
-  va_copy(apcopy, ap);
-  int ret = ::vsnprintf(buf.data(), buf.size(), msg, apcopy);
-  va_end(apcopy);
-
-  string result;
-  if (static_cast<size_t>(ret) < buf.size()) {
-    result.assign(buf.data(), static_cast<size_t>(ret));
-  } else {
-    // we did not provide a long enough buffer on our first attempt. The
-    // return value is the number of bytes (excluding the null byte) that are
-    // needed for formatting.
-    size_t size_with_null = static_cast<size_t>(ret) + 1;
-    result.__resize_default_init(size_with_null - 1);
-    ret = ::vsnprintf(&result[0], size_with_null, msg, ap);
-    _LIBCPP_ASSERT(static_cast<size_t>(ret) == (size_with_null - 1), "TODO");
-  }
-  return result;
-}
-
-static _LIBCPP_ATTRIBUTE_FORMAT(__printf__, 1, 2) string
-format_string(const char* msg, ...) {
-  string ret;
-  va_list ap;
-  va_start(ap, msg);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-  try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
-    ret = format_string_impl(msg, ap);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-  } catch (...) {
-    va_end(ap);
-    throw;
-  }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
-  va_end(ap);
-  return ret;
-}
-
-error_code capture_errno() {
-  _LIBCPP_ASSERT(errno != 0, "Expected errno to be non-zero");
-  return error_code(errno, generic_category());
-}
-
-#if defined(_LIBCPP_WIN32API)
-error_code make_windows_error(int err) {
-  return make_error_code(__win_err_to_errc(err));
-}
-#endif
-
-template <class T>
-T error_value();
-template <>
-_LIBCPP_CONSTEXPR_SINCE_CXX14 void error_value<void>() {}
-template <>
-bool error_value<bool>() {
-  return false;
-}
-#if __SIZEOF_SIZE_T__ != __SIZEOF_LONG_LONG__
-template <>
-size_t error_value<size_t>() {
-  return size_t(-1);
-}
-#endif
-template <>
-uintmax_t error_value<uintmax_t>() {
-  return uintmax_t(-1);
-}
-template <>
-_LIBCPP_CONSTEXPR_SINCE_CXX14 file_time_type error_value<file_time_type>() {
-  return file_time_type::min();
-}
-template <>
-path error_value<path>() {
-  return {};
-}
-
-template <class T>
-struct ErrorHandler {
-  const char* func_name_;
-  error_code* ec_ = nullptr;
-  const path* p1_ = nullptr;
-  const path* p2_ = nullptr;
-
-  ErrorHandler(const char* fname, error_code* ec, const path* p1 = nullptr,
-               const path* p2 = nullptr)
-      : func_name_(fname), ec_(ec), p1_(p1), p2_(p2) {
-    if (ec_)
-      ec_->clear();
-  }
-
-  T report(const error_code& ec) const {
-    if (ec_) {
-      *ec_ = ec;
-      return error_value<T>();
-    }
-    string what = string("in ") + func_name_;
-    switch (bool(p1_) + bool(p2_)) {
-    case 0:
-      __throw_filesystem_error(what, ec);
-    case 1:
-      __throw_filesystem_error(what, *p1_, ec);
-    case 2:
-      __throw_filesystem_error(what, *p1_, *p2_, ec);
-    }
-    __libcpp_unreachable();
-  }
-
-  _LIBCPP_ATTRIBUTE_FORMAT(__printf__, 3, 0)
-  void report_impl(const error_code& ec, const char* msg, va_list ap) const {
-    if (ec_) {
-      *ec_ = ec;
-      return;
-    }
-    string what =
-        string("in ") + func_name_ + ": " + format_string_impl(msg, ap);
-    switch (bool(p1_) + bool(p2_)) {
-    case 0:
-      __throw_filesystem_error(what, ec);
-    case 1:
-      __throw_filesystem_error(what, *p1_, ec);
-    case 2:
-      __throw_filesystem_error(what, *p1_, *p2_, ec);
-    }
-    __libcpp_unreachable();
-  }
-
-  _LIBCPP_ATTRIBUTE_FORMAT(__printf__, 3, 4)
-  T report(const error_code& ec, const char* msg, ...) const {
-    va_list ap;
-    va_start(ap, msg);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-    try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
-      report_impl(ec, msg, ap);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-    } catch (...) {
-      va_end(ap);
-      throw;
-    }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
-    va_end(ap);
-    return error_value<T>();
-  }
-
-  T report(errc const& err) const {
-    return report(make_error_code(err));
-  }
-
-  _LIBCPP_ATTRIBUTE_FORMAT(__printf__, 3, 4)
-  T report(errc const& err, const char* msg, ...) const {
-    va_list ap;
-    va_start(ap, msg);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-    try {
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
-      report_impl(make_error_code(err), msg, ap);
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-    } catch (...) {
-      va_end(ap);
-      throw;
-    }
-#endif // _LIBCPP_HAS_NO_EXCEPTIONS
-    va_end(ap);
-    return error_value<T>();
-  }
-
-private:
-  ErrorHandler(ErrorHandler const&) = delete;
-  ErrorHandler& operator=(ErrorHandler const&) = delete;
-};
-
 using chrono::duration;
 using chrono::duration_cast;
 
-#if defined(_LIBCPP_WIN32API)
-// Various C runtime versions (UCRT, or the legacy msvcrt.dll used by
-// some mingw toolchains) provide different stat function implementations,
-// with a number of limitations with respect to what we want from the
-// stat function. Instead provide our own (in the anonymous detail namespace
-// in posix_compat.h) which does exactly what we want, along with our own
-// stat structure and flag macros.
-
-struct TimeSpec {
-  int64_t tv_sec;
-  int64_t tv_nsec;
-};
-struct StatT {
-  unsigned st_mode;
-  TimeSpec st_atim;
-  TimeSpec st_mtim;
-  uint64_t st_dev; // FILE_ID_INFO::VolumeSerialNumber
-  struct FileIdStruct {
-    unsigned char id[16]; // FILE_ID_INFO::FileId
-    bool operator==(const FileIdStruct &other) const {
-      for (int i = 0; i < 16; i++)
-        if (id[i] != other.id[i])
-          return false;
-      return true;
-    }
-  } st_ino;
-  uint32_t st_nlink;
-  uintmax_t st_size;
-};
-
-#else
-using TimeSpec = struct timespec;
-using TimeVal = struct timeval;
-using StatT = struct stat;
-#endif
-
 template <class FileTimeT, class TimeT,
           bool IsFloat = is_floating_point<typename FileTimeT::rep>::value>
 struct time_util_base {
@@ -307,7 +77,7 @@ struct time_util_base {
           .count();
 
 private:
-  static _LIBCPP_CONSTEXPR_SINCE_CXX14 fs_duration get_min_nsecs() {
+  static _LIBCPP_CONSTEXPR fs_duration get_min_nsecs() {
     return duration_cast<fs_duration>(
         fs_nanoseconds(min_nsec_timespec) -
         duration_cast<fs_nanoseconds>(fs_seconds(1)));
@@ -327,7 +97,9 @@ struct time_util_base {
     return max_seconds >= numeric_limits<TimeT>::max() &&
            min_seconds <= numeric_limits<TimeT>::min();
   }
+#if _LIBCPP_STD_VER >= 14
   static_assert(check_range(), "the representable range is unacceptable small");
+#endif
 };
 
 template <class FileTimeT, class TimeT>
@@ -531,75 +303,19 @@ bool set_file_times(const path& p, std::array<TimeSpec, 2> const& TS,
 #endif
 }
 
-#if defined(DT_BLK)
-template <class DirEntT, class = decltype(DirEntT::d_type)>
-static file_type get_file_type(DirEntT* ent, int) {
-  switch (ent->d_type) {
-  case DT_BLK:
-    return file_type::block;
-  case DT_CHR:
-    return file_type::character;
-  case DT_DIR:
-    return file_type::directory;
-  case DT_FIFO:
-    return file_type::fifo;
-  case DT_LNK:
-    return file_type::symlink;
-  case DT_REG:
-    return file_type::regular;
-  case DT_SOCK:
-    return file_type::socket;
-  // Unlike in lstat, hitting "unknown" here simply means that the underlying
-  // filesystem doesn't support d_type. Report is as 'none' so we correctly
-  // set the cache to empty.
-  case DT_UNKNOWN:
-    break;
-  }
-  return file_type::none;
-}
-#endif // defined(DT_BLK)
-
-template <class DirEntT>
-static file_type get_file_type(DirEntT*, long) {
-  return file_type::none;
-}
+#endif // !_LIBCPP_WIN32API
 
-static pair<string_view, file_type> posix_readdir(DIR* dir_stream,
-                                                  error_code& ec) {
-  struct dirent* dir_entry_ptr = nullptr;
-  errno = 0; // zero errno in order to detect errors
-  ec.clear();
-  if ((dir_entry_ptr = ::readdir(dir_stream)) == nullptr) {
-    if (errno)
-      ec = capture_errno();
-    return {};
-  } else {
-    return {dir_entry_ptr->d_name, get_file_type(dir_entry_ptr, 0)};
-  }
-}
+file_time_type __extract_last_write_time(const path& p, const StatT& st,
+                                         error_code* ec) {
+  using detail::fs_time;
+  ErrorHandler<file_time_type> err("last_write_time", ec, &p);
 
-#else // _LIBCPP_WIN32API
+  auto ts = detail::extract_mtime(st);
+  if (!fs_time::is_representable(ts))
+    return err.report(errc::value_too_large);
 
-static file_type get_file_type(const WIN32_FIND_DATAW& data) {
-  if (data.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT &&
-      data.dwReserved0 == IO_REPARSE_TAG_SYMLINK)
-    return file_type::symlink;
-  if (data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
-    return file_type::directory;
-  return file_type::regular;
+  return fs_time::convert_from_timespec(ts);
 }
-static uintmax_t get_file_size(const WIN32_FIND_DATAW& data) {
-  return (static_cast<uint64_t>(data.nFileSizeHigh) << 32) + data.nFileSizeLow;
-}
-static file_time_type get_write_time(const WIN32_FIND_DATAW& data) {
-  ULARGE_INTEGER tmp;
-  const FILETIME& time = data.ftLastWriteTime;
-  tmp.u.LowPart = time.dwLowDateTime;
-  tmp.u.HighPart = time.dwHighDateTime;
-  return file_time_type(file_time_type::duration(tmp.QuadPart));
-}
-
-#endif // !_LIBCPP_WIN32API
 
 } // namespace
 } // end namespace detail
@@ -608,4 +324,4 @@ _LIBCPP_END_NAMESPACE_FILESYSTEM
 
 _LIBCPP_DIAGNOSTIC_POP
 
-#endif // FILESYSTEM_COMMON_H
+#endif // FILESYSTEM_TIME_UTILS_H
diff --git a/libcxx/test/libcxx/input.output/filesystems/class.directory_entry/directory_entry.mods/last_write_time.pass.cpp b/libcxx/test/libcxx/input.output/filesystems/class.directory_entry/directory_entry.mods/last_write_time.pass.cpp
index 4c6ac405282ef..e581963d7a33b 100644
--- a/libcxx/test/libcxx/input.output/filesystems/class.directory_entry/directory_entry.mods/last_write_time.pass.cpp
+++ b/libcxx/test/libcxx/input.output/filesystems/class.directory_entry/directory_entry.mods/last_write_time.pass.cpp
@@ -7,10 +7,10 @@
 //===----------------------------------------------------------------------===//
 
 // UNSUPPORTED: c++03
-// ADDITIONAL_COMPILE_FLAGS: -I %S/../../../../../../src/filesystem
+// ADDITIONAL_COMPILE_FLAGS: -I %S/../../../../../../src
 
-// This test relies on calling functions from the libcxx internal header
-// filesystem_common.h; the Windows implementation uses different
+// This test relies on calling functions from the libcxx internal headers
+// of <filesystem>; the Windows implementation uses different
 // internals and doesn't provide the same set_file_times function as for
 // other platforms.
 // UNSUPPORTED: windows
@@ -30,7 +30,7 @@
 #include "test_macros.h"
 #include "filesystem_test_helper.h"
 
-#include "filesystem_common.h"
+#include "filesystem/time_utils.h"
 
 using namespace fs::detail;
 
diff --git a/libcxx/test/libcxx/input.output/filesystems/convert_file_time.pass.cpp b/libcxx/test/libcxx/input.output/filesystems/convert_file_time.pass.cpp
index b86282ccbedb5..2d057cbf265a6 100644
--- a/libcxx/test/libcxx/input.output/filesystems/convert_file_time.pass.cpp
+++ b/libcxx/test/libcxx/input.output/filesystems/convert_file_time.pass.cpp
@@ -12,7 +12,7 @@
 
 // typedef TrivialClock file_time_type;
 
-// ADDITIONAL_COMPILE_FLAGS: -I %S/../../../../src/filesystem -Wno-macro-redefined
+// ADDITIONAL_COMPILE_FLAGS: -I %S/../../../../src -Wno-macro-redefined
 
 #include <cassert>
 #include <chrono>
@@ -22,7 +22,7 @@
 #include <string>
 #include <type_traits>
 
-#include "filesystem_common.h"
+#include "filesystem/time_utils.h"
 
 #ifndef __SIZEOF_INT128__
 #define TEST_HAS_NO_INT128_T
diff --git a/libcxx/utils/data/ignore_format.txt b/libcxx/utils/data/ignore_format.txt
index 2281cc22c0aba..dfe30a2c3f4c1 100644
--- a/libcxx/utils/data/ignore_format.txt
+++ b/libcxx/utils/data/ignore_format.txt
@@ -790,11 +790,18 @@ libcxx/src/condition_variable_destructor.cpp
 libcxx/src/debug.cpp
 libcxx/src/exception.cpp
 libcxx/src/experimental/memory_resource.cpp
+libcxx/src/filesystem/directory_entry.cpp
 libcxx/src/filesystem/directory_iterator.cpp
-libcxx/src/filesystem/filesystem_common.h
+libcxx/src/filesystem/error.h
+libcxx/src/filesystem/file_descriptor.h
+libcxx/src/filesystem/filesystem_clock.cpp
+libcxx/src/filesystem/filesystem_error.cpp
 libcxx/src/filesystem/int128_builtins.cpp
 libcxx/src/filesystem/operations.cpp
+libcxx/src/filesystem/path.cpp
+libcxx/src/filesystem/path_parser.h
 libcxx/src/filesystem/posix_compat.h
+libcxx/src/filesystem/time_utils.h
 libcxx/src/functional.cpp
 libcxx/src/future.cpp
 libcxx/src/hash.cpp

From c5a412dad5b89996cdd1661075c63dbf884f8e5c Mon Sep 17 00:00:00 2001
From: melonedo <funanzeng@gmail.com>
Date: Wed, 14 Jun 2023 21:42:57 +0800
Subject: [PATCH 020/130] [RISCV] Add support for XCVbitmanip extension in
 CV32E40P

Implement XCVbitmanip intrinsics for CV32E40P according to the specification.

This commit is part of a patch-set to upstream the 7 vendor specific extensions of CV32E40P.

Contributors: @CharKeaney, @jeremybennett, @lewis-revill, @liaolucy, @simoncook, @xmj.

Spec: https://github.com/openhwgroup/cv32e40p/blob/62bec66b36182215e18c9cf10f723567e23878e9/docs/source/instruction_set_extensions.rst

Reviewed By: craig.topper

Differential Revision: https://reviews.llvm.org/D152915
---
 llvm/docs/RISCVUsage.rst                      |   3 +
 .../RISCV/Disassembler/RISCVDisassembler.cpp  |   3 +
 llvm/lib/Target/RISCV/RISCVFeatures.td        |   8 +
 llvm/lib/Target/RISCV/RISCVInstrInfo.td       |   1 +
 llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td    |  68 +++++
 .../test/MC/RISCV/corev/XCVbitmanip-invalid.s | 267 ++++++++++++++++++
 llvm/test/MC/RISCV/corev/XCVbitmanip.s        | 248 ++++++++++++++++
 7 files changed, 598 insertions(+)
 create mode 100644 llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
 create mode 100644 llvm/test/MC/RISCV/corev/XCVbitmanip-invalid.s
 create mode 100644 llvm/test/MC/RISCV/corev/XCVbitmanip.s

diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst
index 7ea7dba043605..017e5c2aba782 100644
--- a/llvm/docs/RISCVUsage.rst
+++ b/llvm/docs/RISCVUsage.rst
@@ -279,3 +279,6 @@ The current vendor extensions supported are:
 
 ``XSfvcp``
   LLVM implements `version 1.0.0 of the SiFive Vector Coprocessor Interface (VCIX) Software Specification <https://sifive.cdn.prismic.io/sifive/c3829e36-8552-41f0-a841-79945784241b_vcix-spec-software.pdf>`_ by SiFive.  All instructions are prefixed with `sf.vc.` as described in the specification, and the riscv-toolchain-convention document linked above.
+
+``XCVbitmanip``
+  LLVM implements `version 1.3.1 of the Core-V bit manipulation custom instructions specification <https://github.com/openhwgroup/cv32e40p/blob/62bec66b36182215e18c9cf10f723567e23878e9/docs/source/instruction_set_extensions.rst>`_ by Core-V.  All instructions are prefixed with `cv.` as described in the specification.
diff --git a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
index e10fb7d4b039c..202bfb4792550 100644
--- a/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
+++ b/llvm/lib/Target/RISCV/Disassembler/RISCVDisassembler.cpp
@@ -558,6 +558,9 @@ DecodeStatus RISCVDisassembler::getInstruction(MCInst &MI, uint64_t &Size,
                           "XTHeadVdot custom opcode table");
     TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXSfvcp, DecoderTableXSfvcp32,
                           "SiFive VCIX custom opcode table");
+    TRY_TO_DECODE_FEATURE(RISCV::FeatureVendorXCVbitmanip,
+                          DecoderTableXCVbitmanip32,
+                          "CORE-V Bit Manipulation custom opcode table");
     TRY_TO_DECODE(true, DecoderTable32, "RISCV32 table");
 
     return MCDisassembler::Fail;
diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td
index 63ddc87f60742..8243b34fb23c8 100644
--- a/llvm/lib/Target/RISCV/RISCVFeatures.td
+++ b/llvm/lib/Target/RISCV/RISCVFeatures.td
@@ -752,6 +752,14 @@ def HasVendorXSfvcp : Predicate<"Subtarget->hasVendorXSfvcp()">,
                                 AssemblerPredicate<(all_of FeatureVendorXSfvcp),
                                 "'XSfvcp' (SiFive Custom Vector Coprocessor Interface Instructions)">;
 
+
+def FeatureVendorXCVbitmanip
+    : SubtargetFeature<"xcvbitmanip", "HasVendorXCVbitmanip", "true",
+                       "'XCVbitmanip' (Bit Manipulation)">;
+def HasVendorXCVbitmanip : Predicate<"Subtarget->hasVendorXCVbitmanip()">,
+                                AssemblerPredicate<(all_of FeatureVendorXCVbitmanip),
+                                "'XCVbitmanip' (Bit Manipulation)">;
+
 //===----------------------------------------------------------------------===//
 // LLVM specific features and extensions
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.td b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
index ed3b1d9cdb8fb..eb675d9581571 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.td
@@ -1923,3 +1923,4 @@ include "RISCVInstrInfoZicond.td"
 include "RISCVInstrInfoXVentana.td"
 include "RISCVInstrInfoXTHead.td"
 include "RISCVInstrInfoXSf.td"
+include "RISCVInstrInfoXCV.td"
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td b/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
new file mode 100644
index 0000000000000..3a69993ee96b0
--- /dev/null
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoXCV.td
@@ -0,0 +1,68 @@
+//===-- RISCVInstrInfoXCV.td - CORE-V instructions ---------*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file describes the vendor extensions defined by Core-V extensions.
+//
+//===----------------------------------------------------------------------===//
+
+let DecoderNamespace = "XCVbitmanip" in {
+  class RVInstBitManipRII<bits<2> funct2, bits<3> funct3, dag outs, dag ins,
+                      string opcodestr, string argstr>
+      : RVInstI<funct3, OPC_CUSTOM_2, outs, ins, opcodestr, argstr> {
+    bits<5> is3;
+    bits<5> is2;
+    let imm12 = {funct2, is3, is2};
+  }
+
+  class CVBitManipRII<bits<2> funct2, bits<3> funct3, string opcodestr, 
+                      Operand i3type = uimm5>
+      : RVInstBitManipRII<funct2, funct3, (outs GPR:$rd), 
+                          (ins GPR:$rs1, i3type:$is3, uimm5:$is2),
+                          opcodestr, "$rd, $rs1, $is3, $is2">;
+
+  class CVBitManipRR<bits<7> funct7, string opcodestr>
+      : RVInstR<funct7, 0b011, OPC_CUSTOM_1, (outs GPR:$rd), 
+                (ins GPR:$rs1, GPR:$rs2), opcodestr, "$rd, $rs1, $rs2">;
+
+  class CVBitManipR<bits<7> funct7, string opcodestr>
+      : RVInstR<funct7, 0b011, OPC_CUSTOM_1, (outs GPR:$rd), 
+                (ins GPR:$rs1, GPR:$rs2), opcodestr, "$rd, $rs1"> {
+    let rs2 = 0b00000;
+  }
+}
+
+let Predicates = [HasVendorXCVbitmanip, IsRV32], 
+    hasSideEffects = 0, mayLoad = 0, mayStore = 0 in {
+  def CV_EXTRACT : CVBitManipRII<0b00, 0b000, "cv.extract">;
+  def CV_EXTRACTU : CVBitManipRII<0b01, 0b000, "cv.extractu">;
+
+  def CV_BCLR : CVBitManipRII<0b00, 0b001, "cv.bclr">;
+  def CV_BSET : CVBitManipRII<0b01, 0b001, "cv.bset">;
+  def CV_BITREV : CVBitManipRII<0b11, 0b001, "cv.bitrev", uimm2>;
+
+  def CV_EXTRACTR : CVBitManipRR<0b0011000, "cv.extractr">;
+  def CV_EXTRACTUR : CVBitManipRR<0b0011001, "cv.extractur">;
+
+  let Constraints = "$rd = $rd_wb" in {
+    def CV_INSERT : RVInstBitManipRII<0b10, 0b000, (outs GPR:$rd_wb),
+                             (ins GPR:$rd, GPR:$rs1, uimm5:$is3, uimm5:$is2),
+                             "cv.insert", "$rd, $rs1, $is3, $is2">;
+    def CV_INSERTR : RVInstR<0b0011010, 0b011, OPC_CUSTOM_1, (outs GPR:$rd_wb), 
+                             (ins GPR:$rd, GPR:$rs1, GPR:$rs2), 
+                             "cv.insertr", "$rd, $rs1, $rs2">;
+  }
+
+  def CV_BCLRR : CVBitManipRR<0b0011100, "cv.bclrr">;
+  def CV_BSETR : CVBitManipRR<0b0011101, "cv.bsetr">;
+
+  def CV_ROR : CVBitManipRR<0b0100000, "cv.ror">;
+  def CV_FF1 : CVBitManipR<0b0100001, "cv.ff1">;
+  def CV_FL1 : CVBitManipR<0b0100010, "cv.fl1">;
+  def CV_CLB : CVBitManipR<0b0100011, "cv.clb">;
+  def CV_CNT : CVBitManipR<0b0100100, "cv.cnt">;
+}
diff --git a/llvm/test/MC/RISCV/corev/XCVbitmanip-invalid.s b/llvm/test/MC/RISCV/corev/XCVbitmanip-invalid.s
new file mode 100644
index 0000000000000..684b3f220a994
--- /dev/null
+++ b/llvm/test/MC/RISCV/corev/XCVbitmanip-invalid.s
@@ -0,0 +1,267 @@
+# RUN: not llvm-mc -triple=riscv32 --mattr=+xcvbitmanip %s 2>&1 \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ERROR
+
+cv.extract t0, t1
+# CHECK-ERROR: too few operands for instruction
+
+cv.extract t0, t1, 0
+# CHECK-ERROR: too few operands for instruction
+
+cv.extract t0, t1, t2
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.extract t0, t1, t2, t3
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.extract t0, t1, 0, 32
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.extract t0, t1, 0, -1
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.extract t0, t1, 32, 0
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.extract t0, t1, -1, 0
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.extractu t0, t1
+# CHECK-ERROR: too few operands for instruction
+
+cv.extractu t0, t1, 0
+# CHECK-ERROR: too few operands for instruction
+
+cv.extractu t0, t1, t2
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.extractu t0, t1, t2, t3
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.extractu t0, t1, 0, 32
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.extractu t0, t1, 0, -1
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.extractu t0, t1, 32, 0
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.extractu t0, t1, -1, 0
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.insert t0, t1
+# CHECK-ERROR: too few operands for instruction
+
+cv.insert t0, t1, 0
+# CHECK-ERROR: too few operands for instruction
+
+cv.insert t0, t1, t2
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.insert t0, t1, t2, t3
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.insert t0, t1, 0, 32
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.insert t0, t1, 0, -1
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.insert t0, t1, 32, 0
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.insert t0, t1, -1, 0
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.bclr t0, t1
+# CHECK-ERROR: too few operands for instruction
+
+cv.bclr t0, t1, 0
+# CHECK-ERROR: too few operands for instruction
+
+cv.bclr t0, t1, t2
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.bclr t0, t1, t2, t3
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.bclr t0, t1, 0, 32
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.bclr t0, t1, 0, -1
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.bclr t0, t1, 32, 0
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.bclr t0, t1, -1, 0
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.bset t0, t1
+# CHECK-ERROR: too few operands for instruction
+
+cv.bset t0, t1, 0
+# CHECK-ERROR: too few operands for instruction
+
+cv.bset t0, t1, t2
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.bset t0, t1, t2, t3
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.bset t0, t1, 0, 32
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.bset t0, t1, 0, -1
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.bset t0, t1, 32, 0
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.bset t0, t1, -1, 0
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.bitrev t0, t1
+# CHECK-ERROR: too few operands for instruction
+
+cv.bitrev t0, t1, 0
+# CHECK-ERROR: too few operands for instruction
+
+cv.bitrev t0, t1, t2
+# CHECK-ERROR: immediate must be an integer in the range [0, 3]
+
+cv.bitrev t0, t1, t2, t3
+# CHECK-ERROR: immediate must be an integer in the range [0, 3]
+
+cv.bitrev t0, t1, 0, 32
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.bitrev t0, t1, 0, -1
+# CHECK-ERROR: immediate must be an integer in the range [0, 31]
+
+cv.bitrev t0, t1, 32, 0
+# CHECK-ERROR: immediate must be an integer in the range [0, 3]
+
+cv.bitrev t0, t1, -1, 0
+# CHECK-ERROR: immediate must be an integer in the range [0, 3]
+
+cv.extractr t0
+# CHECK-ERROR: too few operands for instruction
+
+cv.extractr t0, 0
+# CHECK-ERROR: invalid operand for instruction
+
+cv.extractr t0, t1, 0
+# CHECK-ERROR: invalid operand for instruction
+
+cv.extractr t0, t1
+# CHECK-ERROR: too few operands for instruction
+
+cv.extractur t0
+# CHECK-ERROR: too few operands for instruction
+
+cv.extractur t0, 0
+# CHECK-ERROR: invalid operand for instruction
+
+cv.extractur t0, t1, 0
+# CHECK-ERROR: invalid operand for instruction
+
+cv.extractur t0, t1
+# CHECK-ERROR: too few operands for instruction
+
+cv.insertr t0
+# CHECK-ERROR: too few operands for instruction
+
+cv.insertr t0, 0
+# CHECK-ERROR: invalid operand for instruction
+
+cv.insertr t0, t1, 0
+# CHECK-ERROR: invalid operand for instruction
+
+cv.insertr t0, t1
+# CHECK-ERROR: too few operands for instruction
+
+cv.bclrr t0
+# CHECK-ERROR: too few operands for instruction
+
+cv.bclrr t0, 0
+# CHECK-ERROR: invalid operand for instruction
+
+cv.bclrr t0, t1, 0
+# CHECK-ERROR: invalid operand for instruction
+
+cv.bclrr t0, t1
+# CHECK-ERROR: too few operands for instruction
+
+cv.bsetr t0
+# CHECK-ERROR: too few operands for instruction
+
+cv.bsetr t0, 0
+# CHECK-ERROR: invalid operand for instruction
+
+cv.bsetr t0, t1, 0
+# CHECK-ERROR: invalid operand for instruction
+
+cv.bsetr t0, t1
+# CHECK-ERROR: too few operands for instruction
+
+cv.ror t0
+# CHECK-ERROR: too few operands for instruction
+
+cv.ror t0, 0
+# CHECK-ERROR: invalid operand for instruction
+
+cv.ror t0, t1, 0
+# CHECK-ERROR: invalid operand for instruction
+
+cv.ror t0, t1
+# CHECK-ERROR: too few operands for instruction
+
+cv.ff1 t0
+# CHECK-ERROR: too few operands for instruction
+
+cv.ff1 t0, 0
+# CHECK-ERROR: invalid operand for instruction
+
+cv.ff1 t0, t1, 0
+# CHECK-ERROR: invalid operand for instruction
+
+cv.ff1 t0, t1, t2
+# CHECK-ERROR: invalid operand for instruction 
+
+cv.fl1 t0
+# CHECK-ERROR: too few operands for instruction
+
+cv.fl1 t0, 0
+# CHECK-ERROR: invalid operand for instruction
+
+cv.fl1 t0, t1, 0
+# CHECK-ERROR: invalid operand for instruction
+
+cv.fl1 t0, t1, t2
+# CHECK-ERROR: invalid operand for instruction 
+
+cv.clb t0
+# CHECK-ERROR: too few operands for instruction
+
+cv.clb t0, 0
+# CHECK-ERROR: invalid operand for instruction
+
+cv.clb t0, t1, 0
+# CHECK-ERROR: invalid operand for instruction
+
+cv.clb t0, t1, t2
+# CHECK-ERROR: invalid operand for instruction 
+
+cv.cnt t0
+# CHECK-ERROR: too few operands for instruction
+
+cv.cnt t0, 0
+# CHECK-ERROR: invalid operand for instruction
+
+cv.cnt t0, t1, 0
+# CHECK-ERROR: invalid operand for instruction
+
+cv.cnt t0, t1, t2
+# CHECK-ERROR: invalid operand for instruction 
+
diff --git a/llvm/test/MC/RISCV/corev/XCVbitmanip.s b/llvm/test/MC/RISCV/corev/XCVbitmanip.s
new file mode 100644
index 0000000000000..dab8395c93f01
--- /dev/null
+++ b/llvm/test/MC/RISCV/corev/XCVbitmanip.s
@@ -0,0 +1,248 @@
+# RUN: llvm-mc -triple=riscv32 --mattr=+xcvbitmanip -show-encoding %s \
+# RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INSTR
+# RUN: llvm-mc -filetype=obj -triple=riscv32 -mattr=+xcvbitmanip < %s \
+# RUN:     | llvm-objdump --mattr=+xcvbitmanip -M no-aliases -d -r - \
+# RUN:     | FileCheck --check-prefix=CHECK-INSTR %s
+# RUN: not llvm-mc -triple riscv32 %s 2>&1 \
+# RUN:     | FileCheck -check-prefix=CHECK-NO-EXT %s
+
+cv.extract t0, t1, 0, 1
+# CHECK-INSTR: cv.extract t0, t1, 0, 1
+# CHECK-ENCODING: [0xdb,0x02,0x13,0x00]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.extract a0, a1, 17, 18
+# CHECK-INSTR: cv.extract a0, a1, 17, 18
+# CHECK-ENCODING: [0x5b,0x85,0x25,0x23]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.extract s0, s1, 30, 31
+# CHECK-INSTR: cv.extract s0, s1, 30, 31
+# CHECK-ENCODING: [0x5b,0x84,0xf4,0x3d]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.extractu t0, t1, 0, 1
+# CHECK-INSTR: cv.extractu t0, t1, 0, 1
+# CHECK-ENCODING: [0xdb,0x02,0x13,0x40]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.extractu a0, a1, 17, 18
+# CHECK-INSTR: cv.extractu a0, a1, 17, 18
+# CHECK-ENCODING: [0x5b,0x85,0x25,0x63]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.extractu s0, s1, 30, 31
+# CHECK-INSTR: cv.extractu s0, s1, 30, 31
+# CHECK-ENCODING: [0x5b,0x84,0xf4,0x7d]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.insert t0, t1, 0, 1
+# CHECK-INSTR: cv.insert t0, t1, 0, 1
+# CHECK-ENCODING: [0xdb,0x02,0x13,0x80]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.insert a0, a1, 17, 18
+# CHECK-INSTR: cv.insert a0, a1, 17, 18
+# CHECK-ENCODING: [0x5b,0x85,0x25,0xa3]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.insert s0, s1, 30, 31
+# CHECK-INSTR: cv.insert s0, s1, 30, 31
+# CHECK-ENCODING: [0x5b,0x84,0xf4,0xbd]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.bclr t0, t1, 0, 1
+# CHECK-INSTR: cv.bclr t0, t1, 0, 1
+# CHECK-ENCODING: [0xdb,0x12,0x13,0x00]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.bclr a0, a1, 17, 18
+# CHECK-INSTR: cv.bclr a0, a1, 17, 18
+# CHECK-ENCODING: [0x5b,0x95,0x25,0x23]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.bclr s0, s1, 30, 31
+# CHECK-INSTR: cv.bclr s0, s1, 30, 31
+# CHECK-ENCODING: [0x5b,0x94,0xf4,0x3d]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.bset t0, t1, 0, 1
+# CHECK-INSTR: cv.bset t0, t1, 0, 1
+# CHECK-ENCODING: [0xdb,0x12,0x13,0x40]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.bset a0, a1, 17, 18
+# CHECK-INSTR: cv.bset a0, a1, 17, 18
+# CHECK-ENCODING: [0x5b,0x95,0x25,0x63]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.bset s0, s1, 30, 31
+# CHECK-INSTR: cv.bset s0, s1, 30, 31
+# CHECK-ENCODING: [0x5b,0x94,0xf4,0x7d]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.bitrev t0, t1, 0, 1
+# CHECK-INSTR: cv.bitrev t0, t1, 0, 1
+# CHECK-ENCODING: [0xdb,0x12,0x13,0xc0]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.bitrev a0, a1, 1, 18
+# CHECK-INSTR: cv.bitrev a0, a1, 1, 18
+# CHECK-ENCODING: [0x5b,0x95,0x25,0xc3]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.bitrev s0, s1, 2, 31
+# CHECK-INSTR: cv.bitrev s0, s1, 2, 31
+# CHECK-ENCODING: [0x5b,0x94,0xf4,0xc5]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.extractr t0, t1, t2
+# CHECK-INSTR: cv.extractr t0, t1, t2
+# CHECK-ENCODING: [0xab,0x32,0x73,0x30]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.extractr a0, a1, a2
+# CHECK-INSTR: cv.extractr a0, a1, a2
+# CHECK-ENCODING: [0x2b,0xb5,0xc5,0x30]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.extractr s0, s1, s2
+# CHECK-INSTR: cv.extractr s0, s1, s2
+# CHECK-ENCODING: [0x2b,0xb4,0x24,0x31]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.extractur t0, t1, t2
+# CHECK-INSTR: cv.extractur t0, t1, t2
+# CHECK-ENCODING: [0xab,0x32,0x73,0x32]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.extractur a0, a1, a2
+# CHECK-INSTR: cv.extractur a0, a1, a2
+# CHECK-ENCODING: [0x2b,0xb5,0xc5,0x32]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.extractur s0, s1, s2
+# CHECK-INSTR: cv.extractur s0, s1, s2
+# CHECK-ENCODING: [0x2b,0xb4,0x24,0x33]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.insertr t0, t1, t2
+# CHECK-INSTR: cv.insertr t0, t1, t2
+# CHECK-ENCODING: [0xab,0x32,0x73,0x34]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.insertr a0, a1, a2
+# CHECK-INSTR: cv.insertr a0, a1, a2
+# CHECK-ENCODING: [0x2b,0xb5,0xc5,0x34]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.insertr s0, s1, s2
+# CHECK-INSTR: cv.insertr s0, s1, s2
+# CHECK-ENCODING: [0x2b,0xb4,0x24,0x35]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.bclrr t0, t1, t2
+# CHECK-INSTR: cv.bclrr t0, t1, t2
+# CHECK-ENCODING: [0xab,0x32,0x73,0x38]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.bclrr a0, a1, a2
+# CHECK-INSTR: cv.bclrr a0, a1, a2
+# CHECK-ENCODING: [0x2b,0xb5,0xc5,0x38]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.bclrr s0, s1, s2
+# CHECK-INSTR: cv.bclrr s0, s1, s2
+# CHECK-ENCODING: [0x2b,0xb4,0x24,0x39]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.bsetr t0, t1, t2
+# CHECK-INSTR: cv.bsetr t0, t1, t2
+# CHECK-ENCODING: [0xab,0x32,0x73,0x3a]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.bsetr a0, a1, a2
+# CHECK-INSTR: cv.bsetr a0, a1, a2
+# CHECK-ENCODING: [0x2b,0xb5,0xc5,0x3a]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.bsetr s0, s1, s2
+# CHECK-INSTR: cv.bsetr s0, s1, s2
+# CHECK-ENCODING: [0x2b,0xb4,0x24,0x3b]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.ror t0, t1, t2
+# CHECK-INSTR: cv.ror t0, t1, t2
+# CHECK-ENCODING: [0xab,0x32,0x73,0x40]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.ror a0, a1, a2
+# CHECK-INSTR: cv.ror a0, a1, a2
+# CHECK-ENCODING: [0x2b,0xb5,0xc5,0x40]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.ror s0, s1, s2
+# CHECK-INSTR: cv.ror s0, s1, s2
+# CHECK-ENCODING: [0x2b,0xb4,0x24,0x41]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.ff1 t0, t1
+# CHECK-INSTR: cv.ff1 t0, t1
+# CHECK-ENCODING: [0xab,0x32,0x03,0x42]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.ff1 a0, a1
+# CHECK-INSTR: cv.ff1 a0, a1
+# CHECK-ENCODING: [0x2b,0xb5,0x05,0x42]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.ff1 s0, s1
+# CHECK-INSTR: cv.ff1 s0, s1
+# CHECK-ENCODING: [0x2b,0xb4,0x04,0x42]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.fl1 t0, t1
+# CHECK-INSTR: cv.fl1 t0, t1
+# CHECK-ENCODING: [0xab,0x32,0x03,0x44]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.fl1 a0, a1
+# CHECK-INSTR: cv.fl1 a0, a1
+# CHECK-ENCODING: [0x2b,0xb5,0x05,0x44]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.fl1 s0, s1
+# CHECK-INSTR: cv.fl1 s0, s1
+# CHECK-ENCODING: [0x2b,0xb4,0x04,0x44]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.clb t0, t1
+# CHECK-INSTR: cv.clb t0, t1
+# CHECK-ENCODING: [0xab,0x32,0x03,0x46]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.clb a0, a1
+# CHECK-INSTR: cv.clb a0, a1
+# CHECK-ENCODING: [0x2b,0xb5,0x05,0x46]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.clb s0, s1
+# CHECK-INSTR: cv.clb s0, s1
+# CHECK-ENCODING: [0x2b,0xb4,0x04,0x46]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.cnt t0, t1
+# CHECK-INSTR: cv.cnt t0, t1
+# CHECK-ENCODING: [0xab,0x32,0x03,0x48]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.cnt a0, a1
+# CHECK-INSTR: cv.cnt a0, a1
+# CHECK-ENCODING: [0x2b,0xb5,0x05,0x48]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+
+cv.cnt s0, s1
+# CHECK-INSTR: cv.cnt s0, s1
+# CHECK-ENCODING: [0x2b,0xb4,0x04,0x48]
+# CHECK-NO-EXT: instruction requires the following: 'XCVbitmanip' (Bit Manipulation){{$}}
+

From 9d4c1be8b0793ee94e0906e4ff40d1ff325470de Mon Sep 17 00:00:00 2001
From: David Truby <david@truby.dev>
Date: Fri, 16 Jun 2023 12:45:54 +0100
Subject: [PATCH 021/130] [mlir] Add support for LLVMIR comdat operation

The LLVM comdat operation specifies how to deduplicate globals with the
same key in two different object files. This is necessary on Windows
where e.g. two object files with linkonce globals will not link unless
a comdat for those globals is specified. It is also supported in the ELF
format.

Differential Revision: https://reviews.llvm.org/D150796
---
 .../mlir/Dialect/LLVMIR/LLVMAttrDefs.td       |  9 +++
 mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td | 25 +++++++++
 mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td   | 51 +++++++++++++++++
 .../include/mlir/Target/LLVMIR/ModuleImport.h | 10 ++++
 .../mlir/Target/LLVMIR/ModuleTranslation.h    |  1 +
 mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp    | 55 ++++++++++++++++++-
 mlir/lib/Target/LLVMIR/ModuleImport.cpp       | 46 ++++++++++++++++
 mlir/lib/Target/LLVMIR/ModuleTranslation.cpp  | 30 +++++++++-
 mlir/test/Dialect/LLVMIR/comdat.mlir          | 16 ++++++
 mlir/test/Dialect/LLVMIR/global.mlir          |  8 +++
 mlir/test/Dialect/LLVMIR/invalid.mlir         | 13 +++++
 mlir/test/Target/LLVMIR/Import/comdat.ll      | 20 +++++++
 mlir/test/Target/LLVMIR/llvmir-invalid.mlir   | 11 ++++
 mlir/test/Target/LLVMIR/llvmir.mlir           | 30 ++++++++++
 14 files changed, 322 insertions(+), 3 deletions(-)
 create mode 100644 mlir/test/Dialect/LLVMIR/comdat.mlir
 create mode 100644 mlir/test/Target/LLVMIR/Import/comdat.ll

diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
index 05ee0832360dc..62eca517c828c 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
@@ -29,6 +29,15 @@ def CConvAttr : LLVM_Attr<"CConv", "cconv"> {
   let assemblyFormat = "`<` $CallingConv `>`";
 }
 
+//===----------------------------------------------------------------------===//
+// ComdatAttr
+//===----------------------------------------------------------------------===//
+
+def ComdatAttr : LLVM_Attr<"Comdat", "comdat"> {
+  let parameters = (ins "comdat::Comdat":$comdat);
+  let assemblyFormat = "$comdat";
+}
+
 //===----------------------------------------------------------------------===//
 // LinkageAttr
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td
index 8d8494924e2a9..9eb012a971efb 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td
@@ -589,6 +589,31 @@ def Linkage : DialectAttr<
           "::mlir::LLVM::LinkageAttr::get($_builder.getContext(), $0)";
 }
 
+
+//===----------------------------------------------------------------------===//
+// Comdat
+//===----------------------------------------------------------------------===//
+
+def ComdatAny
+    : LLVM_EnumAttrCase<"Any", "any", "Any", 0>;
+def ComdatExactMatch
+    : LLVM_EnumAttrCase<"ExactMatch", "exactmatch", "ExactMatch", 1>;
+def ComdatLargest
+    : LLVM_EnumAttrCase<"Largest", "largest", "Largest", 2>;
+def ComdatNoDeduplicate
+    : LLVM_EnumAttrCase<"NoDeduplicate", "nodeduplicate", "NoDeduplicate", 3>;
+def ComdatSameSize
+    : LLVM_EnumAttrCase<"SameSize", "samesize", "SameSize", 4>;
+
+def Comdat : LLVM_EnumAttr<
+  "Comdat",
+  "::llvm::Comdat::SelectionKind",
+  "LLVM Comdat Types",
+  [ComdatAny, ComdatExactMatch, ComdatLargest,
+   ComdatNoDeduplicate, ComdatSameSize]> {
+  let cppNamespace = "::mlir::LLVM::comdat";
+}
+
 //===----------------------------------------------------------------------===//
 // UnnamedAddr
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index 53418cfec0735..b4081091a579d 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -1321,6 +1321,7 @@ def LLVM_GlobalOp : LLVM_Op<"mlir.global",
     DefaultValuedAttr<ConfinedAttr<I32Attr, [IntNonNegative]>, "0">:$addr_space,
     OptionalAttr<UnnamedAddr>:$unnamed_addr,
     OptionalAttr<StrAttr>:$section,
+    OptionalAttr<SymbolRefAttr>:$comdat,
     DefaultValuedAttr<Visibility, "mlir::LLVM::Visibility::Default">:$visibility_
   );
   let summary = "LLVM dialect global.";
@@ -1429,6 +1430,7 @@ def LLVM_GlobalOp : LLVM_Op<"mlir.global",
       CArg<"unsigned", "0">:$addrSpace,
       CArg<"bool", "false">:$dsoLocal,
       CArg<"bool", "false">:$thread_local_,
+      CArg<"SymbolRefAttr", "{}">:$comdat,
       CArg<"ArrayRef<NamedAttribute>", "{}">:$attrs)>
   ];
 
@@ -1520,6 +1522,55 @@ def LLVM_GlobalDtorsOp : LLVM_Op<"mlir.global_dtors", [
   let hasVerifier = 1;
 }
 
+def LLVM_ComdatSelectorOp : LLVM_Op<"comdat_selector", [Symbol]> {
+  let arguments = (ins
+    SymbolNameAttr:$sym_name,
+    Comdat:$comdat
+  );
+
+  let summary = "LLVM dialect comdat selector declaration";
+
+  let description = [{
+    Provides access to object file COMDAT section/group functionality.
+
+    Examples:
+    ```mlir
+    llvm.comdat @__llvm_comdat {
+      llvm.comdat_selector @any any
+    }
+    llvm.mlir.global internal constant @has_any_comdat(1 : i64) comdat(@__llvm_comdat::@any) : i64
+    ```
+  }];
+  let assemblyFormat = "$sym_name $comdat attr-dict";
+}
+
+def LLVM_ComdatOp : LLVM_Op<"comdat", [NoTerminator, NoRegionArguments, SymbolTable, Symbol]> {
+  let arguments = (ins
+    SymbolNameAttr:$sym_name
+  );
+  let summary = "LLVM dialect comdat region";
+
+  let description = [{
+    Provides access to object file COMDAT section/group functionality.
+
+    Examples:
+    ```mlir
+    llvm.comdat @__llvm_comdat {
+      llvm.comdat_selector @any any
+    }
+    llvm.mlir.global internal constant @has_any_comdat(1 : i64) comdat(@__llvm_comdat::@any) : i64
+    ```
+  }];
+  let regions = (region SizedRegion<1>:$body);
+
+
+  let skipDefaultBuilders = 1;
+  let builders = [OpBuilder<(ins "StringRef":$symName)>];
+
+  let assemblyFormat = "$sym_name $body attr-dict";
+  let hasRegionVerifier = 1;
+}
+
 def LLVM_LLVMFuncOp : LLVM_Op<"func", [
     AutomaticAllocationScope, IsolatedFromAbove, FunctionOpInterface,
     CallableOpInterface
diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h
index ce622f7cabab3..b9d7463d34c63 100644
--- a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h
+++ b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h
@@ -56,6 +56,10 @@ class ModuleImport {
   /// Converts all functions of the LLVM module to MLIR functions.
   LogicalResult convertFunctions();
 
+  /// Converts all comdat selectors of the LLVM module to MLIR comdat
+  /// operations.
+  LogicalResult convertComdats();
+
   /// Converts all global variables of the LLVM module to MLIR global variables.
   LogicalResult convertGlobals();
 
@@ -284,6 +288,10 @@ class ModuleImport {
   /// metadata that converts to MLIR operations. Creates the global metadata
   /// operation on the first invocation.
   MetadataOp getGlobalMetadataOp();
+  /// Returns a global comdat operation that serves as a container for LLVM
+  /// comdat selectors. Creates the global comdat operation on the first
+  /// invocation.
+  ComdatOp getGlobalComdatOp();
   /// Performs conversion of LLVM TBAA metadata starting from
   /// `node`. On exit from this function all nodes reachable
   /// from `node` are converted, and tbaaMapping map is updated
@@ -312,6 +320,8 @@ class ModuleImport {
   Operation *globalInsertionOp = nullptr;
   /// Operation to insert metadata operations into.
   MetadataOp globalMetadataOp = nullptr;
+  /// Operation to insert comdat selector operations into.
+  ComdatOp globalComdatOp = nullptr;
   /// The current context.
   MLIRContext *context;
   /// The MLIR module being created.
diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
index 949a0c705066f..960416d4913f7 100644
--- a/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
+++ b/mlir/include/mlir/Target/LLVMIR/ModuleTranslation.h
@@ -274,6 +274,7 @@ class ModuleTranslation {
   LogicalResult convertOperation(Operation &op, llvm::IRBuilderBase &builder);
   LogicalResult convertFunctionSignatures();
   LogicalResult convertFunctions();
+  LogicalResult convertComdats();
   LogicalResult convertGlobals();
   LogicalResult convertOneFunction(LLVMFuncOp func);
 
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index c44c60a0bc176..a97632c222287 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -1609,6 +1609,28 @@ AddressOfOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// Verifier for LLVM::ComdatOp.
+//===----------------------------------------------------------------------===//
+
+void ComdatOp::build(OpBuilder &builder, OperationState &result,
+                     StringRef symName) {
+  result.addAttribute(getSymNameAttrName(result.name),
+                      builder.getStringAttr(symName));
+  Region *body = result.addRegion();
+  body->emplaceBlock();
+}
+
+LogicalResult ComdatOp::verifyRegions() {
+  Region &body = getBody();
+  for (Operation &op : body.getOps())
+    if (!isa<ComdatSelectorOp>(op))
+      return op.emitError(
+          "only comdat selector symbols can appear in a comdat region");
+
+  return success();
+}
+
 //===----------------------------------------------------------------------===//
 // Builder, printer and verifier for LLVM::GlobalOp.
 //===----------------------------------------------------------------------===//
@@ -1616,7 +1638,7 @@ AddressOfOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
 void GlobalOp::build(OpBuilder &builder, OperationState &result, Type type,
                      bool isConstant, Linkage linkage, StringRef name,
                      Attribute value, uint64_t alignment, unsigned addrSpace,
-                     bool dsoLocal, bool threadLocal,
+                     bool dsoLocal, bool threadLocal, SymbolRefAttr comdat,
                      ArrayRef<NamedAttribute> attrs) {
   result.addAttribute(getSymNameAttrName(result.name),
                       builder.getStringAttr(name));
@@ -1632,6 +1654,8 @@ void GlobalOp::build(OpBuilder &builder, OperationState &result, Type type,
   if (threadLocal)
     result.addAttribute(getThreadLocal_AttrName(result.name),
                         builder.getUnitAttr());
+  if (comdat)
+    result.addAttribute(getComdatAttrName(result.name), comdat);
 
   // Only add an alignment attribute if the "alignment" input
   // is different from 0. The value must also be a power of two, but
@@ -1668,6 +1692,9 @@ void GlobalOp::print(OpAsmPrinter &p) {
   if (auto value = getValueOrNull())
     p.printAttribute(value);
   p << ')';
+  if (auto comdat = getComdat())
+    p << " comdat(" << *comdat << ')';
+
   // Note that the alignment attribute is printed using the
   // default syntax here, even though it is an inherent attribute
   // (as defined in https://mlir.llvm.org/docs/LangRef/#attributes)
@@ -1676,7 +1703,7 @@ void GlobalOp::print(OpAsmPrinter &p) {
                            getGlobalTypeAttrName(), getConstantAttrName(),
                            getValueAttrName(), getLinkageAttrName(),
                            getUnnamedAddrAttrName(), getThreadLocal_AttrName(),
-                           getVisibility_AttrName()});
+                           getVisibility_AttrName(), getComdatAttrName()});
 
   // Print the trailing type unless it's a string global.
   if (llvm::dyn_cast_or_null<StringAttr>(getValueOrNull()))
@@ -1736,6 +1763,18 @@ static RetTy parseOptionalLLVMKeyword(OpAsmParser &parser,
   return static_cast<RetTy>(index);
 }
 
+static LogicalResult verifyComdat(Operation *op,
+                                  std::optional<SymbolRefAttr> attr) {
+  if (!attr)
+    return success();
+
+  auto *comdatSelector = SymbolTable::lookupNearestSymbolFrom(op, *attr);
+  if (!isa_and_nonnull<ComdatSelectorOp>(comdatSelector))
+    return op->emitError() << "expected comdat symbol";
+
+  return success();
+}
+
 // operation ::= `llvm.mlir.global` linkage? `constant`? `@` identifier
 //               `(` attribute? `)` align? attribute-list? (`:` type)? region?
 // align     ::= `align` `=` UINT64
@@ -1784,6 +1823,15 @@ ParseResult GlobalOp::parse(OpAsmParser &parser, OperationState &result) {
       return failure();
   }
 
+  if (succeeded(parser.parseOptionalKeyword("comdat"))) {
+    SymbolRefAttr comdat;
+    if (parser.parseLParen() || parser.parseAttribute(comdat) ||
+        parser.parseRParen())
+      return failure();
+
+    result.addAttribute(getComdatAttrName(result.name), comdat);
+  }
+
   SmallVector<Type, 1> types;
   if (parser.parseOptionalAttrDict(result.attributes) ||
       parser.parseOptionalColonTypeList(types))
@@ -1882,6 +1930,9 @@ LogicalResult GlobalOp::verify() {
     }
   }
 
+  if (failed(verifyComdat(*this, getComdat())))
+    return failure();
+
   std::optional<uint64_t> alignAttr = getAlignment();
   if (alignAttr.has_value()) {
     uint64_t value = alignAttr.value();
diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
index 5f9eb1835cd2d..fbb20bed6e8ff 100644
--- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
@@ -29,6 +29,7 @@
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/StringSet.h"
+#include "llvm/IR/Comdat.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/InstIterator.h"
@@ -81,6 +82,12 @@ static constexpr StringRef getGlobalMetadataOpName() {
   return "__llvm_global_metadata";
 }
 
+/// Returns the symbol name for the module-level comdat operation. It must not
+/// conflict with the user namespace.
+static constexpr StringRef getGlobalComdatOpName() {
+  return "__llvm_global_comdat";
+}
+
 /// Converts the sync scope identifier of `inst` to the string representation
 /// necessary to build an atomic LLVM dialect operation. Returns the empty
 /// string if the operation has either no sync scope or the default system-level
@@ -167,6 +174,16 @@ MetadataOp ModuleImport::getGlobalMetadataOp() {
              mlirModule.getLoc(), getGlobalMetadataOpName());
 }
 
+ComdatOp ModuleImport::getGlobalComdatOp() {
+  if (globalComdatOp)
+    return globalComdatOp;
+
+  OpBuilder::InsertionGuard guard(builder);
+  builder.setInsertionPointToStart(mlirModule.getBody());
+  return globalComdatOp = builder.create<ComdatOp>(mlirModule.getLoc(),
+                                                   getGlobalComdatOpName());
+}
+
 LogicalResult ModuleImport::processTBAAMetadata(const llvm::MDNode *node) {
   Location loc = mlirModule.getLoc();
   SmallVector<const llvm::MDNode *> workList;
@@ -540,6 +557,20 @@ LogicalResult ModuleImport::convertMetadata() {
   return success();
 }
 
+LogicalResult ModuleImport::convertComdats() {
+  ComdatOp comdat = getGlobalComdatOp();
+  OpBuilder::InsertionGuard guard(builder);
+  builder.setInsertionPointToEnd(&comdat.getBody().back());
+  for (auto &kv : llvmModule->getComdatSymbolTable()) {
+    StringRef name = kv.getKey();
+    llvm::Comdat::SelectionKind selector = kv.getValue().getSelectionKind();
+    builder.create<ComdatSelectorOp>(mlirModule.getLoc(), name,
+                                     convertComdatFromLLVM(selector));
+  }
+
+  return success();
+}
+
 LogicalResult ModuleImport::convertGlobals() {
   for (llvm::GlobalVariable &globalVar : llvmModule->globals()) {
     if (globalVar.getName() == getGlobalCtorsVarName() ||
@@ -857,6 +888,19 @@ LogicalResult ModuleImport::convertGlobal(llvm::GlobalVariable *globalVar) {
   globalOp.setVisibility_(
       convertVisibilityFromLLVM(globalVar->getVisibility()));
 
+  if (globalVar->hasComdat()) {
+    llvm::Comdat *llvmComdat = globalVar->getComdat();
+    ComdatOp comdat = getGlobalComdatOp();
+    if (ComdatSelectorOp selector = dyn_cast<ComdatSelectorOp>(
+            comdat.lookupSymbol(llvmComdat->getName()))) {
+      auto symbolRef =
+          SymbolRefAttr::get(builder.getContext(), getGlobalComdatOpName(),
+                             FlatSymbolRefAttr::get(selector.getSymNameAttr()));
+      globalOp.setComdatAttr(symbolRef);
+    } else
+      return failure();
+  }
+
   return success();
 }
 
@@ -1780,6 +1824,8 @@ mlir::translateLLVMIRToModule(std::unique_ptr<llvm::Module> llvmModule,
     return {};
   if (failed(moduleImport.convertMetadata()))
     return {};
+  if (failed(moduleImport.convertComdats()))
+    return {};
   if (failed(moduleImport.convertGlobals()))
     return {};
   if (failed(moduleImport.convertFunctions()))
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index ee176a43c6ff9..9f072191804e0 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -27,6 +27,7 @@
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/RegionGraphTraits.h"
 #include "mlir/Support/LLVM.h"
+#include "mlir/Support/LogicalResult.h"
 #include "mlir/Target/LLVMIR/LLVMTranslationInterface.h"
 #include "mlir/Target/LLVMIR/TypeToLLVM.h"
 
@@ -724,6 +725,14 @@ LogicalResult ModuleTranslation::convertGlobals() {
                              : llvm::GlobalValue::NotThreadLocal,
         addrSpace);
 
+    if (std::optional<mlir::SymbolRefAttr> comdat = op.getComdat()) {
+      StringRef name = comdat->getLeafReference().getValue();
+      if (!llvmModule->getComdatSymbolTable().contains(name))
+        return emitError(op.getLoc(), "global references non-existant comdat");
+      llvm::Comdat *llvmComdat = llvmModule->getOrInsertComdat(name);
+      var->setComdat(llvmComdat);
+    }
+
     if (op.getUnnamedAddr().has_value())
       var->setUnnamedAddr(convertUnnamedAddrToLLVM(*op.getUnnamedAddr()));
 
@@ -1037,6 +1046,23 @@ LogicalResult ModuleTranslation::convertFunctions() {
   return success();
 }
 
+LogicalResult ModuleTranslation::convertComdats() {
+  for (ComdatOp comdat : getModuleBody(mlirModule).getOps<ComdatOp>()) {
+    for (ComdatSelectorOp selector : comdat.getOps<ComdatSelectorOp>()) {
+      StringRef name = selector.getName();
+      llvm::Module *module = getLLVMModule();
+      if (module->getComdatSymbolTable().contains(name)) {
+        return emitError(selector.getLoc())
+               << "comdat selection symbols must be unique even in different "
+                  "comdat regions";
+      }
+      llvm::Comdat *comdat = module->getOrInsertComdat(name);
+      comdat->setSelectionKind(convertComdatToLLVM(selector.getComdat()));
+    }
+  }
+  return success();
+}
+
 LogicalResult ModuleTranslation::createAccessGroupMetadata() {
   return loopAnnotationTranslation->createAccessGroupMetadata();
 }
@@ -1372,6 +1398,8 @@ mlir::translateModuleToLLVMIR(Operation *module, llvm::LLVMContext &llvmContext,
   ModuleTranslation translator(module, std::move(llvmModule));
   if (failed(translator.convertFunctionSignatures()))
     return nullptr;
+  if (failed(translator.convertComdats()))
+    return nullptr;
   if (failed(translator.convertGlobals()))
     return nullptr;
   if (failed(translator.createAccessGroupMetadata()))
@@ -1387,7 +1415,7 @@ mlir::translateModuleToLLVMIR(Operation *module, llvm::LLVMContext &llvmContext,
   llvm::IRBuilder<> llvmBuilder(llvmContext);
   for (Operation &o : getModuleBody(module).getOperations()) {
     if (!isa<LLVM::LLVMFuncOp, LLVM::GlobalOp, LLVM::GlobalCtorsOp,
-             LLVM::GlobalDtorsOp, LLVM::MetadataOp>(&o) &&
+             LLVM::GlobalDtorsOp, LLVM::MetadataOp, LLVM::ComdatOp>(&o) &&
         !o.hasTrait<OpTrait::IsTerminator>() &&
         failed(translator.convertOperation(o, llvmBuilder))) {
       return nullptr;
diff --git a/mlir/test/Dialect/LLVMIR/comdat.mlir b/mlir/test/Dialect/LLVMIR/comdat.mlir
new file mode 100644
index 0000000000000..e8eaaa15160a0
--- /dev/null
+++ b/mlir/test/Dialect/LLVMIR/comdat.mlir
@@ -0,0 +1,16 @@
+// RUN: mlir-opt -split-input-file -verify-diagnostics %s | FileCheck %s
+
+// CHECK: llvm.comdat @__llvm_comdat
+llvm.comdat @__llvm_comdat {
+  // CHECK: llvm.comdat_selector @any_comdat any
+  llvm.comdat_selector @any_comdat any
+  // CHECK: llvm.comdat_selector @exactmatch_comdat exactmatch
+  llvm.comdat_selector @exactmatch_comdat exactmatch
+  // CHECK: llvm.comdat_selector @largest_comdat largest
+  llvm.comdat_selector @largest_comdat largest
+  // CHECK: llvm.comdat_selector @nodeduplicate_comdat nodeduplicate
+  llvm.comdat_selector @nodeduplicate_comdat nodeduplicate
+  // CHECK: llvm.comdat_selector @samesize_comdat samesize
+  llvm.comdat_selector @samesize_comdat samesize
+}
+
diff --git a/mlir/test/Dialect/LLVMIR/global.mlir b/mlir/test/Dialect/LLVMIR/global.mlir
index 00b73f0549fab..c9a28e0a9eb93 100644
--- a/mlir/test/Dialect/LLVMIR/global.mlir
+++ b/mlir/test/Dialect/LLVMIR/global.mlir
@@ -64,6 +64,14 @@ llvm.mlir.global external @has_dso_local(42 : i64) {dso_local} : i64
 // CHECK: llvm.mlir.global external @has_addr_space(32 : i64) {addr_space = 3 : i32} : i64
 llvm.mlir.global external @has_addr_space(32 : i64) {addr_space = 3: i32} : i64
 
+// CHECK: llvm.comdat @__llvm_comdat
+llvm.comdat @__llvm_comdat {
+  // CHECK: llvm.comdat_selector @any any
+  llvm.comdat_selector @any any
+}
+// CHECK: llvm.mlir.global external @any() comdat(@__llvm_comdat::@any) {addr_space = 0 : i32} : i64
+llvm.mlir.global @any() comdat(@__llvm_comdat::@any) : i64
+
 // CHECK-LABEL: references
 func.func @references() {
   // CHECK: llvm.mlir.addressof @".string" : !llvm.ptr
diff --git a/mlir/test/Dialect/LLVMIR/invalid.mlir b/mlir/test/Dialect/LLVMIR/invalid.mlir
index b88619b1e388d..db8b304182805 100644
--- a/mlir/test/Dialect/LLVMIR/invalid.mlir
+++ b/mlir/test/Dialect/LLVMIR/invalid.mlir
@@ -1423,3 +1423,16 @@ func.func @invalid_target_ext_constant() {
   // expected-error@+1 {{only zero-initializer allowed for target extension types}}
   %0 = llvm.mlir.constant(42 : index) : !llvm.target<"spirv.Event">
 }
+
+// -----
+
+llvm.comdat @__llvm_comdat {
+  // expected-error@+1 {{only comdat selector symbols can appear in a comdat region}}
+  llvm.return
+}
+
+// -----
+
+llvm.mlir.global @not_comdat(0 : i32) : i32
+// expected-error@+1 {{expected comdat symbol}}
+llvm.mlir.global @invalid_comdat_use(0 : i32) comdat(@not_comdat) : i32
diff --git a/mlir/test/Target/LLVMIR/Import/comdat.ll b/mlir/test/Target/LLVMIR/Import/comdat.ll
new file mode 100644
index 0000000000000..8ef38eb8728a1
--- /dev/null
+++ b/mlir/test/Target/LLVMIR/Import/comdat.ll
@@ -0,0 +1,20 @@
+; RUN: mlir-translate -import-llvm %s | FileCheck %s
+
+; CHECK: llvm.mlir.global external @foo(42 : i64) comdat(@__llvm_global_comdat::@foo) {addr_space = 0 : i32} : i64
+@foo = global i64 42, comdat
+; CHECK: llvm.mlir.global external @bar(42 : i64) comdat(@__llvm_global_comdat::@foo) {addr_space = 0 : i32} : i64
+@bar = global i64 42, comdat($foo)
+
+; CHECK: llvm.comdat @__llvm_global_comdat {
+; CHECK-DAG: llvm.comdat_selector @foo any
+$foo = comdat any
+; CHECK-DAG: llvm.comdat_selector @exact exactmatch
+$exact = comdat exactmatch
+; CHECK-DAG: llvm.comdat_selector @largest largest
+$largest = comdat largest
+; CHECK-DAG: llvm.comdat_selector @nodedup nodeduplicate
+$nodedup = comdat nodeduplicate
+; CHECK-DAG: llvm.comdat_selector @same samesize
+$same = comdat samesize
+
+; CHECK: }
diff --git a/mlir/test/Target/LLVMIR/llvmir-invalid.mlir b/mlir/test/Target/LLVMIR/llvmir-invalid.mlir
index 0522a8a14f525..2d6ccff2d436f 100644
--- a/mlir/test/Target/LLVMIR/llvmir-invalid.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-invalid.mlir
@@ -242,3 +242,14 @@ llvm.func @stepvector_intr_wrong_type() -> vector<7xf32> {
   %0 = llvm.intr.experimental.stepvector : vector<7xf32>
   llvm.return %0 : vector<7xf32>
 }
+
+// -----
+
+llvm.comdat @__llvm_comdat {
+  llvm.comdat_selector @foo any
+}
+
+llvm.comdat @__llvm_comdat_1 {
+  // expected-error @below{{comdat selection symbols must be unique even in different comdat regions}}
+  llvm.comdat_selector @foo any
+}
diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir
index 5a56862817ac5..d23b6a7c289cc 100644
--- a/mlir/test/Target/LLVMIR/llvmir.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir.mlir
@@ -1,5 +1,21 @@
 // RUN: mlir-translate -mlir-to-llvmir -split-input-file %s | FileCheck %s
 
+
+// Comdat sections
+llvm.comdat @__llvm_comdat {
+  // CHECK: $any = comdat any
+  llvm.comdat_selector @any any
+  // CHECK: $exactmatch = comdat exactmatch
+  llvm.comdat_selector @exactmatch exactmatch
+  // CHECK: $largest = comdat largest
+  llvm.comdat_selector @largest largest
+  // CHECK: $nodeduplicate = comdat nodeduplicate
+  llvm.comdat_selector @nodeduplicate nodeduplicate
+  // CHECK: $samesize = comdat samesize
+  llvm.comdat_selector @samesize samesize
+}
+
+
 // CHECK: @global_aligned32 = private global i64 42, align 32
 "llvm.mlir.global"() ({}) {sym_name = "global_aligned32", global_type = i64, value = 42 : i64, linkage = #llvm.linkage<private>, alignment = 32} : () -> ()
 
@@ -153,6 +169,20 @@ llvm.mlir.global thread_local @has_thr_local(42 : i64) : i64
 // CHECK: @sectionvar = internal constant [10 x i8] c"teststring", section ".mysection"
 llvm.mlir.global internal constant @sectionvar("teststring")  {section = ".mysection"}: !llvm.array<10 x i8>
 
+//
+// Comdat attribute.
+//
+// CHECK: @has_any_comdat = internal constant i64 1, comdat($any)
+llvm.mlir.global internal constant @has_any_comdat(1 : i64) comdat(@__llvm_comdat::@any) : i64
+// CHECK: @has_exactmatch_comdat = internal constant i64 1, comdat($exactmatch)
+llvm.mlir.global internal constant @has_exactmatch_comdat(1 : i64) comdat(@__llvm_comdat::@exactmatch) : i64
+// CHECK: @has_largest_comdat = internal constant i64 1, comdat($largest)
+llvm.mlir.global internal constant @has_largest_comdat(1 : i64) comdat(@__llvm_comdat::@largest) : i64
+// CHECK: @has_nodeduplicate_comdat = internal constant i64 1, comdat($nodeduplicate)
+llvm.mlir.global internal constant @has_nodeduplicate_comdat(1 : i64) comdat(@__llvm_comdat::@nodeduplicate) : i64
+// CHECK: @has_samesize_comdat = internal constant i64 1, comdat($samesize)
+llvm.mlir.global internal constant @has_samesize_comdat(1 : i64) comdat(@__llvm_comdat::@samesize) : i64
+
 //
 // Declarations of the allocation functions to be linked against. These are
 // inserted before other functions in the module.

From d46d9689f7c3fe16010cd07761858a6392239b3f Mon Sep 17 00:00:00 2001
From: Dmitry Makogon <d.makogon@g.nsu.ru>
Date: Thu, 8 Jun 2023 18:56:18 +0700
Subject: [PATCH 022/130] [BBUtils] Don't add 'then' block to a loop if it's
 terminated with unreachable

SplitBlockAndInsertIfThen utility creates two new blocks,
they're called ThenBlock and Tail (true and false destinations of a conditional
branch correspondingly). The function has a bool parameter Unreachable,
and if it's set, then ThenBlock is terminated with an unreachable.
At the end of the function the new blocks are added to the loop of the split
block. However, in case ThenBlock is terminated with an unreachable,
it cannot belong to any loop.

Differential Revision: https://reviews.llvm.org/D152434
---
 llvm/lib/Transforms/Utils/BasicBlockUtils.cpp                 | 4 +++-
 llvm/test/Transforms/SimpleLoopUnswitch/guards.ll             | 3 ---
 .../nontrivial-unswitch-skip-selects-in-guards.ll             | 3 ---
 3 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
index 4b93b624c2eb0..68ff16f70a5d5 100644
--- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
+++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp
@@ -1518,7 +1518,9 @@ Instruction *llvm::SplitBlockAndInsertIfThen(Value *Cond,
 
   if (LI) {
     if (Loop *L = LI->getLoopFor(Head)) {
-      L->addBasicBlockToLoop(ThenBlock, *LI);
+      // unreachable-terminated blocks cannot belong to any loop.
+      if (!Unreachable)
+        L->addBasicBlockToLoop(ThenBlock, *LI);
       L->addBasicBlockToLoop(Tail, *LI);
     }
   }
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll b/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll
index 52580382d1d55..07668616ff86d 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/guards.ll
@@ -2,9 +2,6 @@
 ; RUN: opt -passes='simple-loop-unswitch<nontrivial>' -simple-loop-unswitch-guards -S < %s | FileCheck %s
 ; RUN: opt -passes='loop-mssa(simple-loop-unswitch<nontrivial>),verify<loops>' -simple-loop-unswitch-guards  -verify-memoryssa -verify-loop-info -S < %s | FileCheck %s
 
-; XFAIL: *
-; REQUIRES: asserts
-
 declare void @llvm.experimental.guard(i1, ...)
 
 define void @test_simple_case(i1 %cond, i32 %N) {
diff --git a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-skip-selects-in-guards.ll b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-skip-selects-in-guards.ll
index e7bf39dd74e42..543ee092e810c 100644
--- a/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-skip-selects-in-guards.ll
+++ b/llvm/test/Transforms/SimpleLoopUnswitch/nontrivial-unswitch-skip-selects-in-guards.ll
@@ -7,9 +7,6 @@ declare ptr @pluto()
 declare void @llvm.experimental.guard(i1, ...)
 declare void @widget()
 
-; XFAIL: *
-; REQUIRES: asserts
-
 define void @foo(ptr addrspace(1) %arg, i64 %arg1) personality ptr @pluto {
 ; CHECK-LABEL: @foo(
 ; CHECK-NEXT:  bb:

From 028cf8c016b21961bd229f24ddffcad1cb95aa7a Mon Sep 17 00:00:00 2001
From: Adrian Munera <adrian.munera@bsc.es>
Date: Thu, 15 Jun 2023 14:27:01 -0500
Subject: [PATCH 023/130] [OpenMP] Implement printing TDGs to dot files

This patch implements the "__kmp_print_tdg_dot" function, that prints a task dependency graph into a dot file containing the tasks and their dependencies.

It is activated through a new environment variable "KMP_TDG_DOT"

Reviewed By: tianshilei1992

Differential Revision: https://reviews.llvm.org/D150962
---
 openmp/runtime/src/kmp.h                      |  1 +
 openmp/runtime/src/kmp_global.cpp             |  1 +
 openmp/runtime/src/kmp_settings.cpp           | 11 +++
 openmp/runtime/src/kmp_tasking.cpp            | 36 +++++++++
 .../tasking/omp_record_replay_print_dot.cpp   | 80 +++++++++++++++++++
 5 files changed, 129 insertions(+)
 create mode 100644 openmp/runtime/test/tasking/omp_record_replay_print_dot.cpp

diff --git a/openmp/runtime/src/kmp.h b/openmp/runtime/src/kmp.h
index 251a5f0ce76fd..f50fb01f793f0 100644
--- a/openmp/runtime/src/kmp.h
+++ b/openmp/runtime/src/kmp.h
@@ -2535,6 +2535,7 @@ typedef struct kmp_tdg_info {
   kmp_int32 rec_num_taskred;
 } kmp_tdg_info_t;
 
+extern int __kmp_tdg_dot;
 extern kmp_int32 __kmp_max_tdgs;
 extern kmp_tdg_info_t **__kmp_global_tdgs;
 extern kmp_int32 __kmp_curr_tdg_idx;
diff --git a/openmp/runtime/src/kmp_global.cpp b/openmp/runtime/src/kmp_global.cpp
index 9557c519f835a..4ce0691abf8d6 100644
--- a/openmp/runtime/src/kmp_global.cpp
+++ b/openmp/runtime/src/kmp_global.cpp
@@ -559,6 +559,7 @@ int *__kmp_nesting_nth_level;
 
 #if OMPX_TASKGRAPH
 // TDG record & replay
+int __kmp_tdg_dot = 0;
 kmp_int32 __kmp_max_tdgs = 100;
 kmp_tdg_info_t **__kmp_global_tdgs = NULL;
 kmp_int32 __kmp_curr_tdg_idx =
diff --git a/openmp/runtime/src/kmp_settings.cpp b/openmp/runtime/src/kmp_settings.cpp
index 7092d6eb9104c..181db20a5f58d 100644
--- a/openmp/runtime/src/kmp_settings.cpp
+++ b/openmp/runtime/src/kmp_settings.cpp
@@ -1267,6 +1267,16 @@ static void __kmp_std_print_max_tdgs(kmp_str_buf_t *buffer, char const *name,
                                      void *data) {
   __kmp_stg_print_int(buffer, name, __kmp_max_tdgs);
 } // __kmp_std_print_max_tdgs
+
+static void __kmp_stg_parse_tdg_dot(char const *name, char const *value,
+                                   void *data) {
+  __kmp_stg_parse_bool(name, value, &__kmp_tdg_dot);
+} // __kmp_stg_parse_tdg_dot
+
+static void __kmp_stg_print_tdg_dot(kmp_str_buf_t *buffer, char const *name,
+                                   void *data) {
+  __kmp_stg_print_bool(buffer, name, __kmp_tdg_dot);
+} // __kmp_stg_print_tdg_dot
 #endif
 
 static void __kmp_stg_parse_num_hidden_helper_threads(char const *name,
@@ -5624,6 +5634,7 @@ static kmp_setting_t __kmp_stg_table[] = {
 #if OMPX_TASKGRAPH
     {"KMP_MAX_TDGS", __kmp_stg_parse_max_tdgs, __kmp_std_print_max_tdgs, NULL,
      0, 0},
+    {"KMP_TDG_DOT", __kmp_stg_parse_tdg_dot, __kmp_stg_print_tdg_dot, NULL, 0, 0},
 #endif
 
 #if OMPT_SUPPORT
diff --git a/openmp/runtime/src/kmp_tasking.cpp b/openmp/runtime/src/kmp_tasking.cpp
index ee452440176bf..fefa609927e89 100644
--- a/openmp/runtime/src/kmp_tasking.cpp
+++ b/openmp/runtime/src/kmp_tasking.cpp
@@ -5467,6 +5467,39 @@ static kmp_tdg_info_t *__kmp_find_tdg(kmp_int32 tdg_id) {
   return res;
 }
 
+// __kmp_print_tdg_dot: prints the TDG to a dot file
+// tdg:    ID of the TDG
+void __kmp_print_tdg_dot(kmp_tdg_info_t *tdg) {
+  kmp_int32 tdg_id = tdg->tdg_id;
+  KA_TRACE(10, ("__kmp_print_tdg_dot(enter): T#%d tdg_id=%d \n", gtid, tdg_id));
+
+  char file_name[20];
+  sprintf(file_name, "tdg_%d.dot", tdg_id);
+  kmp_safe_raii_file_t tdg_file(file_name, "w");
+
+  kmp_int32 num_tasks = KMP_ATOMIC_LD_RLX(&tdg->num_tasks);
+  fprintf(tdg_file,
+          "digraph TDG {\n"
+          "   compound=true\n"
+          "   subgraph cluster {\n"
+          "      label=TDG_%d\n",
+          tdg_id);
+  for (kmp_int32 i = 0; i < num_tasks; i++) {
+    fprintf(tdg_file, "      %d[style=bold]\n", i);
+  }
+  fprintf(tdg_file, "   }\n");
+  for (kmp_int32 i = 0; i < num_tasks; i++) {
+    kmp_int32 nsuccessors = tdg->record_map[i].nsuccessors;
+    kmp_int32 *successors = tdg->record_map[i].successors;
+    if (nsuccessors > 0) {
+      for (kmp_int32 j = 0; j < nsuccessors; j++)
+        fprintf(tdg_file, "   %d -> %d \n", i, successors[j]);
+    }
+  }
+  fprintf(tdg_file, "}");
+  KA_TRACE(10, ("__kmp_print_tdg_dot(exit): T#%d tdg_id=%d \n", gtid, tdg_id));
+}
+
 // __kmp_start_record: launch the execution of a previous
 // recorded TDG
 // gtid:   Global Thread ID
@@ -5636,6 +5669,9 @@ void __kmp_end_record(kmp_int32 gtid, kmp_tdg_info_t *tdg) {
                       this_record_map[i].npredecessors);
   }
   KMP_ATOMIC_ST_RLX(&__kmp_tdg_task_id, 0);
+
+  if (__kmp_tdg_dot)
+    __kmp_print_tdg_dot(tdg);
 }
 
 // __kmpc_end_record_task: wrapper around __kmp_end_record to mark
diff --git a/openmp/runtime/test/tasking/omp_record_replay_print_dot.cpp b/openmp/runtime/test/tasking/omp_record_replay_print_dot.cpp
new file mode 100644
index 0000000000000..2fe55f0815429
--- /dev/null
+++ b/openmp/runtime/test/tasking/omp_record_replay_print_dot.cpp
@@ -0,0 +1,80 @@
+// REQUIRES: ompx_taskgraph
+// RUN: %libomp-cxx-compile-and-run
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <cassert>
+
+// Compiler-generated code (emulation)
+typedef struct ident {
+    void* dummy;
+} ident_t;
+
+#ifdef __cplusplus
+extern "C" {
+  int __kmpc_global_thread_num(ident_t *);
+  int __kmpc_start_record_task(ident_t *, int, int, int);
+  void __kmpc_end_record_task(ident_t *, int, int , int);
+}
+#endif
+
+void func(int *num_exec) {
+  #pragma omp atomic
+  (*num_exec)++;
+}
+
+std::string tdg_string= "digraph TDG {\n"
+"   compound=true\n"
+"   subgraph cluster {\n"
+"      label=TDG_0\n"
+"      0[style=bold]\n"
+"      1[style=bold]\n"
+"      2[style=bold]\n"
+"      3[style=bold]\n"
+"   }\n"
+"   0 -> 1 \n"
+"   1 -> 2 \n"
+"   1 -> 3 \n"
+"}";
+
+int main() {
+  int num_exec = 0;
+  int x, y;
+
+  setenv("KMP_TDG_DOT","TRUE",1);
+  remove("tdg_0.dot");
+
+  #pragma omp parallel
+  #pragma omp single
+  {
+    int gtid = __kmpc_global_thread_num(nullptr);
+    int res = __kmpc_start_record_task(nullptr, gtid, /* kmp_tdg_flags */ 0, /* tdg_id */ 0);
+    if (res) {
+      #pragma omp task depend(out : x)
+      func(&num_exec);
+      #pragma omp task depend(in : x) depend(out : y)
+      func(&num_exec);
+      #pragma omp task depend(in : y)
+      func(&num_exec);
+      #pragma omp task depend(in : y)
+      func(&num_exec);
+    }
+
+    __kmpc_end_record_task(nullptr, gtid, /* kmp_tdg_flags */ 0, /* tdg_id */ 0);
+  }
+
+  assert(num_exec == 4);
+
+  std::ifstream tdg_file("tdg_0.dot");
+  assert(tdg_file.is_open());
+
+  std::stringstream tdg_file_stream;
+  tdg_file_stream << tdg_file.rdbuf();
+  int equal = tdg_string.compare(tdg_file_stream.str());
+
+  assert(equal == 0);
+
+  std::cout << "Passed" << std::endl;
+  return 0;
+}
+// CHECK: Passed

From ce9d3f09b4b2a2ae229c3e1f56f27cd31bc4660c Mon Sep 17 00:00:00 2001
From: Alexandros Lamprineas <alexandros.lamprineas@arm.com>
Date: Mon, 19 Jun 2023 11:08:03 +0100
Subject: [PATCH 024/130] [FuncSpec] Promote stack values before
 specialization.

After each iteration of the function specializer, constant stack values
are promoted to constant globals in order to enable recursive function
specialization. This should also be done once before running the
specializer. Enables specialization of _QMbrute_forcePdigits_2 from
SPEC2017:548.exchange2_r.

Differential Revision: https://reviews.llvm.org/D152799
---
 .../Transforms/IPO/FunctionSpecialization.h   |  13 +-
 .../Transforms/IPO/FunctionSpecialization.cpp | 128 ++++++++----------
 .../function-specialization-recursive.ll      |  59 --------
 .../promoteContantStackValues.ll              |  95 +++++++++++++
 4 files changed, 152 insertions(+), 143 deletions(-)
 delete mode 100644 llvm/test/Transforms/FunctionSpecialization/function-specialization-recursive.ll
 create mode 100644 llvm/test/Transforms/FunctionSpecialization/promoteContantStackValues.ll

diff --git a/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h b/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h
index e24d72fae10fb..f780385f7f67d 100644
--- a/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h
+++ b/llvm/include/llvm/Transforms/IPO/FunctionSpecialization.h
@@ -206,10 +206,9 @@ class FunctionSpecializer {
   /// is a function argument.
   Constant *getConstantStackValue(CallInst *Call, Value *Val);
 
-  /// Iterate over the argument tracked functions see if there
-  /// are any new constant values for the call instruction via
-  /// stack variables.
-  void promoteConstantStackValues();
+  /// See if there are any new constant values for the callers of \p F via
+  /// stack variables and promote them to global variables.
+  void promoteConstantStackValues(Function *F);
 
   /// Clean up fully specialized functions.
   void removeDeadFunctions();
@@ -217,9 +216,6 @@ class FunctionSpecializer {
   /// Remove any ssa_copy intrinsics that may have been introduced.
   void cleanUpSSA();
 
-  // Compute the code metrics for function \p F.
-  CodeMetrics &analyzeFunction(Function *F);
-
   /// @brief  Find potential specialization opportunities.
   /// @param F Function to specialize
   /// @param SpecCost Cost of specializing a function. Final score is benefit
@@ -238,9 +234,6 @@ class FunctionSpecializer {
   /// @return The new, cloned function
   Function *createSpecialization(Function *F, const SpecSig &S);
 
-  /// Compute and return the cost of specializing function \p F.
-  Cost getSpecializationCost(Function *F);
-
   /// Determine if it is possible to specialise the function for constant values
   /// of the formal parameter \p A.
   bool isArgumentInteresting(Argument *A);
diff --git a/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp b/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
index 1792b24936bf8..93d2eaaddf310 100644
--- a/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
+++ b/llvm/lib/Transforms/IPO/FunctionSpecialization.cpp
@@ -397,49 +397,37 @@ Constant *FunctionSpecializer::getConstantStackValue(CallInst *Call,
 //       ret void
 //     }
 //
-void FunctionSpecializer::promoteConstantStackValues() {
-  // Iterate over the argument tracked functions see if there
-  // are any new constant values for the call instruction via
-  // stack variables.
-  for (Function &F : M) {
-    if (!Solver.isArgumentTrackedFunction(&F))
+// See if there are any new constant values for the callers of \p F via
+// stack variables and promote them to global variables.
+void FunctionSpecializer::promoteConstantStackValues(Function *F) {
+  for (User *U : F->users()) {
+
+    auto *Call = dyn_cast<CallInst>(U);
+    if (!Call)
       continue;
 
-    for (auto *User : F.users()) {
+    if (!Solver.isBlockExecutable(Call->getParent()))
+      continue;
 
-      auto *Call = dyn_cast<CallInst>(User);
-      if (!Call)
-        continue;
+    for (const Use &U : Call->args()) {
+      unsigned Idx = Call->getArgOperandNo(&U);
+      Value *ArgOp = Call->getArgOperand(Idx);
+      Type *ArgOpType = ArgOp->getType();
 
-      if (!Solver.isBlockExecutable(Call->getParent()))
+      if (!Call->onlyReadsMemory(Idx) || !ArgOpType->isPointerTy())
         continue;
 
-      bool Changed = false;
-      for (const Use &U : Call->args()) {
-        unsigned Idx = Call->getArgOperandNo(&U);
-        Value *ArgOp = Call->getArgOperand(Idx);
-        Type *ArgOpType = ArgOp->getType();
-
-        if (!Call->onlyReadsMemory(Idx) || !ArgOpType->isPointerTy())
-          continue;
-
-        auto *ConstVal = getConstantStackValue(Call, ArgOp);
-        if (!ConstVal)
-          continue;
-
-        Value *GV = new GlobalVariable(M, ConstVal->getType(), true,
-                                       GlobalValue::InternalLinkage, ConstVal,
-                                       "funcspec.arg");
-        if (ArgOpType != ConstVal->getType())
-          GV = ConstantExpr::getBitCast(cast<Constant>(GV), ArgOpType);
+      auto *ConstVal = getConstantStackValue(Call, ArgOp);
+      if (!ConstVal)
+        continue;
 
-        Call->setArgOperand(Idx, GV);
-        Changed = true;
-      }
+      Value *GV = new GlobalVariable(M, ConstVal->getType(), true,
+                                     GlobalValue::InternalLinkage, ConstVal,
+                                     "funcspec.arg");
+      if (ArgOpType != ConstVal->getType())
+        GV = ConstantExpr::getBitCast(cast<Constant>(GV), ArgOpType);
 
-      // Add the changed CallInst to Solver Worklist
-      if (Changed)
-        Solver.visitCall(*Call);
+      Call->setArgOperand(Idx, GV);
     }
   }
 }
@@ -504,17 +492,37 @@ bool FunctionSpecializer::run() {
     if (!isCandidateFunction(&F))
       continue;
 
-    Cost SpecCost = getSpecializationCost(&F);
-    if (!SpecCost.isValid()) {
-      LLVM_DEBUG(dbgs() << "FnSpecialization: Invalid specialization cost for "
-                        << F.getName() << "\n");
-      continue;
+    auto [It, Inserted] = FunctionMetrics.try_emplace(&F);
+    CodeMetrics &Metrics = It->second;
+    //Analyze the function.
+    if (Inserted) {
+      SmallPtrSet<const Value *, 32> EphValues;
+      CodeMetrics::collectEphemeralValues(&F, &GetAC(F), EphValues);
+      for (BasicBlock &BB : F)
+        Metrics.analyzeBasicBlock(&BB, GetTTI(F), EphValues);
     }
 
+    // If the code metrics reveal that we shouldn't duplicate the function,
+    // or if the code size implies that this function is easy to get inlined,
+    // then we shouldn't specialize it.
+    if (Metrics.notDuplicatable || !Metrics.NumInsts.isValid() ||
+        (!ForceSpecialization && !F.hasFnAttribute(Attribute::NoInline) &&
+         Metrics.NumInsts < MinFunctionSize))
+      continue;
+
+    // TODO: For now only consider recursive functions when running multiple
+    // times. This should change if specialization on literal constants gets
+    // enabled.
+    if (!Inserted && !Metrics.isRecursive && !SpecializeLiteralConstant)
+      continue;
+
     LLVM_DEBUG(dbgs() << "FnSpecialization: Specialization cost for "
-                      << F.getName() << " is " << SpecCost << "\n");
+                      << F.getName() << " is " << Metrics.NumInsts << "\n");
+
+    if (Inserted && Metrics.isRecursive)
+      promoteConstantStackValues(&F);
 
-    if (!findSpecializations(&F, SpecCost, AllSpecs, SM)) {
+    if (!findSpecializations(&F, Metrics.NumInsts, AllSpecs, SM)) {
       LLVM_DEBUG(
           dbgs() << "FnSpecialization: No possible specializations found for "
                  << F.getName() << "\n");
@@ -622,7 +630,10 @@ bool FunctionSpecializer::run() {
   // Rerun the solver to notify the users of the modified callsites.
   Solver.solveWhileResolvedUndefs();
 
-  promoteConstantStackValues();
+  for (Function *F : OriginalFuncs)
+    if (FunctionMetrics[F].isRecursive)
+      promoteConstantStackValues(F);
+
   return true;
 }
 
@@ -637,20 +648,6 @@ void FunctionSpecializer::removeDeadFunctions() {
   FullySpecialized.clear();
 }
 
-// Compute the code metrics for function \p F.
-CodeMetrics &FunctionSpecializer::analyzeFunction(Function *F) {
-  auto I = FunctionMetrics.insert({F, CodeMetrics()});
-  CodeMetrics &Metrics = I.first->second;
-  if (I.second) {
-    // The code metrics were not cached.
-    SmallPtrSet<const Value *, 32> EphValues;
-    CodeMetrics::collectEphemeralValues(F, &(GetAC)(*F), EphValues);
-    for (BasicBlock &BB : *F)
-      Metrics.analyzeBasicBlock(&BB, (GetTTI)(*F), EphValues);
-  }
-  return Metrics;
-}
-
 /// Clone the function \p F and remove the ssa_copy intrinsics added by
 /// the SCCPSolver in the cloned version.
 static Function *cloneCandidateFunction(Function *F) {
@@ -802,23 +799,6 @@ Function *FunctionSpecializer::createSpecialization(Function *F,
   return Clone;
 }
 
-/// Compute and return the cost of specializing function \p F.
-Cost FunctionSpecializer::getSpecializationCost(Function *F) {
-  CodeMetrics &Metrics = analyzeFunction(F);
-  // If the code metrics reveal that we shouldn't duplicate the function, we
-  // shouldn't specialize it. Set the specialization cost to Invalid.
-  // Or if the lines of codes implies that this function is easy to get
-  // inlined so that we shouldn't specialize it.
-  if (Metrics.notDuplicatable || !Metrics.NumInsts.isValid() ||
-      (!ForceSpecialization && !F->hasFnAttribute(Attribute::NoInline) &&
-       Metrics.NumInsts < MinFunctionSize))
-    return InstructionCost::getInvalid();
-
-  // Otherwise, set the specialization cost to be the cost of all the
-  // instructions in the function.
-  return Metrics.NumInsts;
-}
-
 /// Compute a bonus for replacing argument \p A with constant \p C.
 Cost FunctionSpecializer::getSpecializationBonus(Argument *A, Constant *C,
                                                  InstCostVisitor &Visitor) {
diff --git a/llvm/test/Transforms/FunctionSpecialization/function-specialization-recursive.ll b/llvm/test/Transforms/FunctionSpecialization/function-specialization-recursive.ll
deleted file mode 100644
index 2cee26a911a47..0000000000000
--- a/llvm/test/Transforms/FunctionSpecialization/function-specialization-recursive.ll
+++ /dev/null
@@ -1,59 +0,0 @@
-; RUN: opt -passes="ipsccp<func-spec>,inline,instcombine" -force-specialization -funcspec-max-iters=2 -S < %s | FileCheck %s --check-prefix=ITERS2
-; RUN: opt -passes="ipsccp<func-spec>,inline,instcombine" -force-specialization -funcspec-max-iters=3 -S < %s | FileCheck %s --check-prefix=ITERS3
-; RUN: opt -passes="ipsccp<func-spec>,inline,instcombine" -force-specialization -funcspec-max-iters=4 -S < %s | FileCheck %s --check-prefix=ITERS4
-
-@low = internal constant i32 0, align 4
-@high = internal constant i32 6, align 4
-
-define internal void @recursiveFunc(ptr nocapture readonly %lo, i32 %step, ptr nocapture readonly %hi) {
-  %lo.temp = alloca i32, align 4
-  %hi.temp = alloca i32, align 4
-  %lo.load = load i32, ptr %lo, align 4
-  %hi.load = load i32, ptr %hi, align 4
-  %cmp = icmp ne i32 %lo.load, %hi.load
-  br i1 %cmp, label %block6, label %ret.block
-
-block6:
-  call void @print_val(i32 %lo.load, i32 %hi.load)
-  %add = add nsw i32 %lo.load, %step
-  %sub = sub nsw i32 %hi.load, %step
-  store i32 %add, ptr %lo.temp, align 4
-  store i32 %sub, ptr %hi.temp, align 4
-  call void @recursiveFunc(ptr nonnull %lo.temp, i32 %step, ptr nonnull %hi.temp)
-  br label %ret.block
-
-ret.block:
-  ret void
-}
-
-; ITERS2:  @funcspec.arg.4 = internal constant i32 2
-; ITERS2:  @funcspec.arg.5 = internal constant i32 4
-
-; ITERS3:  @funcspec.arg.7 = internal constant i32 3
-; ITERS3:  @funcspec.arg.8 = internal constant i32 3
-
-define i32 @main() {
-; ITERS2-LABEL: @main(
-; ITERS2-NEXT:    call void @print_val(i32 0, i32 6)
-; ITERS2-NEXT:    call void @print_val(i32 1, i32 5)
-; ITERS2-NEXT:    call void @recursiveFunc(ptr nonnull @funcspec.arg.4, i32 1, ptr nonnull @funcspec.arg.5)
-; ITERS2-NEXT:    ret i32 0
-;
-; ITERS3-LABEL: @main(
-; ITERS3-NEXT:    call void @print_val(i32 0, i32 6)
-; ITERS3-NEXT:    call void @print_val(i32 1, i32 5)
-; ITERS3-NEXT:    call void @print_val(i32 2, i32 4)
-; ITERS3-NEXT:    call void @recursiveFunc(ptr nonnull @funcspec.arg.7, i32 1, ptr nonnull @funcspec.arg.8)
-; ITERS3-NEXT:    ret i32 0
-;
-; ITERS4-LABEL: @main(
-; ITERS4-NEXT:    call void @print_val(i32 0, i32 6)
-; ITERS4-NEXT:    call void @print_val(i32 1, i32 5)
-; ITERS4-NEXT:    call void @print_val(i32 2, i32 4)
-; ITERS4-NEXT:    ret i32 0
-;
-  call void @recursiveFunc(ptr nonnull @low, i32 1, ptr nonnull @high)
-  ret i32 0
-}
-
-declare dso_local void @print_val(i32, i32)
diff --git a/llvm/test/Transforms/FunctionSpecialization/promoteContantStackValues.ll b/llvm/test/Transforms/FunctionSpecialization/promoteContantStackValues.ll
new file mode 100644
index 0000000000000..256cbebae0620
--- /dev/null
+++ b/llvm/test/Transforms/FunctionSpecialization/promoteContantStackValues.ll
@@ -0,0 +1,95 @@
+; RUN: opt -passes="ipsccp<func-spec>,inline,instcombine" -force-specialization -funcspec-max-iters=1 -S < %s | FileCheck %s --check-prefix=ITERS1
+; RUN: opt -passes="ipsccp<func-spec>,inline,instcombine" -force-specialization -funcspec-max-iters=2 -S < %s | FileCheck %s --check-prefix=ITERS2
+; RUN: opt -passes="ipsccp<func-spec>,inline,instcombine" -force-specialization -funcspec-max-iters=3 -S < %s | FileCheck %s --check-prefix=ITERS3
+; RUN: opt -passes="ipsccp<func-spec>,inline,instcombine" -force-specialization -funcspec-max-iters=4 -S < %s | FileCheck %s --check-prefix=ITERS4
+
+@low = internal constant i32 0, align 4
+@high = internal constant i32 6, align 4
+
+define internal void @recursiveFunc(ptr nocapture readonly %lo, i32 %step, ptr nocapture readonly %hi) {
+  %lo.temp = alloca i32, align 4
+  %hi.temp = alloca i32, align 4
+  %lo.load = load i32, ptr %lo, align 4
+  %hi.load = load i32, ptr %hi, align 4
+  %cmp = icmp ne i32 %lo.load, %hi.load
+  br i1 %cmp, label %block6, label %ret.block
+
+block6:
+  call void @print_val(i32 %lo.load, i32 %hi.load)
+  %add = add nsw i32 %lo.load, %step
+  %sub = sub nsw i32 %hi.load, %step
+  store i32 %add, ptr %lo.temp, align 4
+  store i32 %sub, ptr %hi.temp, align 4
+  call void @recursiveFunc(ptr nonnull %lo.temp, i32 %step, ptr nonnull %hi.temp)
+  br label %ret.block
+
+ret.block:
+  ret void
+}
+
+; ITERS1:  @funcspec.arg = internal constant i32 0
+; ITERS1:  @funcspec.arg.1 = internal constant i32 6
+; ITERS1:  @funcspec.arg.3 = internal constant i32 1
+; ITERS1:  @funcspec.arg.4 = internal constant i32 5
+
+; ITERS2:  @funcspec.arg = internal constant i32 0
+; ITERS2:  @funcspec.arg.1 = internal constant i32 6
+; ITERS2:  @funcspec.arg.3 = internal constant i32 1
+; ITERS2:  @funcspec.arg.4 = internal constant i32 5
+; ITERS2:  @funcspec.arg.6 = internal constant i32 2
+; ITERS2:  @funcspec.arg.7 = internal constant i32 4
+
+; ITERS3:  @funcspec.arg = internal constant i32 0
+; ITERS3:  @funcspec.arg.1 = internal constant i32 6
+; ITERS3:  @funcspec.arg.3 = internal constant i32 1
+; ITERS3:  @funcspec.arg.4 = internal constant i32 5
+; ITERS3:  @funcspec.arg.6 = internal constant i32 2
+; ITERS3:  @funcspec.arg.7 = internal constant i32 4
+; ITERS3:  @funcspec.arg.9 = internal constant i32 3
+; ITERS3:  @funcspec.arg.10 = internal constant i32 3
+
+; ITERS4:  @funcspec.arg = internal constant i32 0
+; ITERS4:  @funcspec.arg.1 = internal constant i32 6
+; ITERS4:  @funcspec.arg.3 = internal constant i32 1
+; ITERS4:  @funcspec.arg.4 = internal constant i32 5
+; ITERS4:  @funcspec.arg.6 = internal constant i32 2
+; ITERS4:  @funcspec.arg.7 = internal constant i32 4
+; ITERS4:  @funcspec.arg.9 = internal constant i32 3
+; ITERS4:  @funcspec.arg.10 = internal constant i32 3
+
+define i32 @main() {
+; ITERS1-LABEL: @main(
+; ITERS1-NEXT:    call void @print_val(i32 0, i32 6)
+; ITERS1-NEXT:    call void @recursiveFunc(ptr nonnull @funcspec.arg.3, i32 1, ptr nonnull @funcspec.arg.4)
+; ITERS1-NEXT:    ret i32 0
+;
+; ITERS2-LABEL: @main(
+; ITERS2-NEXT:    call void @print_val(i32 0, i32 6)
+; ITERS2-NEXT:    call void @print_val(i32 1, i32 5)
+; ITERS2-NEXT:    call void @recursiveFunc(ptr nonnull @funcspec.arg.6, i32 1, ptr nonnull @funcspec.arg.7)
+; ITERS2-NEXT:    ret i32 0
+;
+; ITERS3-LABEL: @main(
+; ITERS3-NEXT:    call void @print_val(i32 0, i32 6)
+; ITERS3-NEXT:    call void @print_val(i32 1, i32 5)
+; ITERS3-NEXT:    call void @print_val(i32 2, i32 4)
+; ITERS3-NEXT:    call void @recursiveFunc(ptr nonnull @funcspec.arg.9, i32 1, ptr nonnull @funcspec.arg.10)
+; ITERS3-NEXT:    ret i32 0
+;
+; ITERS4-LABEL: @main(
+; ITERS4-NEXT:    call void @print_val(i32 0, i32 6)
+; ITERS4-NEXT:    call void @print_val(i32 1, i32 5)
+; ITERS4-NEXT:    call void @print_val(i32 2, i32 4)
+; ITERS4-NEXT:    ret i32 0
+;
+  %lo.temp = alloca i32, align 4
+  %hi.temp = alloca i32, align 4
+  %lo.load = load i32, ptr @low, align 4
+  %hi.load = load i32, ptr @high, align 4
+  store i32 %lo.load, ptr %lo.temp, align 4
+  store i32 %hi.load, ptr %hi.temp, align 4
+  call void @recursiveFunc(ptr nonnull %lo.temp, i32 1, ptr nonnull %hi.temp)
+  ret i32 0
+}
+
+declare dso_local void @print_val(i32, i32)

From cbd9585ba2c9c50160075bdc9de20ccead286d60 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Mon, 19 Jun 2023 15:38:35 +0200
Subject: [PATCH 025/130] [CVP] Add test for PR63330 (NFC)

---
 .../urem-expansion.ll                          | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll b/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll
index 620cd5e0651ca..cc4644d83aaf5 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll
@@ -326,3 +326,21 @@ define i8 @large.divisor.with.overflow.v2.unbound.x(i8 %x) {
   %rem = urem i8 %x, 128
   ret i8 %rem
 }
+
+define i1 @icmp_after_expansion(i8 noundef %x) {
+; CHECK-LABEL: @icmp_after_expansion(
+; CHECK-NEXT:    [[CMP_X_UPPER:%.*]] = icmp ult i8 [[X:%.*]], 6
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP_X_UPPER]])
+; CHECK-NEXT:    [[X_FROZEN:%.*]] = freeze i8 [[X]]
+; CHECK-NEXT:    [[REM_UREM:%.*]] = sub nuw i8 [[X_FROZEN]], 3
+; CHECK-NEXT:    [[REM_CMP:%.*]] = icmp ult i8 [[X_FROZEN]], 3
+; CHECK-NEXT:    [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X_FROZEN]], i8 [[REM_UREM]]
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[REM]], 3
+; CHECK-NEXT:    ret i1 [[CMP]]
+;
+  %cmp.x.upper = icmp ult i8 %x, 6
+  call void @llvm.assume(i1 %cmp.x.upper)
+  %rem = urem i8 %x, 3
+  %cmp = icmp eq i8 %rem, 3
+  ret i1 %cmp
+}

From 90b55f2bba365c1e83c3be60583b449e07f846f3 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Mon, 19 Jun 2023 15:41:39 +0200
Subject: [PATCH 026/130] [CVP] Don't freeze value if guaranteed non-undef

Avoid inserting the freeze if not necessary, as this allows LVI
to continue reasoning about the expression.
---
 .../Transforms/Scalar/CorrelatedValuePropagation.cpp   |  4 +++-
 .../CorrelatedValuePropagation/cond-at-use.ll          |  7 +++----
 .../CorrelatedValuePropagation/urem-expansion.ll       | 10 ++++------
 3 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
index 9f503fb43bc35..488297110d6fa 100644
--- a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
+++ b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
@@ -766,7 +766,9 @@ static bool expandUDivOrURem(BinaryOperator *Instr, const ConstantRange &XCR,
   if (IsRem) {
     // NOTE: this transformation introduces two uses of X,
     //       but it may be undef so we must freeze it first.
-    Value *FrozenX = B.CreateFreeze(X, X->getName() + ".frozen");
+    Value *FrozenX = X;
+    if (!isGuaranteedNotToBeUndefOrPoison(X))
+      FrozenX = B.CreateFreeze(X, X->getName() + ".frozen");
     auto *AdjX = B.CreateNUWSub(FrozenX, Y, Instr->getName() + ".urem");
     auto *Cmp =
         B.CreateICmp(ICmpInst::ICMP_ULT, FrozenX, Y, Instr->getName() + ".cmp");
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/cond-at-use.ll b/llvm/test/Transforms/CorrelatedValuePropagation/cond-at-use.ll
index dbb63203cc945..8e5ec878bb894 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/cond-at-use.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/cond-at-use.ll
@@ -346,10 +346,9 @@ define i16 @urem_elide(i16 noundef %x) {
 
 define i16 @urem_expand(i16 noundef %x) {
 ; CHECK-LABEL: @urem_expand(
-; CHECK-NEXT:    [[X_FROZEN:%.*]] = freeze i16 [[X:%.*]]
-; CHECK-NEXT:    [[UREM_UREM:%.*]] = sub nuw i16 [[X_FROZEN]], 42
-; CHECK-NEXT:    [[UREM_CMP:%.*]] = icmp ult i16 [[X_FROZEN]], 42
-; CHECK-NEXT:    [[UREM:%.*]] = select i1 [[UREM_CMP]], i16 [[X_FROZEN]], i16 [[UREM_UREM]]
+; CHECK-NEXT:    [[UREM_UREM:%.*]] = sub nuw i16 [[X:%.*]], 42
+; CHECK-NEXT:    [[UREM_CMP:%.*]] = icmp ult i16 [[X]], 42
+; CHECK-NEXT:    [[UREM:%.*]] = select i1 [[UREM_CMP]], i16 [[X]], i16 [[UREM_UREM]]
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp ult i16 [[X]], 84
 ; CHECK-NEXT:    [[SEL:%.*]] = select i1 [[CMP]], i16 [[UREM]], i16 24
 ; CHECK-NEXT:    ret i16 [[SEL]]
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll b/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll
index cc4644d83aaf5..2f1ca2483f67d 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll
@@ -331,12 +331,10 @@ define i1 @icmp_after_expansion(i8 noundef %x) {
 ; CHECK-LABEL: @icmp_after_expansion(
 ; CHECK-NEXT:    [[CMP_X_UPPER:%.*]] = icmp ult i8 [[X:%.*]], 6
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP_X_UPPER]])
-; CHECK-NEXT:    [[X_FROZEN:%.*]] = freeze i8 [[X]]
-; CHECK-NEXT:    [[REM_UREM:%.*]] = sub nuw i8 [[X_FROZEN]], 3
-; CHECK-NEXT:    [[REM_CMP:%.*]] = icmp ult i8 [[X_FROZEN]], 3
-; CHECK-NEXT:    [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X_FROZEN]], i8 [[REM_UREM]]
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i8 [[REM]], 3
-; CHECK-NEXT:    ret i1 [[CMP]]
+; CHECK-NEXT:    [[REM_UREM:%.*]] = sub nuw i8 [[X]], 3
+; CHECK-NEXT:    [[REM_CMP:%.*]] = icmp ult i8 [[X]], 3
+; CHECK-NEXT:    [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X]], i8 [[REM_UREM]]
+; CHECK-NEXT:    ret i1 false
 ;
   %cmp.x.upper = icmp ult i8 %x, 6
   call void @llvm.assume(i1 %cmp.x.upper)

From 3956a34e4fc6994021e8f3cbcb81a02161ce15e0 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Wed, 14 Jun 2023 15:36:37 -0700
Subject: [PATCH 027/130] [libc++] Move non operator new definitions outside of
 new.cpp

This makes it such that new.cpp contains only the definitions of
operator new and operator delete, like its libc++abi counterpart.

Differential Revision: https://reviews.llvm.org/D153136
---
 libcxx/src/CMakeLists.txt  |  1 +
 libcxx/src/new.cpp         | 23 -----------------------
 libcxx/src/new_helpers.cpp | 30 ++++++++++++++++++++++++++++++
 3 files changed, 31 insertions(+), 23 deletions(-)
 create mode 100644 libcxx/src/new_helpers.cpp

diff --git a/libcxx/src/CMakeLists.txt b/libcxx/src/CMakeLists.txt
index dede38652e2f9..5280436108e7f 100644
--- a/libcxx/src/CMakeLists.txt
+++ b/libcxx/src/CMakeLists.txt
@@ -36,6 +36,7 @@ set(LIBCXX_SOURCES
   mutex_destructor.cpp
   new.cpp
   new_handler.cpp
+  new_helpers.cpp
   optional.cpp
   random_shuffle.cpp
   ryu/d2fixed.cpp
diff --git a/libcxx/src/new.cpp b/libcxx/src/new.cpp
index b8d24e73f0812..a9920ba09849b 100644
--- a/libcxx/src/new.cpp
+++ b/libcxx/src/new.cpp
@@ -10,29 +10,6 @@
 #include <cstdlib>
 #include <new>
 
-namespace std
-{
-
-#ifndef __GLIBCXX__
-const nothrow_t nothrow{};
-#endif
-
-#ifndef LIBSTDCXX
-
-void
-__throw_bad_alloc()
-{
-#ifndef _LIBCPP_HAS_NO_EXCEPTIONS
-    throw bad_alloc();
-#else
-    _VSTD::abort();
-#endif
-}
-
-#endif // !LIBSTDCXX
-
-}  // std
-
 #if !defined(__GLIBCXX__) &&                                                   \
     !defined(_LIBCPP_ABI_VCRUNTIME) &&      \
     !defined(_LIBCPP_DISABLE_NEW_DELETE_DEFINITIONS)
diff --git a/libcxx/src/new_helpers.cpp b/libcxx/src/new_helpers.cpp
new file mode 100644
index 0000000000000..45a0a661db0e4
--- /dev/null
+++ b/libcxx/src/new_helpers.cpp
@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include <cstdlib>
+#include <new>
+
+namespace std { // purposefully not versioned
+
+#ifndef __GLIBCXX__
+const nothrow_t nothrow{};
+#endif
+
+#ifndef LIBSTDCXX
+
+void __throw_bad_alloc() {
+#  ifndef _LIBCPP_HAS_NO_EXCEPTIONS
+  throw bad_alloc();
+#  else
+  std::abort();
+#  endif
+}
+
+#endif // !LIBSTDCXX
+
+} // namespace std

From aabea3d320c87561fe98b56c9f53cca1c6d18869 Mon Sep 17 00:00:00 2001
From: Nicolas Vasilache <nicolasvasilache@users.noreply.github.com>
Date: Mon, 19 Jun 2023 13:26:00 +0000
Subject: [PATCH 028/130] [mlir][Vector] Let VectorToLLVM operate on
 non-ModuleOp

Restriction to ModuleOp is ancient and unnecessarily restrictive.
---
 mlir/include/mlir/Conversion/Passes.td | 14 +-------------
 1 file changed, 1 insertion(+), 13 deletions(-)

diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index ac53af4709098..1e16d7b1dbdf8 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -361,10 +361,6 @@ def GpuToLLVMConversionPass : Pass<"gpu-to-llvm", "ModuleOp"> {
   }];
 
   let options = [
-    Option<"hostBarePtrCallConv", "use-bare-pointers-for-host", "bool",
-           /*default=*/"false",
-           "Use bare pointers to pass memref arguments to host functions. "
-           "All memrefs must have static shape.">,
     Option<"kernelBarePtrCallConv", "use-bare-pointers-for-kernels", "bool",
            /*default=*/"false",
              "Use bare pointers to pass memref arguments to kernels. "
@@ -426,10 +422,6 @@ def ConvertGpuOpsToNVVMOps : Pass<"convert-gpu-to-nvvm", "gpu::GPUModuleOp"> {
            "Bitwidth of the index type, 0 to use size of machine word">,
     Option<"hasRedux", "has-redux", "bool", /*default=*/"false",
            "Target gpu supports redux">,
-    Option<"useBarePtrCallConv", "use-bare-ptr-memref-call-conv", "bool",
-           /*default=*/"false",
-           "Replace memref arguments in GPU functions with bare pointers. "
-           "All memrefs must have static shape.">,
     Option<"useOpaquePointers", "use-opaque-pointers", "bool",
                    /*default=*/"true", "Generate LLVM IR using opaque pointers "
                    "instead of typed pointers">,
@@ -1065,7 +1057,7 @@ def ConvertVectorToSCF : Pass<"convert-vector-to-scf"> {
 // VectorToLLVM
 //===----------------------------------------------------------------------===//
 
-def ConvertVectorToLLVMPass : Pass<"convert-vector-to-llvm", "ModuleOp"> {
+def ConvertVectorToLLVMPass : Pass<"convert-vector-to-llvm"> {
   let summary = "Lower the operations from the vector dialect into the LLVM "
                 "dialect";
   let description = [{
@@ -1100,10 +1092,6 @@ def ConvertVectorToLLVMPass : Pass<"convert-vector-to-llvm", "ModuleOp"> {
            "bool", /*default=*/"false",
            "Enables the use of ArmSVE dialect while lowering the vector "
        "dialect.">,
-    Option<"armSME", "enable-arm-sme",
-           "bool", /*default=*/"false",
-           "Enables the use of ArmSME dialect while lowering the vector "
-       "dialect.">,
     Option<"x86Vector", "enable-x86vector",
            "bool", /*default=*/"false",
            "Enables the use of X86Vector dialect while lowering the vector "

From 8c80d01a95ba0d75c29191de0ea38cce48c9978f Mon Sep 17 00:00:00 2001
From: Nicolas Vasilache <nicolasvasilache@users.noreply.github.com>
Date: Mon, 19 Jun 2023 13:33:30 +0000
Subject: [PATCH 029/130] [mlir][NVGPU] NFC - Add a more convenient C++ builder
 for nvgpu::MmaSyncOp

---
 mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td | 10 +++++++---
 mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp  |  9 +++++++++
 2 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td
index 5bb02b082575a..e595e9dffbe0b 100644
--- a/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td
+++ b/mlir/include/mlir/Dialect/NVGPU/IR/NVGPU.td
@@ -158,8 +158,7 @@ def NVGPU_MmaSyncOp : NVGPU_MmaSyncOp<"mma.sync"> {
                        AnyVector:$matrixB,
                        AnyVector:$matrixC,
                        I64ArrayAttr:$mmaShape,
-                       OptionalAttr<UnitAttr>:$tf32Enabled
-                       );
+                       OptionalAttr<UnitAttr>:$tf32Enabled);
 
   let results = (outs AnyVector:$res);
 
@@ -167,7 +166,12 @@ def NVGPU_MmaSyncOp : NVGPU_MmaSyncOp<"mma.sync"> {
     OpBuilder<(ins "Value":$matrixA,
                    "Value":$matrixB,
                    "Value":$matrixC,
-                   "ArrayAttr":$mmaShape)>
+                   "ArrayAttr":$mmaShape)>,
+    OpBuilder<(ins "Value":$matrixA,
+                   "Value":$matrixB,
+                   "Value":$matrixC,
+                   "ArrayRef<int64_t>":$mmaShape,
+                   CArg<"bool", "false">:$tf32Enabled)>
   ];
 
   let assemblyFormat = [{
diff --git a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
index 77c853a2c35f4..0472d27906ead 100644
--- a/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
+++ b/mlir/lib/Dialect/NVGPU/IR/NVGPUDialect.cpp
@@ -96,6 +96,15 @@ void MmaSyncOp::build(::mlir::OpBuilder &odsBuilder,
         mmaShape, UnitAttr());
 }
 
+void MmaSyncOp::build(::mlir::OpBuilder &odsBuilder,
+                      ::mlir::OperationState &odsState, Value matrixA,
+                      Value matrixB, Value matrixC, ArrayRef<int64_t> mmaShape,
+                      bool tf32Enabled) {
+  build(odsBuilder, odsState, matrixC.getType(), matrixA, matrixB, matrixC,
+        odsBuilder.getI64ArrayAttr(mmaShape),
+        tf32Enabled ? odsBuilder.getUnitAttr() : UnitAttr());
+}
+
 /// Performs verification for MmaSyncOp and MmaSparseSyncOp.
 static LogicalResult verifyMmaSyncOp(Operation *op,
                                      TypedValue<VectorType> matrixA,

From 798b6419bc8446ba4d30c61e182ec628ea44ce40 Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Mon, 19 Jun 2023 15:35:48 +0100
Subject: [PATCH 030/130] [LSR] Add test for for issue leading to revert of
 abfeda5af329b5.

Add unit test triggering an assertion with abfeda5af329b5.
---
 ...ostinc-with-fixups-with-different-loops.ll | 53 +++++++++++++++++++
 1 file changed, 53 insertions(+)
 create mode 100644 llvm/test/Transforms/LoopStrengthReduce/AArch64/postinc-with-fixups-with-different-loops.ll

diff --git a/llvm/test/Transforms/LoopStrengthReduce/AArch64/postinc-with-fixups-with-different-loops.ll b/llvm/test/Transforms/LoopStrengthReduce/AArch64/postinc-with-fixups-with-different-loops.ll
new file mode 100644
index 0000000000000..60d5588dfea11
--- /dev/null
+++ b/llvm/test/Transforms/LoopStrengthReduce/AArch64/postinc-with-fixups-with-different-loops.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
+; RUN: opt -loop-reduce -S %s | FileCheck %s
+
+target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+target triple = "arm64-apple-macosx13.0.0"
+
+declare void @use(i32)
+
+define i32 @test() {
+; CHECK-LABEL: define i32 @test() {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[LOOP_1:%.*]]
+; CHECK:       loop.1:
+; CHECK-NEXT:    [[LSR_IV1:%.*]] = phi i32 [ [[LSR_IV_NEXT2:%.*]], [[LOOP_1]] ], [ -1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[LSR_IV_NEXT2]] = add i32 [[LSR_IV1]], 1
+; CHECK-NEXT:    br i1 false, label [[LOOP_2_PH:%.*]], label [[LOOP_1]]
+; CHECK:       loop.2.ph:
+; CHECK-NEXT:    [[LSR_IV_NEXT2_LCSSA:%.*]] = phi i32 [ [[LSR_IV_NEXT2]], [[LOOP_1]] ]
+; CHECK-NEXT:    br label [[LOOP_2:%.*]]
+; CHECK:       loop.2:
+; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i64 [ [[LSR_IV_NEXT:%.*]], [[LOOP_2]] ], [ 1, [[LOOP_2_PH]] ]
+; CHECK-NEXT:    [[IV_2:%.*]] = phi i32 [ [[LSR_IV_NEXT2_LCSSA]], [[LOOP_2_PH]] ], [ [[IV_2_NEXT:%.*]], [[LOOP_2]] ]
+; CHECK-NEXT:    call void @use(i32 [[IV_2]])
+; CHECK-NEXT:    [[IV_2_NEXT]] = add i32 [[IV_2]], 1
+; CHECK-NEXT:    [[LSR_IV_NEXT]] = add nsw i64 [[LSR_IV]], -1
+; CHECK-NEXT:    [[EC:%.*]] = icmp eq i64 [[LSR_IV_NEXT]], 0
+; CHECK-NEXT:    br i1 [[EC]], label [[EXIT:%.*]], label [[LOOP_2]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret i32 [[IV_2_NEXT]]
+;
+entry:
+  br label %loop.1
+
+loop.1:
+  %iv.1 = phi i32 [ 0, %entry ], [ %iv.1.next, %loop.1 ]
+  %iv.1.next = add i32 %iv.1, 1
+  br i1 false, label %loop.2.ph, label %loop.1
+
+loop.2.ph:
+  br label %loop.2
+
+loop.2:
+  %iv.2 = phi i32 [ %iv.1, %loop.2.ph ], [ %iv.2.next, %loop.2 ]
+  %iv.3 = phi i64 [ 0, %loop.2.ph ], [ %iv.3.next, %loop.2 ]
+  call void @use(i32 %iv.2)
+  %iv.2.next = add i32 %iv.2, 1
+  %iv.3.next = add i64 %iv.3, 1
+  %ec = icmp eq i64 %iv.3, 0
+  br i1 %ec, label %exit, label %loop.2
+
+exit:
+  ret i32 %iv.2.next
+}

From fa45f81ff7ea9fc2a2a40fea8dd7626ecc3a8dbb Mon Sep 17 00:00:00 2001
From: Roger Ferrer Ibanez <roger.ferrer@bsc.es>
Date: Mon, 19 Jun 2023 14:37:46 +0000
Subject: [PATCH 031/130] [clang][Serialization][RISCV] Increase the number of
 reserved predefined type IDs

In D152070 we added many new intrinsic types required for the RISC-V
Vector Extension.

This was crashing when loading the AST as those types are intrinsically
added to the AST (they don't come from the disk).

The total number required now by clang exceeds 400 so increasing the
value to 500 solves the problem. This value was already increased in
D92715 but I assume this has some impact on the on-disk format.

Also add a static assert to avoid this happening again in the future.

Differential Revision: https://reviews.llvm.org/D153111
---
 clang/include/clang/Serialization/ASTBitCodes.h | 10 +++++++++-
 clang/lib/Serialization/ASTReader.cpp           |  4 ++++
 clang/test/Modules/embed-files-compressed.cpp   |  4 ++--
 clang/test/Modules/empty.modulemap              |  4 ++--
 4 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h
index a93eb3d38a480..7019bc5922ebc 100644
--- a/clang/include/clang/Serialization/ASTBitCodes.h
+++ b/clang/include/clang/Serialization/ASTBitCodes.h
@@ -1099,6 +1099,8 @@ enum PredefinedTypeIDs {
 // \brief WebAssembly reference types with auto numeration
 #define WASM_TYPE(Name, Id, SingletonId) PREDEF_TYPE_##Id##_ID,
 #include "clang/Basic/WebAssemblyReferenceTypes.def"
+  // Sentinel value. Considered a predefined type but not useable as one.
+  PREDEF_TYPE_LAST_ID
 };
 
 /// The number of predefined type IDs that are reserved for
@@ -1106,7 +1108,13 @@ enum PredefinedTypeIDs {
 ///
 /// Type IDs for non-predefined types will start at
 /// NUM_PREDEF_TYPE_IDs.
-const unsigned NUM_PREDEF_TYPE_IDS = 300;
+const unsigned NUM_PREDEF_TYPE_IDS = 500;
+
+// Ensure we do not overrun the predefined types we reserved
+// in the enum PredefinedTypeIDs above.
+static_assert(PREDEF_TYPE_LAST_ID < NUM_PREDEF_TYPE_IDS,
+              "Too many enumerators in PredefinedTypeIDs. Review the value of "
+              "NUM_PREDEF_TYPE_IDS");
 
 /// Record codes for each kind of type.
 ///
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index a0ccc5aa4a741..cba6791783e8b 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -6983,6 +6983,10 @@ QualType ASTReader::GetType(TypeID ID) {
   if (Index < NUM_PREDEF_TYPE_IDS) {
     QualType T;
     switch ((PredefinedTypeIDs)Index) {
+    case PREDEF_TYPE_LAST_ID:
+      // We should never use this one.
+      llvm_unreachable("Invalid predefined type");
+      break;
     case PREDEF_TYPE_NULL_ID:
       return QualType();
     case PREDEF_TYPE_VOID_ID:
diff --git a/clang/test/Modules/embed-files-compressed.cpp b/clang/test/Modules/embed-files-compressed.cpp
index ae016bc1f9630..873b3082a2fdf 100644
--- a/clang/test/Modules/embed-files-compressed.cpp
+++ b/clang/test/Modules/embed-files-compressed.cpp
@@ -17,7 +17,7 @@
 // RUN: %clang_cc1 -fmodules -I%t -fmodules-cache-path=%t -fmodule-name=a -emit-module %t/modulemap -fmodules-embed-all-files -o %t/a.pcm
 //
 // The above embeds ~4.5MB of highly-predictable /s and \ns into the pcm file.
-// Check that the resulting file is under 40KB:
+// Check that the resulting file is under 60KB:
 //
 // RUN: wc -c %t/a.pcm | FileCheck --check-prefix=CHECK-SIZE %s
-// CHECK-SIZE: {{(^|[^0-9])[123][0-9][0-9][0-9][0-9]($|[^0-9])}}
+// CHECK-SIZE: {{(^|[^0-9])[1-5][0-9][0-9][0-9][0-9]($|[^0-9])}}
diff --git a/clang/test/Modules/empty.modulemap b/clang/test/Modules/empty.modulemap
index 3225d88829ae0..f2d37c19d77bc 100644
--- a/clang/test/Modules/empty.modulemap
+++ b/clang/test/Modules/empty.modulemap
@@ -13,8 +13,8 @@
 // The module file should be identical each time we produce it.
 // RUN: diff %t/base.pcm %t/check.pcm
 //
-// We expect an empty module to be less than 40KB (and at least 10K, for now).
+// We expect an empty module to be less than 60KB (and at least 10K, for now).
 // RUN: wc -c %t/base.pcm | FileCheck --check-prefix=CHECK-SIZE %s
-// CHECK-SIZE: {{(^|[^0-9])[123][0-9][0-9][0-9][0-9]($|[^0-9])}}
+// CHECK-SIZE: {{(^|[^0-9])[1-5][0-9][0-9][0-9][0-9]($|[^0-9])}}
 
 module empty { header "Inputs/empty.h" export * }

From 66511b401042f28c74d2ded3aac76d19a53bd7c4 Mon Sep 17 00:00:00 2001
From: Vladislav Dzhidzhoev <vdzhidzhoev@accesssoftek.com>
Date: Mon, 19 Jun 2023 16:42:05 +0200
Subject: [PATCH 032/130] [DebugMetadata][DwarfDebug] Support function-local
 types in lexical block scopes (4/7)

RFC https://discourse.llvm.org/t/rfc-dwarfdebug-fix-and-improve-handling-imported-entities-types-and-static-local-in-subprogram-and-lexical-block-scopes/68544

Similar to imported declarations, the patch tracks function-local types in
DISubprogram's 'retainedNodes' field. DwarfDebug is adjusted in accordance with
the aforementioned metadata change and provided a support of function-local
types scoped within a lexical block.

The patch assumes that DICompileUnit's 'enums field' no longer tracks local
types and DwarfDebug would assert if any locally-scoped types get placed there.

Authored-by: Kristina Bessonova <kbessonova@accesssoftek.com>

Differential Revision: https://reviews.llvm.org/D144006

Depends on D144005
---
 .../CodeGen/debug-info-codeview-unnamed.c     |  16 +-
 clang/test/CodeGen/debug-info-unused-types.c  |  14 +-
 .../test/CodeGen/debug-info-unused-types.cpp  |  12 +-
 clang/test/CodeGenCXX/debug-info-access.cpp   |   2 +-
 .../CodeGenCXX/debug-info-anon-union-vars.cpp |  12 +-
 .../debug-info-codeview-unnamed.cpp           | 110 +++--
 .../debug-info-gline-tables-only-codeview.cpp |   4 +-
 clang/test/CodeGenCXX/debug-lambda-this.cpp   |   4 +-
 llvm/include/llvm/IR/DIBuilder.h              |   6 +-
 llvm/lib/Bitcode/Reader/MetadataLoader.cpp    |  87 ++--
 .../CodeGen/AsmPrinter/DwarfCompileUnit.cpp   |  60 ++-
 .../lib/CodeGen/AsmPrinter/DwarfCompileUnit.h |  16 +-
 llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp    |  13 +-
 llvm/lib/IR/DIBuilder.cpp                     |  34 +-
 llvm/lib/IR/Verifier.cpp                      |   6 +-
 llvm/test/Bitcode/upgrade-cu-locals.ll        |  68 +--
 llvm/test/Bitcode/upgrade-cu-locals.ll.bc     | Bin 2688 -> 2780 bytes
 .../DebugInfo/Generic/inlined-local-type.ll   | 128 ++++++
 .../Generic/lexical-block-retained-types.ll   |  55 +++
 .../DebugInfo/Generic/lexical-block-types.ll  | 425 ++++++++++++++++++
 .../local-type-as-template-parameter.ll       | 160 +++++++
 .../Generic/verifier-invalid-disubprogram.ll  |   2 +-
 llvm/test/DebugInfo/X86/set.ll                |   4 +-
 23 files changed, 1051 insertions(+), 187 deletions(-)
 create mode 100644 llvm/test/DebugInfo/Generic/inlined-local-type.ll
 create mode 100644 llvm/test/DebugInfo/Generic/lexical-block-retained-types.ll
 create mode 100644 llvm/test/DebugInfo/Generic/lexical-block-types.ll
 create mode 100644 llvm/test/DebugInfo/Generic/local-type-as-template-parameter.ll

diff --git a/clang/test/CodeGen/debug-info-codeview-unnamed.c b/clang/test/CodeGen/debug-info-codeview-unnamed.c
index bd2a7543e56b2..16ffb3682236f 100644
--- a/clang/test/CodeGen/debug-info-codeview-unnamed.c
+++ b/clang/test/CodeGen/debug-info-codeview-unnamed.c
@@ -8,23 +8,23 @@ int main(int argc, char* argv[], char* arge[]) {
   //
   struct { int bar; } one = {42};
   //
-  // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "one"
-  // LINUX-SAME:     type: [[TYPE_OF_ONE:![0-9]+]]
-  // LINUX-SAME:     )
-  // LINUX:      [[TYPE_OF_ONE]] = distinct !DICompositeType(
+  // LINUX:      [[TYPE_OF_ONE:![0-9]+]] = distinct !DICompositeType(
   // LINUX-SAME:     tag: DW_TAG_structure_type
   // LINUX-NOT:      name:
   // LINUX-NOT:      identifier:
   // LINUX-SAME:     )
+  // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "one"
+  // LINUX-SAME:     type: [[TYPE_OF_ONE]]
+  // LINUX-SAME:     )
   //
-  // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "one"
-  // MSVC-SAME:      type: [[TYPE_OF_ONE:![0-9]+]]
-  // MSVC-SAME:      )
-  // MSVC:       [[TYPE_OF_ONE]] = distinct !DICompositeType
+  // MSVC:       [[TYPE_OF_ONE:![0-9]+]] = distinct !DICompositeType
   // MSVC-SAME:      tag: DW_TAG_structure_type
   // MSVC-NOT:       name:
   // MSVC-NOT:       identifier:
   // MSVC-SAME:      )
+  // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "one"
+  // MSVC-SAME:      type: [[TYPE_OF_ONE]]
+  // MSVC-SAME:      )
 
   return 0;
 }
diff --git a/clang/test/CodeGen/debug-info-unused-types.c b/clang/test/CodeGen/debug-info-unused-types.c
index 3e9f7b07658e3..5bcc71f710a5c 100644
--- a/clang/test/CodeGen/debug-info-unused-types.c
+++ b/clang/test/CodeGen/debug-info-unused-types.c
@@ -18,13 +18,13 @@ void quux(void) {
 // CHECK: !DICompileUnit{{.+}}retainedTypes: [[RETTYPES:![0-9]+]]
 // CHECK: [[TYPE0:![0-9]+]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "bar"
 // CHECK: [[TYPE1:![0-9]+]] = !DIEnumerator(name: "BAR"
-// CHECK: [[TYPE2:![0-9]+]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "z"
-// CHECK: [[TYPE3:![0-9]+]] = !DIEnumerator(name: "Z"
-// CHECK: [[RETTYPES]] = !{[[TYPE4:![0-9]+]], [[TYPE5:![0-9]+]], [[TYPE0]], [[TYPE6:![0-9]+]], {{![0-9]+}}, [[TYPE7:![0-9]+]], [[TYPE2]], [[TYPE8:![0-9]+]]}
-// CHECK: [[TYPE4]] = !DIDerivedType(tag: DW_TAG_typedef, name: "my_int"
-// CHECK: [[TYPE5]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "foo"
-// CHECK: [[TYPE6]] = distinct !DICompositeType(tag: DW_TAG_union_type, name: "baz"
-// CHECK: [[TYPE7]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "y"
+// CHECK: [[RETTYPES]] = !{[[TYPE2:![0-9]+]], [[TYPE3:![0-9]+]], [[TYPE0]], [[TYPE4:![0-9]+]], {{![0-9]+}}, [[TYPE5:![0-9]+]], [[TYPE6:![0-9]+]], [[TYPE8:![0-9]+]]}
+// CHECK: [[TYPE2]] = !DIDerivedType(tag: DW_TAG_typedef, name: "my_int"
+// CHECK: [[TYPE3]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "foo"
+// CHECK: [[TYPE4]] = distinct !DICompositeType(tag: DW_TAG_union_type, name: "baz"
+// CHECK: [[TYPE5]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "y"
+// CHECK: [[TYPE6]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "z"
+// CHECK: [[TYPE7:![0-9]+]] = !DIEnumerator(name: "Z"
 // CHECK: [[TYPE8]] = distinct !DICompositeType(tag: DW_TAG_union_type, name: "w"
 
 // Check that debug info is not emitted for the typedef, struct, enum, and
diff --git a/clang/test/CodeGen/debug-info-unused-types.cpp b/clang/test/CodeGen/debug-info-unused-types.cpp
index 023cac159faa4..715fada47eab8 100644
--- a/clang/test/CodeGen/debug-info-unused-types.cpp
+++ b/clang/test/CodeGen/debug-info-unused-types.cpp
@@ -13,12 +13,12 @@ void quux() {
 // CHECK: !DICompileUnit{{.+}}retainedTypes: [[RETTYPES:![0-9]+]]
 // CHECK: [[TYPE0:![0-9]+]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "baz"
 // CHECK: [[TYPE1:![0-9]+]] = !DIEnumerator(name: "BAZ"
-// CHECK: [[TYPE2:![0-9]+]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "z"
-// CHECK: [[TYPE3:![0-9]+]] = !DIEnumerator(name: "Z"
-// CHECK: [[RETTYPES]] = !{[[TYPE4:![0-9]+]], [[TYPE5:![0-9]+]], [[TYPE0]], {{![0-9]+}}, [[TYPE6:![0-9]+]], [[TYPE2]]}
-// CHECK: [[TYPE4]] = !DIDerivedType(tag: DW_TAG_typedef, name: "foo"
-// CHECK: [[TYPE5]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "bar"
-// CHECK: [[TYPE6]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "y"
+// CHECK: [[RETTYPES]] = !{[[TYPE2:![0-9]+]], [[TYPE3:![0-9]+]], [[TYPE0]], {{![0-9]+}}, [[TYPE4:![0-9]+]], [[TYPE5:![0-9]+]]}
+// CHECK: [[TYPE2]] = !DIDerivedType(tag: DW_TAG_typedef, name: "foo"
+// CHECK: [[TYPE3]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "bar"
+// CHECK: [[TYPE4]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "y"
+// CHECK: [[TYPE5]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "z"
+// CHECK: [[TYPE6:![0-9]+]] = !DIEnumerator(name: "Z"
 
 // NODBG-NOT: !DI{{CompositeType|Enumerator|DerivedType}}
 
diff --git a/clang/test/CodeGenCXX/debug-info-access.cpp b/clang/test/CodeGenCXX/debug-info-access.cpp
index 9f2c044843d0f..7c0bf8ccb0384 100644
--- a/clang/test/CodeGenCXX/debug-info-access.cpp
+++ b/clang/test/CodeGenCXX/debug-info-access.cpp
@@ -18,9 +18,9 @@ class B : public A {
   static int public_static;
 
 protected:
+  // CHECK-DAG: !DIDerivedType(tag: DW_TAG_typedef, name: "prot_using",{{.*}} line: [[@LINE+3]],{{.*}} flags: DIFlagProtected)
   // CHECK-DAG: !DIDerivedType(tag: DW_TAG_typedef, name: "prot_typedef",{{.*}} line: [[@LINE+1]],{{.*}} flags: DIFlagProtected)
   typedef int prot_typedef;
-  // CHECK-DAG: !DIDerivedType(tag: DW_TAG_typedef, name: "prot_using",{{.*}} line: [[@LINE+1]],{{.*}} flags: DIFlagProtected)
   using prot_using = prot_typedef;
   prot_using prot_member;
 
diff --git a/clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp b/clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp
index 61b3c7c0526c8..c91cf83c0405f 100644
--- a/clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp
+++ b/clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp
@@ -51,13 +51,13 @@ void instantiate(int x) {
 // CHECK: !DIGlobalVariable(name: "b",{{.*}} file: [[FILE]], line: 6,{{.*}} isLocal: true, isDefinition: true
 // CHECK: !DIGlobalVariable(name: "result", {{.*}} isLocal: false, isDefinition: true
 // CHECK: !DIGlobalVariable(name: "value", {{.*}} isLocal: false, isDefinition: true
-// CHECK: !DILocalVariable(name: "i", {{.*}}, flags: DIFlagArtificial
-// CHECK: !DILocalVariable(name: "c", {{.*}}, flags: DIFlagArtificial
-// CHECK: !DILocalVariable(
-// CHECK-NOT: name:
-// CHECK: type: ![[UNION:[0-9]+]]
-// CHECK: ![[UNION]] = distinct !DICompositeType(tag: DW_TAG_union_type,
+// CHECK: ![[UNION:[0-9]+]] = distinct !DICompositeType(tag: DW_TAG_union_type,
 // CHECK-NOT: name:
 // CHECK: elements
 // CHECK: !DIDerivedType(tag: DW_TAG_member, name: "i", scope: ![[UNION]],
 // CHECK: !DIDerivedType(tag: DW_TAG_member, name: "c", scope: ![[UNION]],
+// CHECK: !DILocalVariable(name: "i", {{.*}}, flags: DIFlagArtificial
+// CHECK: !DILocalVariable(name: "c", {{.*}}, flags: DIFlagArtificial
+// CHECK: !DILocalVariable(
+// CHECK-NOT: name:
+// CHECK: type: ![[UNION]]
diff --git a/clang/test/CodeGenCXX/debug-info-codeview-unnamed.cpp b/clang/test/CodeGenCXX/debug-info-codeview-unnamed.cpp
index b4c79936ab33e..9602ac1b02497 100644
--- a/clang/test/CodeGenCXX/debug-info-codeview-unnamed.cpp
+++ b/clang/test/CodeGenCXX/debug-info-codeview-unnamed.cpp
@@ -3,6 +3,60 @@
 
 int main(int argc, char* argv[], char* arge[]) {
   //
+  // LINUX:      [[TYPE_OF_ONE:![0-9]+]] = distinct !DICompositeType(
+  // LINUX-SAME:     tag: DW_TAG_structure_type
+  // LINUX-NOT:      name:
+  // LINUX-NOT:      identifier:
+  // LINUX-SAME:     )
+  //
+  // MSVC:       [[TYPE_OF_ONE:![0-9]+]] = distinct !DICompositeType
+  // MSVC-SAME:      tag: DW_TAG_structure_type
+  // MSVC-SAME:      name: "<unnamed-type-one>"
+  // MSVC-SAME:      identifier: ".?AU<unnamed-type-one>@?1??main@@9@"
+  // MSVC-SAME:      )
+
+
+  //
+  // LINUX:      [[TYPE_OF_TWO:![0-9]+]] = distinct !DICompositeType(
+  // LINUX-SAME:     tag: DW_TAG_structure_type
+  // LINUX-NOT:      name:
+  // LINUX-NOT:      identifier:
+  // LINUX-SAME:     )
+  //
+  // MSVC:       [[TYPE_OF_TWO:![0-9]+]] = distinct !DICompositeType
+  // MSVC-SAME:      tag: DW_TAG_structure_type
+  // MSVC-SAME:      name: "<unnamed-type-two>"
+  // MSVC-SAME:      identifier: ".?AU<unnamed-type-two>@?2??main@@9@"
+  // MSVC-SAME:      )
+
+
+  //
+  // LINUX:      [[TYPE_OF_THREE:![0-9]+]] = distinct !DICompositeType(
+  // LINUX-SAME:     tag: DW_TAG_structure_type
+  // LINUX-SAME:     name: "named"
+  // LINUX-NOT:      identifier:
+  // LINUX-SAME:     )
+  //
+  // MSVC:       [[TYPE_OF_THREE:![0-9]+]] = distinct !DICompositeType
+  // MSVC-SAME:      tag: DW_TAG_structure_type
+  // MSVC-SAME:      name: "named"
+  // MSVC-SAME:      identifier: ".?AUnamed@?1??main@@9@"
+  // MSVC-SAME:      )
+
+  //
+  // LINUX:      [[TYPE_OF_FOUR:![0-9]+]] = distinct !DICompositeType(
+  // LINUX-SAME:     tag: DW_TAG_class_type
+  // LINUX-NOT:      name:
+  // LINUX-NOT:      identifier:
+  // LINUX-SAME:     )
+  //
+  // MSVC:       [[TYPE_OF_FOUR:![0-9]+]] = distinct !DICompositeType
+  // MSVC-SAME:      tag: DW_TAG_class_type
+  // MSVC-SAME:      name: "<lambda_0>"
+  // MSVC-SAME:      identifier: ".?AV<lambda_0>@?0??main@@9@"
+  // MSVC-SAME:      )
+
+
   // In CodeView, the LF_MFUNCTION entry for "bar()" refers to the forward
   // reference of the unnamed struct. Visual Studio requires a unique
   // identifier to match the LF_STRUCTURE forward reference to the definition.
@@ -10,21 +64,11 @@ int main(int argc, char* argv[], char* arge[]) {
   struct { void bar() {} } one;
   //
   // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "one"
-  // LINUX-SAME:     type: [[TYPE_OF_ONE:![0-9]+]]
-  // LINUX-SAME:     )
-  // LINUX:      [[TYPE_OF_ONE]] = distinct !DICompositeType(
-  // LINUX-SAME:     tag: DW_TAG_structure_type
-  // LINUX-NOT:      name:
-  // LINUX-NOT:      identifier:
+  // LINUX-SAME:     type: [[TYPE_OF_ONE]]
   // LINUX-SAME:     )
   //
   // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "one"
-  // MSVC-SAME:      type: [[TYPE_OF_ONE:![0-9]+]]
-  // MSVC-SAME:      )
-  // MSVC:       [[TYPE_OF_ONE]] = distinct !DICompositeType
-  // MSVC-SAME:      tag: DW_TAG_structure_type
-  // MSVC-SAME:      name: "<unnamed-type-one>"
-  // MSVC-SAME:      identifier: ".?AU<unnamed-type-one>@?1??main@@9@"
+  // MSVC-SAME:      type: [[TYPE_OF_ONE]]
   // MSVC-SAME:      )
 
 
@@ -36,21 +80,11 @@ int main(int argc, char* argv[], char* arge[]) {
   int decltype(two)::*ptr2unnamed = &decltype(two)::bar;
   //
   // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "two"
-  // LINUX-SAME:     type: [[TYPE_OF_TWO:![0-9]+]]
-  // LINUX-SAME:     )
-  // LINUX:      [[TYPE_OF_TWO]] = distinct !DICompositeType(
-  // LINUX-SAME:     tag: DW_TAG_structure_type
-  // LINUX-NOT:      name:
-  // LINUX-NOT:      identifier:
+  // LINUX-SAME:     type: [[TYPE_OF_TWO]]
   // LINUX-SAME:     )
   //
   // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "two"
-  // MSVC-SAME:      type: [[TYPE_OF_TWO:![0-9]+]]
-  // MSVC-SAME:      )
-  // MSVC:       [[TYPE_OF_TWO]] = distinct !DICompositeType
-  // MSVC-SAME:      tag: DW_TAG_structure_type
-  // MSVC-SAME:      name: "<unnamed-type-two>"
-  // MSVC-SAME:      identifier: ".?AU<unnamed-type-two>@?2??main@@9@"
+  // MSVC-SAME:      type: [[TYPE_OF_TWO]]
   // MSVC-SAME:      )
 
 
@@ -61,21 +95,11 @@ int main(int argc, char* argv[], char* arge[]) {
   struct named { int bar; int named::* p2mem; } three = { 42, &named::bar };
   //
   // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "three"
-  // LINUX-SAME:     type: [[TYPE_OF_THREE:![0-9]+]]
-  // LINUX-SAME:     )
-  // LINUX:      [[TYPE_OF_THREE]] = distinct !DICompositeType(
-  // LINUX-SAME:     tag: DW_TAG_structure_type
-  // LINUX-SAME:     name: "named"
-  // LINUX-NOT:      identifier:
+  // LINUX-SAME:     type: [[TYPE_OF_THREE]]
   // LINUX-SAME:     )
   //
   // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "three"
-  // MSVC-SAME:      type: [[TYPE_OF_THREE:![0-9]+]]
-  // MSVC-SAME:      )
-  // MSVC:       [[TYPE_OF_THREE]] = distinct !DICompositeType
-  // MSVC-SAME:      tag: DW_TAG_structure_type
-  // MSVC-SAME:      name: "named"
-  // MSVC-SAME:      identifier: ".?AUnamed@?1??main@@9@"
+  // MSVC-SAME:      type: [[TYPE_OF_THREE]]
   // MSVC-SAME:      )
 
 
@@ -87,21 +111,11 @@ int main(int argc, char* argv[], char* arge[]) {
   auto four = [argc](int i) -> int { return argc == i ? 1 : 0; };
   //
   // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "four"
-  // LINUX-SAME:     type: [[TYPE_OF_FOUR:![0-9]+]]
-  // LINUX-SAME:     )
-  // LINUX:      [[TYPE_OF_FOUR]] = distinct !DICompositeType(
-  // LINUX-SAME:     tag: DW_TAG_class_type
-  // LINUX-NOT:      name:
-  // LINUX-NOT:      identifier:
+  // LINUX-SAME:     type: [[TYPE_OF_FOUR]]
   // LINUX-SAME:     )
   //
   // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "four"
-  // MSVC-SAME:      type: [[TYPE_OF_FOUR:![0-9]+]]
-  // MSVC-SAME:      )
-  // MSVC:       [[TYPE_OF_FOUR]] = distinct !DICompositeType
-  // MSVC-SAME:      tag: DW_TAG_class_type
-  // MSVC-SAME:      name: "<lambda_0>"
-  // MSVC-SAME:      identifier: ".?AV<lambda_0>@?0??main@@9@"
+  // MSVC-SAME:      type: [[TYPE_OF_FOUR]]
   // MSVC-SAME:      )
 
   return 0;
diff --git a/clang/test/CodeGenCXX/debug-info-gline-tables-only-codeview.cpp b/clang/test/CodeGenCXX/debug-info-gline-tables-only-codeview.cpp
index 6b9c9a243decd..122e4aa62ea7d 100644
--- a/clang/test/CodeGenCXX/debug-info-gline-tables-only-codeview.cpp
+++ b/clang/test/CodeGenCXX/debug-info-gline-tables-only-codeview.cpp
@@ -51,9 +51,9 @@ void test() {
   // CHECK-SAME:                              name: "<lambda_2_1>",
   c.lambda_params();
 
-  // CHECK: !DISubprogram(name: "operator()", scope: ![[LAMBDA1:[0-9]+]],
-  // CHECK: ![[LAMBDA1]] = !DICompositeType(tag: DW_TAG_class_type,
+  // CHECK: ![[LAMBDA1:[0-9]+]] = !DICompositeType(tag: DW_TAG_class_type,
   // CHECK-SAME:                            name: "<lambda_1>",
   // CHECK-SAME:                            flags: DIFlagFwdDecl
+  // CHECK: !DISubprogram(name: "operator()", scope: ![[LAMBDA1]],
   c.lambda2();
 }
diff --git a/clang/test/CodeGenCXX/debug-lambda-this.cpp b/clang/test/CodeGenCXX/debug-lambda-this.cpp
index eecbac6520ac9..3d659e7bfd004 100644
--- a/clang/test/CodeGenCXX/debug-lambda-this.cpp
+++ b/clang/test/CodeGenCXX/debug-lambda-this.cpp
@@ -13,10 +13,10 @@ int D::d(int x) {
 }
 
 // CHECK: ![[D:[0-9]+]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "D",
-// CHECK: ![[POINTER:.*]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: ![[D]], size: 64)
 // CHECK: !DIDerivedType(tag: DW_TAG_member, name: "this",
 // CHECK-SAME:           line: 11
-// CHECK-SAME:           baseType: ![[POINTER]]
+// CHECK-SAME:           baseType: ![[POINTER:[0-9]+]]
 // CHECK-SAME:           size: 64
 // CHECK-NOT:            offset: 0
 // CHECK-SAME:           ){{$}}
+// CHECK: ![[POINTER]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: ![[D]], size: 64)
diff --git a/llvm/include/llvm/IR/DIBuilder.h b/llvm/include/llvm/IR/DIBuilder.h
index ecd6dd7b0a4f8..0aa180aec6d8a 100644
--- a/llvm/include/llvm/IR/DIBuilder.h
+++ b/llvm/include/llvm/IR/DIBuilder.h
@@ -49,7 +49,7 @@ namespace llvm {
     Function *LabelFn;       ///< llvm.dbg.label
     Function *AssignFn;      ///< llvm.dbg.assign
 
-    SmallVector<TrackingMDNodeRef, 4> AllEnumTypes;
+    SmallVector<TrackingMDNodeRef, 4> EnumTypes;
     /// Track the RetainTypes, since they can be updated later on.
     SmallVector<TrackingMDNodeRef, 4> AllRetainTypes;
     SmallVector<DISubprogram *, 4> AllSubprograms;
@@ -64,8 +64,8 @@ namespace llvm {
     SmallVector<TrackingMDNodeRef, 4> UnresolvedNodes;
     bool AllowUnresolvedNodes;
 
-    /// Each subprogram's preserved local variables, labels and imported
-    /// entities.
+    /// Each subprogram's preserved local variables, labels, imported entities,
+    /// and types.
     ///
     /// Do not use a std::vector.  Some versions of libc++ apparently copy
     /// instead of move on grow operations, and TrackingMDRef is expensive to
diff --git a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
index 0a9a80688a419..321eae6fcb3b5 100644
--- a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
+++ b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
@@ -548,6 +548,8 @@ class MetadataLoader::MetadataLoaderImpl {
 
   /// Move local imports from DICompileUnit's 'imports' field to
   /// DISubprogram's retainedNodes.
+  /// Move fucntion-local enums from DICompileUnit's enums
+  /// to DISubprogram's retainedNodes.
   void upgradeCULocals() {
     if (NamedMDNode *CUNodes = TheModule.getNamedMetadata("llvm.dbg.cu")) {
       for (unsigned I = 0, E = CUNodes->getNumOperands(); I != E; ++I) {
@@ -555,48 +557,71 @@ class MetadataLoader::MetadataLoaderImpl {
         if (!CU)
           continue;
 
-        if (auto *RawImported = CU->getRawImportedEntities()) {
-          // Collect a set of imported entities to be moved.
-          SetVector<Metadata *> EntitiesToRemove;
+        SetVector<Metadata *> MetadataToRemove;
+        // Collect imported entities to be moved.
+        if (CU->getRawImportedEntities()) {
           for (Metadata *Op : CU->getImportedEntities()->operands()) {
             auto *IE = cast<DIImportedEntity>(Op);
-            if (auto *S = dyn_cast_or_null<DILocalScope>(IE->getScope())) {
-              EntitiesToRemove.insert(IE);
+            if (dyn_cast_or_null<DILocalScope>(IE->getScope())) {
+              MetadataToRemove.insert(IE);
             }
           }
+        }
+        // Collect enums to be moved.
+        if (CU->getRawEnumTypes()) {
+          for (Metadata *Op : CU->getEnumTypes()->operands()) {
+            auto *Enum = cast<DICompositeType>(Op);
+            if (dyn_cast_or_null<DILocalScope>(Enum->getScope())) {
+              MetadataToRemove.insert(Enum);
+            }
+          }
+        }
 
-          if (!EntitiesToRemove.empty()) {
-            // Make a new list of CU's 'imports'.
-            SmallVector<Metadata *> NewImports;
-            for (Metadata *Op : CU->getImportedEntities()->operands()) {
-              if (!EntitiesToRemove.contains(cast<DIImportedEntity>(Op))) {
+        if (!MetadataToRemove.empty()) {
+          // Make a new list of CU's 'imports'.
+          SmallVector<Metadata *> NewImports;
+          if (CU->getRawImportedEntities())
+            for (Metadata *Op : CU->getImportedEntities()->operands())
+              if (!MetadataToRemove.contains(Op))
                 NewImports.push_back(Op);
-              }
-            }
 
-            // Find DISubprogram corresponding to each entity.
-            std::map<DISubprogram *, SmallVector<Metadata *>> SPToEntities;
-            for (auto *I : EntitiesToRemove) {
-              auto *Entity = cast<DIImportedEntity>(I);
-              if (auto *SP = findEnclosingSubprogram(
-                      cast<DILocalScope>(Entity->getScope()))) {
-                SPToEntities[SP].push_back(Entity);
-              }
+          // Make a new list of CU's 'enums'.
+          SmallVector<Metadata *> NewEnums;
+          if (CU->getRawEnumTypes())
+            for (Metadata *Op : CU->getEnumTypes()->operands())
+              if (!MetadataToRemove.contains(Op))
+                NewEnums.push_back(Op);
+
+          // Find DISubprogram corresponding to each entity.
+          std::map<DISubprogram *, SmallVector<Metadata *>> SPToEntities;
+          for (auto *I : MetadataToRemove) {
+            DILocalScope *Scope = nullptr;
+            if (auto *Entity = dyn_cast<DIImportedEntity>(I))
+              Scope = cast<DILocalScope>(Entity->getScope());
+            else if (auto *Enum = dyn_cast<DICompositeType>(I))
+              Scope = cast<DILocalScope>(Enum->getScope());
+
+            if (auto *SP = findEnclosingSubprogram(Scope)) {
+              SPToEntities[SP].push_back(I);
             }
+          }
 
-            // Update DISubprograms' retainedNodes.
-            for (auto I = SPToEntities.begin(); I != SPToEntities.end(); ++I) {
-              auto *SP = I->first;
-              auto RetainedNodes = SP->getRetainedNodes();
-              SmallVector<Metadata *> MDs(RetainedNodes.begin(),
-                                          RetainedNodes.end());
-              MDs.append(I->second);
-              SP->replaceRetainedNodes(MDNode::get(Context, MDs));
-            }
+          // Update DISubprograms' retainedNodes.
+          for (auto I = SPToEntities.begin(); I != SPToEntities.end(); ++I) {
+            auto *SP = I->first;
+            auto RetainedNodes = SP->getRetainedNodes();
+            SmallVector<Metadata *> MDs(RetainedNodes.begin(),
+                                        RetainedNodes.end());
+            MDs.append(I->second);
+            SP->replaceRetainedNodes(MDNode::get(Context, MDs));
+          }
 
-            // Remove entities with local scope from CU.
+          // Remove entities with local scope from CU.
+          if (CU->getRawImportedEntities())
             CU->replaceImportedEntities(MDTuple::get(Context, NewImports));
-          }
+          // Remove enums with local scope from CU.
+          if (CU->getRawEnumTypes())
+            CU->replaceEnumTypes(MDTuple::get(Context, NewEnums));
         }
       }
     }
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
index 58ed21379d29b..bf8bdbb7da49f 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
@@ -598,10 +598,9 @@ void DwarfCompileUnit::constructScopeDIE(LexicalScope *Scope,
     return;
 
   // Emit lexical blocks.
-  DIE *ScopeDIE = constructLexicalScopeDIE(Scope);
+  DIE *ScopeDIE = getOrCreateLexicalBlockDIE(Scope, ParentScopeDIE);
   assert(ScopeDIE && "Scope DIE should not be null.");
 
-  ParentScopeDIE.addChild(ScopeDIE);
   createAndAddScopeChildren(Scope, *ScopeDIE);
 }
 
@@ -721,24 +720,39 @@ DIE *DwarfCompileUnit::constructInlinedScopeDIE(LexicalScope *Scope,
   return ScopeDIE;
 }
 
-// Construct new DW_TAG_lexical_block for this scope and attach
-// DW_AT_low_pc/DW_AT_high_pc labels.
-DIE *DwarfCompileUnit::constructLexicalScopeDIE(LexicalScope *Scope) {
+DIE *DwarfCompileUnit::getOrCreateLexicalBlockDIE(LexicalScope *Scope,
+                                                  DIE &ParentScopeDIE) {
   if (DD->isLexicalScopeDIENull(Scope))
     return nullptr;
   const auto *DS = Scope->getScopeNode();
-
-  auto ScopeDIE = DIE::get(DIEValueAllocator, dwarf::DW_TAG_lexical_block);
-  if (Scope->isAbstractScope()) {
-    assert(!getAbstractScopeDIEs().count(DS) &&
-           "Abstract DIE for this scope exists!");
-    getAbstractScopeDIEs()[DS] = ScopeDIE;
-    return ScopeDIE;
+  DIE *ScopeDIE = nullptr;
+
+  // FIXME: We may have a concrete DIE for this scope already created.
+  // This may happen when we emit local variables for an abstract tree of
+  // an inlined function: if a local variable has a templated type with
+  // a function-local type as a template parameter. See PR55680 for details
+  // (see also llvm/test/DebugInfo/Generic/local-type-as-template-parameter.ll).
+  if (!Scope->isAbstractScope() && !Scope->getInlinedAt()) {
+    if (auto It = LexicalBlockDIEs.find(DS); It != LexicalBlockDIEs.end()) {
+      ScopeDIE = It->second;
+      assert(!ScopeDIE->findAttribute(dwarf::DW_AT_low_pc) &&
+             !ScopeDIE->findAttribute(dwarf::DW_AT_ranges));
+      assert(ScopeDIE->getParent() == &ParentScopeDIE);
+    }
   }
-  if (!Scope->getInlinedAt()) {
-    assert(!LexicalBlockDIEs.count(DS) &&
-           "Concrete out-of-line DIE for this scope exists!");
-    LexicalBlockDIEs[DS] = ScopeDIE;
+  if (!ScopeDIE) {
+    ScopeDIE = DIE::get(DIEValueAllocator, dwarf::DW_TAG_lexical_block);
+    ParentScopeDIE.addChild(ScopeDIE);
+
+    if (Scope->isAbstractScope()) {
+      assert(!getAbstractScopeDIEs().count(DS) &&
+             "Abstract DIE for this scope exists!");
+      getAbstractScopeDIEs()[DS] = ScopeDIE;
+      return ScopeDIE;
+    }
+
+    if (!Scope->getInlinedAt())
+      LexicalBlockDIEs[DS] = ScopeDIE;
   }
 
   attachRangesOrLowHighPC(*ScopeDIE, Scope->getRanges());
@@ -1681,15 +1695,21 @@ void DwarfCompileUnit::createBaseTypeDIEs() {
   }
 }
 
-DIE *DwarfCompileUnit::getLexicalBlockDIE(const DILexicalBlock *LB) {
+DIE *DwarfCompileUnit::getLocalContextDIE(const DILexicalBlock *LB) {
   // Assume if there is an abstract tree all the DIEs are already emitted.
   bool isAbstract = getAbstractScopeDIEs().count(LB->getSubprogram());
   if (isAbstract && getAbstractScopeDIEs().count(LB))
     return getAbstractScopeDIEs()[LB];
   assert(!isAbstract && "Missed lexical block DIE in abstract tree!");
 
-  // Return a concrete DIE if it exists or nullptr otherwise.
-  return LexicalBlockDIEs.lookup(LB);
+  // Check if we have a concrete DIE.
+  if (auto It = LexicalBlockDIEs.find(LB); It != LexicalBlockDIEs.end())
+    return It->second;
+
+  // If nothing available found, we cannot just create a new lexical block,
+  // because it isn't known where to put it into the DIE tree.
+  // So, we may only try to find the most close avaiable parent DIE.
+  return getOrCreateContextDIE(LB->getScope()->getNonLexicalBlockFileScope());
 }
 
 DIE *DwarfCompileUnit::getOrCreateContextDIE(const DIScope *Context) {
@@ -1697,7 +1717,7 @@ DIE *DwarfCompileUnit::getOrCreateContextDIE(const DIScope *Context) {
     if (auto *LFScope = dyn_cast<DILexicalBlockFile>(Context))
       Context = LFScope->getNonLexicalBlockFileScope();
     if (auto *LScope = dyn_cast<DILexicalBlock>(Context))
-      return getLexicalBlockDIE(LScope);
+      return getLocalContextDIE(LScope);
 
     // Otherwise the context must be a DISubprogram.
     auto *SPScope = cast<DISubprogram>(Context);
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
index 6ef73ebd4f7f2..5662087bec322 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
@@ -210,14 +210,9 @@ class DwarfCompileUnit final : public DwarfUnit {
   /// DIE to represent this concrete inlined copy of the function.
   DIE *constructInlinedScopeDIE(LexicalScope *Scope, DIE &ParentScopeDIE);
 
-  /// Construct new DW_TAG_lexical_block for this scope and
-  /// attach DW_AT_low_pc/DW_AT_high_pc labels.
-  DIE *constructLexicalScopeDIE(LexicalScope *Scope);
-
-  /// Get a DIE for the given DILexicalBlock.
-  /// Note that this function assumes that the DIE has been already created
-  /// and it's an error, if it hasn't.
-  DIE *getLexicalBlockDIE(const DILexicalBlock *LB);
+  /// Get if available or create a new DW_TAG_lexical_block for the given
+  /// LexicalScope and attach DW_AT_low_pc/DW_AT_high_pc labels.
+  DIE *getOrCreateLexicalBlockDIE(LexicalScope *Scope, DIE &ParentDIE);
 
   /// constructVariableDIE - Construct a DIE for the given DbgVariable.
   DIE *constructVariableDIE(DbgVariable &DV, bool Abstract = false);
@@ -234,6 +229,11 @@ class DwarfCompileUnit final : public DwarfUnit {
   /// This instance of 'getOrCreateContextDIE()' can handle DILocalScope.
   DIE *getOrCreateContextDIE(const DIScope *Ty) override;
 
+  /// Get DW_TAG_lexical_block for the given DILexicalBlock if available,
+  /// or the most close parent DIE, if no correspoding DW_TAG_lexical_block
+  /// exists.
+  DIE *getLocalContextDIE(const DILexicalBlock *LB);
+
   /// Construct a DIE for this subprogram scope.
   DIE &constructSubprogramScopeDIE(const DISubprogram *Sub,
                                    LexicalScope *Scope);
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index b8502b455f587..557291c97265f 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -1229,12 +1229,13 @@ void DwarfDebug::beginModule(Module *M) {
         CU.getOrCreateGlobalVariableDIE(GV, sortGlobalExprs(GVMap[GV]));
     }
 
-    for (auto *Ty : CUNode->getEnumTypes())
+    for (auto *Ty : CUNode->getEnumTypes()) {
+      assert(!isa_and_nonnull<DILocalScope>(Ty->getScope()) &&
+             "Unexpected function-local entity in 'enums' CU field.");
       CU.getOrCreateTypeDIE(cast<DIType>(Ty));
+    }
 
     for (auto *Ty : CUNode->getRetainedTypes()) {
-      // The retained types array by design contains pointers to
-      // MDNodes rather than DIRefs. Unique them here.
       if (DIType *RT = dyn_cast<DIType>(Ty))
         // There is no point in force-emitting a forward declaration.
         CU.getOrCreateTypeDIE(RT);
@@ -1428,9 +1429,13 @@ void DwarfDebug::endModule() {
              "Unexpected function-local entity in 'imports' CU field.");
       CU->getOrCreateImportedEntityDIE(IE);
     }
+
+    // Emit function-local entities.
     for (const auto *D : CU->getDeferredLocalDecls()) {
       if (auto *IE = dyn_cast<DIImportedEntity>(D))
         CU->getOrCreateImportedEntityDIE(IE);
+      else if (auto *Ty = dyn_cast<DIType>(D))
+        CU->getOrCreateTypeDIE(Ty);
       else
         llvm_unreachable("Unexpected local retained node!");
     }
@@ -1528,6 +1533,8 @@ static const DILocalScope *getRetainedNodeScope(const MDNode *N) {
     S = L->getScope();
   else if (const auto *IE = dyn_cast<DIImportedEntity>(N))
     S = IE->getScope();
+  else if (const auto *T = dyn_cast<DIType>(N))
+    S = T->getScope();
   else
     llvm_unreachable("Unexpected retained node!");
 
diff --git a/llvm/lib/IR/DIBuilder.cpp b/llvm/lib/IR/DIBuilder.cpp
index ae1a1a37d7c74..242ca44794262 100644
--- a/llvm/lib/IR/DIBuilder.cpp
+++ b/llvm/lib/IR/DIBuilder.cpp
@@ -29,7 +29,7 @@ DIBuilder::DIBuilder(Module &m, bool AllowUnresolvedNodes, DICompileUnit *CU)
       AllowUnresolvedNodes(AllowUnresolvedNodes) {
   if (CUNode) {
     if (const auto &ETs = CUNode->getEnumTypes())
-      AllEnumTypes.assign(ETs.begin(), ETs.end());
+      EnumTypes.assign(ETs.begin(), ETs.end());
     if (const auto &RTs = CUNode->getRetainedTypes())
       AllRetainTypes.assign(RTs.begin(), RTs.end());
     if (const auto &GVs = CUNode->getGlobalVariables())
@@ -66,10 +66,10 @@ void DIBuilder::finalize() {
     return;
   }
 
-  if (!AllEnumTypes.empty())
+  if (!EnumTypes.empty())
     CUNode->replaceEnumTypes(MDTuple::get(
-        VMContext, SmallVector<Metadata *, 16>(AllEnumTypes.begin(),
-                                               AllEnumTypes.end())));
+        VMContext, SmallVector<Metadata *, 16>(EnumTypes.begin(),
+                                               EnumTypes.end())));
 
   SmallVector<Metadata *, 16> RetainValues;
   // Declarations and definitions of the same type may be retained. Some
@@ -334,10 +334,13 @@ DIDerivedType *DIBuilder::createTypedef(DIType *Ty, StringRef Name,
                                         DIScope *Context, uint32_t AlignInBits,
                                         DINode::DIFlags Flags,
                                         DINodeArray Annotations) {
-  return DIDerivedType::get(VMContext, dwarf::DW_TAG_typedef, Name, File,
-                            LineNo, getNonCompileUnitScope(Context), Ty, 0,
-                            AlignInBits, 0, std::nullopt, Flags, nullptr,
-                            Annotations);
+  auto *T =
+      DIDerivedType::get(VMContext, dwarf::DW_TAG_typedef, Name, File, LineNo,
+                         getNonCompileUnitScope(Context), Ty, 0, AlignInBits, 0,
+                         std::nullopt, Flags, nullptr, Annotations);
+  if (isa_and_nonnull<DILocalScope>(Context))
+    getSubprogramNodesTrackingVector(Context).emplace_back(T);
+  return T;
 }
 
 DIDerivedType *DIBuilder::createFriend(DIType *Ty, DIType *FriendTy) {
@@ -485,6 +488,8 @@ DICompositeType *DIBuilder::createClassType(
       OffsetInBits, Flags, Elements, 0, VTableHolder,
       cast_or_null<MDTuple>(TemplateParams), UniqueIdentifier);
   trackIfUnresolved(R);
+  if (isa_and_nonnull<DILocalScope>(Context))
+    getSubprogramNodesTrackingVector(Context).emplace_back(R);
   return R;
 }
 
@@ -498,6 +503,8 @@ DICompositeType *DIBuilder::createStructType(
       getNonCompileUnitScope(Context), DerivedFrom, SizeInBits, AlignInBits, 0,
       Flags, Elements, RunTimeLang, VTableHolder, nullptr, UniqueIdentifier);
   trackIfUnresolved(R);
+  if (isa_and_nonnull<DILocalScope>(Context))
+    getSubprogramNodesTrackingVector(Context).emplace_back(R);
   return R;
 }
 
@@ -510,6 +517,8 @@ DICompositeType *DIBuilder::createUnionType(
       getNonCompileUnitScope(Scope), nullptr, SizeInBits, AlignInBits, 0, Flags,
       Elements, RunTimeLang, nullptr, nullptr, UniqueIdentifier);
   trackIfUnresolved(R);
+  if (isa_and_nonnull<DILocalScope>(Scope))
+    getSubprogramNodesTrackingVector(Scope).emplace_back(R);
   return R;
 }
 
@@ -542,7 +551,10 @@ DICompositeType *DIBuilder::createEnumerationType(
       getNonCompileUnitScope(Scope), UnderlyingType, SizeInBits, AlignInBits, 0,
       IsScoped ? DINode::FlagEnumClass : DINode::FlagZero, Elements, 0, nullptr,
       nullptr, UniqueIdentifier);
-  AllEnumTypes.emplace_back(CTy);
+  if (isa_and_nonnull<DILocalScope>(Scope))
+    getSubprogramNodesTrackingVector(Scope).emplace_back(CTy);
+  else
+    EnumTypes.emplace_back(CTy);
   trackIfUnresolved(CTy);
   return CTy;
 }
@@ -640,6 +652,8 @@ DIBuilder::createForwardDecl(unsigned Tag, StringRef Name, DIScope *Scope,
       SizeInBits, AlignInBits, 0, DINode::FlagFwdDecl, nullptr, RuntimeLang,
       nullptr, nullptr, UniqueIdentifier);
   trackIfUnresolved(RetTy);
+  if (isa_and_nonnull<DILocalScope>(Scope))
+    getSubprogramNodesTrackingVector(Scope).emplace_back(RetTy);
   return RetTy;
 }
 
@@ -656,6 +670,8 @@ DICompositeType *DIBuilder::createReplaceableCompositeType(
           nullptr, Annotations)
           .release();
   trackIfUnresolved(RetTy);
+  if (isa_and_nonnull<DILocalScope>(Scope))
+    getSubprogramNodesTrackingVector(Scope).emplace_back(RetTy);
   return RetTy;
 }
 
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 293d443d754a4..8f929dc7f48ab 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -1384,9 +1384,9 @@ void Verifier::visitDISubprogram(const DISubprogram &N) {
     CheckDI(Node, "invalid retained nodes list", &N, RawNode);
     for (Metadata *Op : Node->operands()) {
       CheckDI(Op && (isa<DILocalVariable>(Op) || isa<DILabel>(Op) ||
-                     isa<DIImportedEntity>(Op)),
-              "invalid retained nodes, expected DILocalVariable, DILabel or "
-              "DIImportedEntity",
+                     isa<DIImportedEntity>(Op) || isa<DIType>(Op)),
+              "invalid retained nodes, expected DILocalVariable, DILabel, "
+              "DIImportedEntity or DIType",
               &N, Node, Op);
     }
   }
diff --git a/llvm/test/Bitcode/upgrade-cu-locals.ll b/llvm/test/Bitcode/upgrade-cu-locals.ll
index 9a590f0fc0774..7052ab379bea4 100644
--- a/llvm/test/Bitcode/upgrade-cu-locals.ll
+++ b/llvm/test/Bitcode/upgrade-cu-locals.ll
@@ -1,4 +1,4 @@
-; Test moving of local imports from DICompileUnit's 'imports' to DISubprogram's 'retainedNodes'
+; Test moving of local imports/enums from DICompileUnit to DISubprogram's 'retainedNodes'
 ;
 ; RUN: llvm-dis -o - %s.bc | FileCheck %s
 
@@ -31,31 +31,36 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo
 ; CHECK: !4 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t4"
 
 ; CHECK: !5 = !{}
-; CHECK: !6 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !7, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, imports: !5, nameTableKind: GNU)
-
-; CHECK: !14 = distinct !DISubprogram(name: "main", scope: !7, file: !7, line: 2, type: !15, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !18)
-; CHECK: !18 = !{!19}
-; CHECK: !19 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !20, entity: !23,
-; CHECK: !20 = !DILexicalBlock(scope: !21, file: !7, line: 7, column: 35)
-; CHECK: !21 = !DILexicalBlock(scope: !22, file: !7, line: 7, column: 35)
-; CHECK: !22 = !DILexicalBlock(scope: !14, file: !7, line: 7, column: 35)
-; CHECK: !23 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t5", scope: !20,
-
-; CHECK: !25 = distinct !DISubprogram(name: "f1", linkageName: "_Z2f1v", scope: !1, file: !1, line: 3, type: !26, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !28)
-; CHECK: !28 = !{!29, !32, !34}
-; CHECK: !29 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !25, entity: !30,
-; CHECK: !30 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t1",
-; CHECK: !32 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !25, entity: !33,
-; CHECK: !33 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t2",
-; CHECK: !34 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !25, entity: !35,
-; CHECK: !35 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t3",
-
-; CHECK: !40 = distinct !DISubprogram(name: "main2", scope: !7, file: !7, line: 10, type: !15, scopeLine: 10, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !41)
-; CHECK: !41 = !{!42, !44}
-; CHECK: !42 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !40, entity: !43,
-; CHECK: !43 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t6"
-; CHECK: !44 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !40, entity: !45,
-; CHECK: !45 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t7",
+; CHECK: !6 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !7, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !8, imports: !5, nameTableKind: GNU)
+; CHECK: !7 = !DIFile(filename: "b.cpp"
+; CHECK: !8 = !{!9}
+; CHECK: !9 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum2", scope: !6, file: !7, line: 4, size: 8, align: 8, elements: !5)
+
+; CHECK: !16 = distinct !DISubprogram(name: "main", scope: !7, file: !7, line: 2, type: !17, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !20)
+; CHECK: !20 = !{!21}
+; CHECK: !21 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !22, entity: !25,
+; CHECK: !22 = !DILexicalBlock(scope: !23, file: !7, line: 7, column: 35)
+; CHECK: !23 = !DILexicalBlock(scope: !24, file: !7, line: 7, column: 35)
+; CHECK: !24 = !DILexicalBlock(scope: !16, file: !7, line: 7, column: 35)
+; CHECK: !25 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t5", scope: !22,
+
+; CHECK: !27 = distinct !DISubprogram(name: "f1", linkageName: "_Z2f1v", scope: !1, file: !1, line: 3, type: !28, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !30)
+; CHECK: !30 = !{!31, !34, !36}
+; CHECK: !31 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !27, entity: !32,
+; CHECK: !32 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t1",
+; CHECK: !34 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !27, entity: !35,
+; CHECK: !35 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t2",
+; CHECK: !36 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !27, entity: !37,
+; CHECK: !37 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t3",
+
+; CHECK: !42 = distinct !DISubprogram(name: "main2", scope: !7, file: !7, line: 10, type: !17, scopeLine: 10, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !43)
+; CHECK: !43 = !{!44, !46, !48, !49}
+; CHECK: !44 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !42, entity: !45,
+; CHECK: !45 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t6"
+; CHECK: !46 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !42, entity: !47,
+; CHECK: !47 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t7",
+; CHECK: !48 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum", scope: !42, file: !7, line: 3, size: 8, align: 8, elements: !5)
+; CHECK: !49 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum3", scope: !42, file: !7, line: 5, size: 8, align: 8, elements: !5)
 
 
 !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, imports: !2, nameTableKind: GNU)
@@ -82,7 +87,7 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo
 ; Leave t4 in CU
 !14 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !0, entity: !15, file: !1, line: 3)
 !15 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t4", scope: !0, file: !1, line: 1, size: 8, flags: DIFlagTypePassByValue, elements: !7, identifier: "_ZTSN2ns2t4E")
-!16 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !17, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, imports: !18, nameTableKind: GNU)
+!16 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !17, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, imports: !18, enums: !50, nameTableKind: GNU)
 !17 = !DIFile(filename: "b.cpp", directory: "/")
 !18 = !{!19, !28, !31}
 
@@ -116,3 +121,12 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo
 !41 = distinct !DILocation(line: 3, column: 3, scope: !23)
 !42 = !DILocation(line: 3, column: 41, scope: !4, inlinedAt: !41)
 !43 = !DILocation(line: 4, column: 1, scope: !23)
+
+!50 = !{!51, !52, !53}
+; Move to main2
+!51 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum", scope: !29, file: !17, line: 3, size: 8, align: 8, elements: !7)
+; Leave in b.cpp's CU
+!52 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum2", scope: !16, file: !17, line: 4, size: 8, align: 8, elements: !7)
+; Move to main2
+!53 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum3", scope: !29, file: !17, line: 5, size: 8, align: 8, elements: !7)
+
diff --git a/llvm/test/Bitcode/upgrade-cu-locals.ll.bc b/llvm/test/Bitcode/upgrade-cu-locals.ll.bc
index 9d0ea8fd9a3704ea1fb3a92250500addc75f0587..f73d05668af5b16f9ee03e400e2877faab339a0d 100644
GIT binary patch
delta 1080
zcmYk5e`p(J7{{NxT<<RFwOop6V|rQMH7e;k%bOqVBKYItmQkU&t8>^v2T79^Yj3eX
zCa4ST^$4!hY-CFq`$I=`D*iELuKy_e=R!ISm`cGRL56hTA0ibxhWMA%_q{6H%X`oB
z-1ohE-p}*!zJ}+nr}PA4Tzv8e=ETMO&yU}{_3PDJt*yNt=um-!|A&k}H9J|6zN*x)
zz`pltPnw1RR8~Q?N^cBEwg8N+ilr9PDjEQ4=T<*p+z#fi@<&_{iTssC`dw0W77=EH
z*c_9SV-gA2gRx8VvGQFm-JQ(6b&5U_YNDso)49lW?sWONHl>_yJF*18yXh_f*wo~p
zm<*emHz?jF4h4_b4>b}uctVLpliLG87yzu1pRs>l{*5sRvWb=j0!DRwX=wDE!o5S^
zOf<4d(z>W#%y#_LCnWZe`V1s?y-g$s)sK#yRhD8G6zL2mOMNo{bO~2Hg=K}-PyV<3
zq#>kxqe*`rGx=)KWD%^aF+h`3j$?WG#v>f#G%w}o32^!&9A2suj-yWr%l!{I44%|a
zL4<$~0r4859CHsjn1Ia@0;sw`ra%slionW?#uo{nWGL`pFMPtkj$s~}XJN(nVGSA#
z4zOq@#XFE-No&eR!r!V%R>}e1C})ip;CBXV%(E=PadK-*f}<Bi)~=kQK=3&0zR$P?
zFab!z?O}~vR!8z0DHMT0x@`|!-w+nJg$D{euBG5HOG<>AheRU;84oDiwJ=Y7|E^bU
zPf#aaZ~liW_I^`?Z3Zt9R(*qaScJ9SUwv_3Q<kDq+B#R0>fX~?T-W%k-X<>8UgL=j
zd}ekQW|i;f7Vq_tsi(X3-JsT;L+v-o1XsyBp4;T7(%n-D;dAR-CS<UohPl7N`l(K4
zr0Ys`qSe~^Wumbb`*GsCKMuR^K<6vz%<9a-OTmGKYrRRW?O0>w{Q6hxpZ$6M`h_o>
zw~J42>AA|JXZ&N&#J$xHv5k3on>Xg(ukv3b;h>Ln<a)b(XM#LwT?+4icV}nkh0TA6
zo&B4bdNC2MEc==)o>a+U<_nRGCQVoQRnT!plNqI)a{8Ff@)_sAV5(!c!vJ$=_ozlX
z<-P*D&c#bd?4=WAs{s;qIV!309p_*u{7g7Dd~zy26wOR#W081lI2)ci5Q#<(#PJXw
bjz`1MY<74kUf$<AES)45abZx`X{vt#JnkA(

delta 1011
zcmYjPT}&fY6h1R;?+nAR8K7ukpbT3~NOmQ+|Ckt^#_fv6bXy??4aQDuk;YOAbd5%0
zrv<}iQ#P<|%$oR6v-sc!%?9*sAEv1*sKCpvhU}79AB@J-1png8D&A>B@aE3>`OdxH
z`DW%Ebf_IG{nX^@Bi~TU>Tgesr^SEYyYtha=j~yWY!HBg<7MPOE-e79rviCYP}RO~
z{gy^qhXoMu+C9!tM`Yd7K=*rCbF3X;n<h+~OH`|YIw-tvv~}+*HOenH<t0qpa4y8=
zHS>ZdEU$I902B_4N`Kn>%1fgo0Y`Gqk@ou2o|DYupHkvUM?9G>Oq(ubP_5n(a4sm=
zSie0NwAk3&#7VXnE9Y+cR78YGZ0=kSKu|xb4S$DwriFb<1z>Yhhyjpv9A9-m_myaT
zL(g*3@Li&q)A9VPEq8uL*&bj)mDsk61ggK}NN7uR?PWy%O-xFsCjs;lE_N)IMV>_D
zKJ<VL#yUlNktuhbP{;%YAPW0`^xgcQLag7?M(?gN<`w4afdQb^OCrcbyRM_~7ovJk
zNHh~b24E!gv^6YS+3S+7BYC33XFb+2(9pq;LKfhUKA1%l7^Y!o3AQ_nWw=G*URruh
zMutPA&9iE$Y~eK#oI*-qW-;T0g%$|!2;gj`Q;a5?4q3o2P@nm%-O&sQe%1rxYjsbU
z=Cxy{^=ua}TlJP3D30MR*uj5hdmoe^XJpi20TX^o;0UY4X?z)1s<H2RgJn{|6$#z<
zR;n`*f?W)!5|HZNntA7khsjPT59ziZB&B9$Lu%dtZpAW|tfCTdC6Y?b=`i<!I2Ng*
zNfkNTu9e33_iOjYx4+mM|MHjft$U@G=auN@>vPZ8&dgmG{g!t_+n1JiKi&Q4*X8vq
zpX}VuKUzy9i#hYNWpjYqoMKH>Ktt-x<rKPch!#4@&scBPKP}i01Gz9h`{B{i(UTAU
zA-4KAF_pZ}UMzRsq;W(hvxgQP(I6jBp>;^$Xi)Hp1+A%zQVaK+&J4<A;u~v5YD|{n
zlyG{H{@geu1nPyCNvpFYYBLlg1s69s5SfVLsK@JZx`(jG?sT~nm&;-I+LfWlT#gBk
cSBb^kZo4~HxXhgw;!~;V+4S6CBGKsbANlGKqW}N^

diff --git a/llvm/test/DebugInfo/Generic/inlined-local-type.ll b/llvm/test/DebugInfo/Generic/inlined-local-type.ll
new file mode 100644
index 0000000000000..a6f7d294de928
--- /dev/null
+++ b/llvm/test/DebugInfo/Generic/inlined-local-type.ll
@@ -0,0 +1,128 @@
+; RUN: %llc_dwarf -O0 -filetype=obj < %s | llvm-dwarfdump -debug-info - | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s
+; REQUIRES: object-emission
+
+; inline __attribute__((always_inline))
+; int removed() { struct A {int i;}; struct A a; return a.i++; }
+;
+; __attribute__((always_inline))
+; int not_removed() { struct B {int i;}; struct B b; return b.i++; }
+;
+; int foo() { return removed() + not_removed(); }}
+
+; Ensure that function-local types have the correct subprogram parent even if
+; those subprograms are inlined.
+
+; CHECK: DW_TAG_compile_unit
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_AT_abstract_origin	({{0x.*}} "not_removed")
+; CHECK:     DW_TAG_variable
+; CHECK:     NULL
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_AT_name	("removed")
+; CHECK: [[A:0x.*]]: DW_TAG_structure_type
+; CHECK:       DW_AT_name	("A")
+; CHECK:       DW_TAG_member
+; CHECK:       NULL
+; CHECK:     DW_TAG_variable
+; CHECK:       DW_AT_type	([[A]] "A")
+; CHECK:     NULL
+; CHECK:   DW_TAG_base_type
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_AT_name	("not_removed")
+; CHECK: [[B:0x.*]]: DW_TAG_structure_type
+; CHECK:       DW_AT_name	("B")
+; CHECK:       DW_TAG_member
+; CHECK:       NULL
+; CHECK:     DW_TAG_variable
+; CHECK:       DW_AT_type	([[B]] "B")
+; CHECK:     NULL
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_TAG_inlined_subroutine
+; CHECK:       DW_TAG_variable
+; CHECK:       NULL
+; CHECK:     DW_TAG_inlined_subroutine
+; CHECK:       DW_TAG_variable
+; CHECK:       NULL
+; CHECK:     NULL
+; CHECK:   NULL
+
+%struct.B = type { i32 }
+%struct.A = type { i32 }
+
+define dso_local i32 @not_removed() !dbg !12 {
+  %1 = alloca %struct.B, align 4
+  call void @llvm.dbg.declare(metadata %struct.B* %1, metadata !18, metadata !DIExpression()), !dbg !22
+  %2 = getelementptr inbounds %struct.B, %struct.B* %1, i32 0, i32 0, !dbg !23
+  %3 = load i32, i32* %2, align 4, !dbg !24
+  %4 = add nsw i32 %3, 1, !dbg !24
+  store i32 %4, i32* %2, align 4, !dbg !24
+  ret i32 %3, !dbg !25
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+define dso_local i32 @foo() !dbg !26 {
+  %1 = alloca %struct.A, align 4
+  %2 = alloca %struct.B, align 4
+  call void @llvm.dbg.declare(metadata %struct.A* %1, metadata !27, metadata !DIExpression()), !dbg !32
+  %3 = getelementptr inbounds %struct.A, %struct.A* %1, i32 0, i32 0, !dbg !34
+  %4 = load i32, i32* %3, align 4, !dbg !35
+  %5 = add nsw i32 %4, 1, !dbg !35
+  store i32 %5, i32* %3, align 4, !dbg !35
+  call void @llvm.dbg.declare(metadata %struct.B* %2, metadata !18, metadata !DIExpression()), !dbg !36
+  %6 = getelementptr inbounds %struct.B, %struct.B* %2, i32 0, i32 0, !dbg !38
+  %7 = load i32, i32* %6, align 4, !dbg !39
+  %8 = add nsw i32 %7, 1, !dbg !39
+  store i32 %8, i32* %6, align 4, !dbg !39
+  %9 = add nsw i32 %4, %7, !dbg !40
+  ret i32 %9, !dbg !41
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3, !4, !5, !6, !7, !8, !9, !10}
+!llvm.ident = !{!11}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "inlined-local-type.cpp", directory: "")
+!2 = !{i32 7, !"Dwarf Version", i32 4}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"wchar_size", i32 4}
+!5 = !{i32 1, !"branch-target-enforcement", i32 0}
+!6 = !{i32 1, !"sign-return-address", i32 0}
+!7 = !{i32 1, !"sign-return-address-all", i32 0}
+!8 = !{i32 1, !"sign-return-address-with-bkey", i32 0}
+!9 = !{i32 7, !"uwtable", i32 1}
+!10 = !{i32 7, !"frame-pointer", i32 1}
+!11 = !{!"clang version 14.0.0"}
+!12 = distinct !DISubprogram(name: "not_removed", scope: !13, file: !13, line: 5, type: !14, scopeLine: 5, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !43)
+!13 = !DIFile(filename: "inlined-local-type.cpp", directory: "")
+!14 = !DISubroutineType(types: !15)
+!15 = !{!16}
+!16 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!17 = !{}
+!18 = !DILocalVariable(name: "b", scope: !12, file: !13, line: 5, type: !19)
+!19 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "B", scope: !12, file: !13, line: 5, size: 32, elements: !20)
+!20 = !{!21}
+!21 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !19, file: !13, line: 5, baseType: !16, size: 32)
+!22 = !DILocation(line: 5, column: 49, scope: !12)
+!23 = !DILocation(line: 5, column: 61, scope: !12)
+!24 = !DILocation(line: 5, column: 62, scope: !12)
+!25 = !DILocation(line: 5, column: 52, scope: !12)
+!26 = distinct !DISubprogram(name: "foo", scope: !13, file: !13, line: 7, type: !14, scopeLine: 7, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !17)
+!27 = !DILocalVariable(name: "a", scope: !28, file: !13, line: 2, type: !29)
+!28 = distinct !DISubprogram(name: "removed", scope: !13, file: !13, line: 2, type: !14, scopeLine: 2, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !42)
+!29 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A", scope: !28, file: !13, line: 2, size: 32, elements: !30)
+!30 = !{!31}
+!31 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !29, file: !13, line: 2, baseType: !16, size: 32)
+!32 = !DILocation(line: 2, column: 45, scope: !28, inlinedAt: !33)
+!33 = distinct !DILocation(line: 7, column: 20, scope: !26)
+!34 = !DILocation(line: 2, column: 57, scope: !28, inlinedAt: !33)
+!35 = !DILocation(line: 2, column: 58, scope: !28, inlinedAt: !33)
+!36 = !DILocation(line: 5, column: 49, scope: !12, inlinedAt: !37)
+!37 = distinct !DILocation(line: 7, column: 32, scope: !26)
+!38 = !DILocation(line: 5, column: 61, scope: !12, inlinedAt: !37)
+!39 = !DILocation(line: 5, column: 62, scope: !12, inlinedAt: !37)
+!40 = !DILocation(line: 7, column: 30, scope: !26)
+!41 = !DILocation(line: 7, column: 13, scope: !26)
+!42 = !{!29}
+!43 = !{!19}
diff --git a/llvm/test/DebugInfo/Generic/lexical-block-retained-types.ll b/llvm/test/DebugInfo/Generic/lexical-block-retained-types.ll
new file mode 100644
index 0000000000000..64de1323c98c6
--- /dev/null
+++ b/llvm/test/DebugInfo/Generic/lexical-block-retained-types.ll
@@ -0,0 +1,55 @@
+; RUN: %llc_dwarf -O0 -filetype=obj < %s | llvm-dwarfdump - | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s
+
+; Test that retained unused (unreferenced) types emission.
+
+; Compiled from
+; $ clang -cc1 -debug-info-kind=unused-types test.cpp -emit-llvm
+
+; void test_unused() {
+;   struct Y {};
+;   {
+;     struct X {};
+;   }
+; }
+
+; CHECK: DW_TAG_compile_unit
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_AT_name	("test_unused")
+; CHECK:     DW_TAG_structure_type
+; CHECK:       DW_AT_name	("Y")
+
+; FIXME: here should be DW_TAG_lexical_block as a parent of structure 'X'.
+; But it's not possible to reliably emit a lexical block for which a LexicalScope
+; wasn't created, so we just fallback to the most close parent DIE
+; (see DwarfCompileUnit::getOrCreateLexicalBlockDIE() for details).
+
+; CHECK:     DW_TAG_structure_type
+; CHECK:       DW_AT_name	("X")
+; CHECK:     NULL
+; CHECK:   NULL
+
+define dso_local void @_Z11test_unusedv() !dbg !5 {
+entry:
+  ret void, !dbg !16
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!13, !14}
+!llvm.ident = !{!15}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 15.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, retainedTypes: !2, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "<stdin>", directory: "/")
+!2 = !{!3, !10}
+!3 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Y", scope: !5, file: !4, line: 2, size: 8, flags: DIFlagTypePassByValue, elements: !8)
+!4 = !DIFile(filename: "test.cpp", directory: "/")
+!5 = distinct !DISubprogram(name: "test_unused", linkageName: "_Z11test_unusedv", scope: !4, file: !4, line: 1, type: !6, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !9)
+!6 = !DISubroutineType(types: !7)
+!7 = !{null}
+!8 = !{}
+!9 = !{!3, !10}
+!10 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "X", scope: !11, file: !4, line: 4, size: 8, flags: DIFlagTypePassByValue, elements: !8)
+!11 = distinct !DILexicalBlock(scope: !5, file: !4, line: 3, column: 3)
+!13 = !{i32 2, !"Debug Info Version", i32 3}
+!14 = !{i32 1, !"wchar_size", i32 4}
+!15 = !{!"clang version 15.0.0"}
+!16 = !DILocation(line: 6, column: 1, scope: !5)
diff --git a/llvm/test/DebugInfo/Generic/lexical-block-types.ll b/llvm/test/DebugInfo/Generic/lexical-block-types.ll
new file mode 100644
index 0000000000000..aef59984351a7
--- /dev/null
+++ b/llvm/test/DebugInfo/Generic/lexical-block-types.ll
@@ -0,0 +1,425 @@
+; RUN: %llc_dwarf -O0 -filetype=obj < %s | llvm-dwarfdump -debug-info - | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s
+; REQUIRES: object-emission
+
+; inline __attribute__((always_inline))
+; void removed() {
+;   struct A1 { int i; };
+;   typedef int Int1;
+;   {
+;     struct I1 { Int1 j; };
+;     struct C1 { typedef char Char1; Char1 c; };
+;     A1 a1; a1.i++;
+;     {
+;       I1 i1; i1.j++;
+;       C1 c1; c1.c++;
+;     }
+;   }
+; }
+;
+; __attribute__((always_inline))
+; void not_removed() {
+;   struct A2 { int i; };
+;   typedef int Int2;
+;   {
+;     struct I2 { Int2 j; };
+;     struct C2 { typedef char Char2; Char2 c; };
+;     A2 a2; a2.i++;
+;     {
+;       I2 i2; i2.j++;
+;       C2 c2; c2.c++;
+;     }
+;   }
+; }
+;
+; void foo() {
+;   struct A3 { int i; };
+;   typedef int Int3;
+;   {
+;     struct I3 { Int3 j; };
+;     {
+;       struct C3 { typedef char Char3; Char3 c; };
+;       A3 a3; a3.i++;
+;       {
+;         I3 i3; i3.j++;
+;         C3 c3; c3.c++;
+;       }
+;     }
+;   }
+;   removed();
+;   not_removed();
+; }
+;
+; CHECK: DW_TAG_compile_unit
+
+; Out-of-line definition of `not_removed()` shouldn't contain any debug info for types.
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_AT_abstract_origin	{{.*}} "_Z11not_removedv"
+; CHECK:     DW_TAG_lexical_block
+; CHECK:       DW_TAG_variable
+; CHECK:         DW_AT_abstract_origin	{{.*}} "a2"
+; CHECK:       DW_TAG_lexical_block
+; CHECK:         DW_TAG_variable
+; CHECK:           DW_AT_abstract_origin	{{.*}} "i2"
+; CHECK:         DW_TAG_variable
+; CHECK:           DW_AT_abstract_origin	{{.*}} "c2"
+; CHECK:         NULL
+; CHECK:       NULL
+; CHECK:     NULL
+
+; Abstract definition of `removed()`.
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_AT_name	("removed")
+; CHECK:     DW_AT_inline	(DW_INL_inlined)
+
+; I1 and C1 defined in the first lexical block, typedef Char1 is a child of C1.
+; CHECK:     DW_TAG_lexical_block
+; CHECK:       DW_TAG_variable
+; CHECK:         DW_AT_name	("a1")
+; CHECK:         DW_AT_type	{{.*}} "A1"
+; CHECK:       DW_TAG_lexical_block
+; CHECK:         DW_TAG_variable
+; CHECK:           DW_AT_type	{{.*}} "I1"
+; CHECK:         DW_TAG_variable
+; CHECK:           DW_AT_type	{{.*}} "C1"
+; CHECK:         NULL
+; CHECK:       DW_TAG_structure_type
+; CHECK:         DW_AT_name	("I1")
+; CHECK:         DW_TAG_member
+; CHECK:           DW_AT_type	{{.*}} "Int1"
+; CHECK:         NULL
+; CHECK:       DW_TAG_structure_type
+; CHECK:         DW_AT_name	("C1")
+; CHECK:         DW_TAG_member
+; CHECK:           DW_AT_type	{{.*}} "Char1"
+; CHECK:         DW_TAG_typedef
+; CHECK:           DW_AT_name	("Char1")
+; CHECK:         NULL
+; CHECK:       NULL
+
+; A1 and typedef Int1 defined in the subprogram scope.
+; CHECK:     DW_TAG_structure_type
+; CHECK:       DW_AT_name	("A1")
+; CHECK:       DW_TAG_member
+; CHECK:       NULL
+; CHECK:     DW_TAG_typedef
+; CHECK:       DW_AT_name	("Int1")
+; CHECK:     NULL
+
+; CHECK:   DW_TAG_base_type
+; CHECK:   DW_TAG_base_type
+
+; Abstract definition of `not_removed()`.
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_AT_name	("not_removed")
+; CHECK:     DW_AT_inline	(DW_INL_inlined)
+
+; I2 and C2 defined in the first lexical block, typedef Char2 is a child of C2.
+; CHECK:     DW_TAG_lexical_block
+; CHECK:       DW_TAG_variable
+; CHECK:         DW_AT_name	("a2")
+; CHECK:         DW_AT_type	{{.*}} "A2"
+; CHECK:       DW_TAG_lexical_block
+; CHECK:         DW_TAG_variable
+; CHECK:           DW_AT_name	("i2")
+; CHECK:           DW_AT_type	{{.*}} "I2"
+; CHECK:         DW_TAG_variable
+; CHECK:           DW_AT_name	("c2")
+; CHECK:           DW_AT_type	{{.*}} "C2"
+; CHECK:         NULL
+; CHECK:       DW_TAG_structure_type
+; CHECK:         DW_AT_name	("I2")
+; CHECK:         DW_TAG_member
+; CHECK:           DW_AT_type	{{.*}} "Int2"
+; CHECK:         NULL
+; CHECK:       DW_TAG_structure_type
+; CHECK:         DW_AT_name	("C2")
+; CHECK:         DW_TAG_member
+; CHECK:           DW_AT_type	{{.*}} "Char2"
+; CHECK:         DW_TAG_typedef
+; CHECK:           DW_AT_name	("Char2")
+; CHECK:         NULL
+; CHECK:       NULL
+
+; A2 and typedef Int2 defined in subprogram scope.
+; CHECK:     DW_TAG_structure_type
+; CHECK:       DW_AT_name	("A2")
+; CHECK:       DW_TAG_member
+; CHECK:       NULL
+; CHECK:     DW_TAG_typedef
+; CHECK:       DW_AT_name	("Int2")
+; CHECK:     NULL
+
+; Definition of `foo()`.
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_AT_name	("foo")
+
+; CHECK:     DW_TAG_lexical_block
+; CHECK:       DW_TAG_lexical_block
+; CHECK:         DW_TAG_variable
+; CHECK:           DW_AT_name	("a3")
+; CHECK:           DW_AT_type	{{.*}} "A3"
+; CHECK:         DW_TAG_lexical_block
+; CHECK:           DW_TAG_variable
+; CHECK:             DW_AT_name	("i3")
+; CHECK:             DW_AT_type	{{.*}} "I3"
+; CHECK:           DW_TAG_variable
+; CHECK:             DW_AT_name	("c3")
+; CHECK:             DW_AT_type	{{.*}} "C3"
+; CHECK:           NULL
+
+; C3 has the inner lexical block scope, typedef Char3 is a child of C3.
+; CHECK:         DW_TAG_structure_type
+; CHECK:           DW_AT_name	("C3")
+; CHECK:           DW_TAG_member
+; CHECK:             DW_AT_type	{{.*}} "Char3"
+; CHECK:           DW_TAG_typedef
+; CHECK:             DW_AT_name	("Char3")
+; CHECK:           NULL
+; CHECK:         NULL
+
+; I3 has the outer lexical block scope.
+; CHECK:       DW_TAG_structure_type
+; CHECK:         DW_AT_name	("I3")
+; CHECK:         DW_TAG_member
+; CHECK:           DW_AT_type	{{.*}} "Int3"
+; CHECK:         NULL
+; CHECK:       NULL
+
+; CHECK:     DW_TAG_inlined_subroutine
+; CHECK:       DW_AT_abstract_origin	{{.*}} "_Z7removedv"
+; CHECK:       DW_TAG_lexical_block
+; CHECK:         DW_TAG_variable
+; CHECK:         DW_TAG_lexical_block
+; CHECK:           DW_TAG_variable
+; CHECK:           DW_TAG_variable
+; CHECK:           NULL
+; CHECK:         NULL
+; CHECK:       NULL
+
+; CHECK:     DW_TAG_inlined_subroutine
+; CHECK:       DW_AT_abstract_origin	{{.*}} "_Z11not_removedv"
+; CHECK:       DW_TAG_lexical_block
+; CHECK:         DW_TAG_variable
+; CHECK:         DW_TAG_lexical_block
+; CHECK:           DW_TAG_variable
+; CHECK:           DW_TAG_variable
+; CHECK:           NULL
+; CHECK:         NULL
+; CHECK:       NULL
+
+; A3 and Int3 defined within the subprogam scope.
+; CHECK:     DW_TAG_structure_type
+; CHECK:       DW_AT_name	("A3")
+; CHECK:       DW_TAG_member
+; CHECK:       NULL
+; CHECK:     DW_TAG_typedef
+; CHECK:       DW_AT_name	("Int3")
+; CHECK:     NULL
+; CHECK:   NULL
+
+%struct.A2 = type { i32 }
+%struct.I2 = type { i32 }
+%struct.C2 = type { i8 }
+%struct.A1 = type { i32 }
+%struct.I1 = type { i32 }
+%struct.C1 = type { i8 }
+%struct.A3 = type { i32 }
+%struct.I3 = type { i32 }
+%struct.C3 = type { i8 }
+
+define dso_local void @_Z11not_removedv() !dbg !8 {
+entry:
+  %a2 = alloca %struct.A2, align 4
+  %i2 = alloca %struct.I2, align 4
+  %c2 = alloca %struct.C2, align 1
+  call void @llvm.dbg.declare(metadata %struct.A2* %a2, metadata !12, metadata !DIExpression()), !dbg !18
+  %i = getelementptr inbounds %struct.A2, %struct.A2* %a2, i32 0, i32 0, !dbg !19
+  %0 = load i32, i32* %i, align 4, !dbg !20
+  %inc = add nsw i32 %0, 1, !dbg !20
+  store i32 %inc, i32* %i, align 4, !dbg !20
+  call void @llvm.dbg.declare(metadata %struct.I2* %i2, metadata !21, metadata !DIExpression()), !dbg !27
+  %j = getelementptr inbounds %struct.I2, %struct.I2* %i2, i32 0, i32 0, !dbg !28
+  %1 = load i32, i32* %j, align 4, !dbg !29
+  %inc1 = add nsw i32 %1, 1, !dbg !29
+  store i32 %inc1, i32* %j, align 4, !dbg !29
+  call void @llvm.dbg.declare(metadata %struct.C2* %c2, metadata !30, metadata !DIExpression()), !dbg !36
+  %c = getelementptr inbounds %struct.C2, %struct.C2* %c2, i32 0, i32 0, !dbg !37
+  %2 = load i8, i8* %c, align 1, !dbg !38
+  %inc2 = add i8 %2, 1, !dbg !38
+  store i8 %inc2, i8* %c, align 1, !dbg !38
+  ret void, !dbg !39
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+define dso_local void @_Z3foov() !dbg !40 {
+entry:
+  %a1.i = alloca %struct.A1, align 4
+  %i1.i = alloca %struct.I1, align 4
+  %c1.i = alloca %struct.C1, align 1
+  %a2.i = alloca %struct.A2, align 4
+  %i2.i = alloca %struct.I2, align 4
+  %c2.i = alloca %struct.C2, align 1
+  %a3 = alloca %struct.A3, align 4
+  %i3 = alloca %struct.I3, align 4
+  %c3 = alloca %struct.C3, align 1
+  call void @llvm.dbg.declare(metadata %struct.A3* %a3, metadata !41, metadata !DIExpression()), !dbg !47
+  %i = getelementptr inbounds %struct.A3, %struct.A3* %a3, i32 0, i32 0, !dbg !48
+  %0 = load i32, i32* %i, align 4, !dbg !49
+  %inc = add nsw i32 %0, 1, !dbg !49
+  store i32 %inc, i32* %i, align 4, !dbg !49
+  call void @llvm.dbg.declare(metadata %struct.I3* %i3, metadata !50, metadata !DIExpression()), !dbg !56
+  %j = getelementptr inbounds %struct.I3, %struct.I3* %i3, i32 0, i32 0, !dbg !57
+  %1 = load i32, i32* %j, align 4, !dbg !58
+  %inc1 = add nsw i32 %1, 1, !dbg !58
+  store i32 %inc1, i32* %j, align 4, !dbg !58
+  call void @llvm.dbg.declare(metadata %struct.C3* %c3, metadata !59, metadata !DIExpression()), !dbg !64
+  %c = getelementptr inbounds %struct.C3, %struct.C3* %c3, i32 0, i32 0, !dbg !65
+  %2 = load i8, i8* %c, align 1, !dbg !66
+  %inc2 = add i8 %2, 1, !dbg !66
+  store i8 %inc2, i8* %c, align 1, !dbg !66
+  call void @llvm.dbg.declare(metadata %struct.A1* %a1.i, metadata !67, metadata !DIExpression()), !dbg !73
+  %i.i3 = getelementptr inbounds %struct.A1, %struct.A1* %a1.i, i32 0, i32 0, !dbg !75
+  %3 = load i32, i32* %i.i3, align 4, !dbg !76
+  %inc.i4 = add nsw i32 %3, 1, !dbg !76
+  store i32 %inc.i4, i32* %i.i3, align 4, !dbg !76
+  call void @llvm.dbg.declare(metadata %struct.I1* %i1.i, metadata !77, metadata !DIExpression()), !dbg !83
+  %j.i5 = getelementptr inbounds %struct.I1, %struct.I1* %i1.i, i32 0, i32 0, !dbg !84
+  %4 = load i32, i32* %j.i5, align 4, !dbg !85
+  %inc1.i6 = add nsw i32 %4, 1, !dbg !85
+  store i32 %inc1.i6, i32* %j.i5, align 4, !dbg !85
+  call void @llvm.dbg.declare(metadata %struct.C1* %c1.i, metadata !86, metadata !DIExpression()), !dbg !91
+  %c.i7 = getelementptr inbounds %struct.C1, %struct.C1* %c1.i, i32 0, i32 0, !dbg !92
+  %5 = load i8, i8* %c.i7, align 1, !dbg !93
+  %inc2.i8 = add i8 %5, 1, !dbg !93
+  store i8 %inc2.i8, i8* %c.i7, align 1, !dbg !93
+  call void @llvm.dbg.declare(metadata %struct.A2* %a2.i, metadata !12, metadata !DIExpression()), !dbg !94
+  %i.i = getelementptr inbounds %struct.A2, %struct.A2* %a2.i, i32 0, i32 0, !dbg !96
+  %6 = load i32, i32* %i.i, align 4, !dbg !97
+  %inc.i = add nsw i32 %6, 1, !dbg !97
+  store i32 %inc.i, i32* %i.i, align 4, !dbg !97
+  call void @llvm.dbg.declare(metadata %struct.I2* %i2.i, metadata !21, metadata !DIExpression()), !dbg !98
+  %j.i = getelementptr inbounds %struct.I2, %struct.I2* %i2.i, i32 0, i32 0, !dbg !99
+  %7 = load i32, i32* %j.i, align 4, !dbg !100
+  %inc1.i = add nsw i32 %7, 1, !dbg !100
+  store i32 %inc1.i, i32* %j.i, align 4, !dbg !100
+  call void @llvm.dbg.declare(metadata %struct.C2* %c2.i, metadata !30, metadata !DIExpression()), !dbg !101
+  %c.i = getelementptr inbounds %struct.C2, %struct.C2* %c2.i, i32 0, i32 0, !dbg !102
+  %8 = load i8, i8* %c.i, align 1, !dbg !103
+  %inc2.i = add i8 %8, 1, !dbg !103
+  store i8 %inc2.i, i8* %c.i, align 1, !dbg !103
+  ret void, !dbg !104
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "test.cpp", directory: "/")
+!2 = !{i32 7, !"Dwarf Version", i32 4}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"wchar_size", i32 4}
+!5 = !{i32 7, !"uwtable", i32 1}
+!6 = !{i32 7, !"frame-pointer", i32 2}
+!7 = !{!"clang version 14.0.0"}
+!8 = distinct !DISubprogram(name: "not_removed", linkageName: "_Z11not_removedv", scope: !1, file: !1, line: 17, type: !9, scopeLine: 17, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !105)
+!9 = !DISubroutineType(types: !10)
+!10 = !{null}
+!11 = !{}
+!12 = !DILocalVariable(name: "a2", scope: !13, file: !1, line: 23, type: !14)
+!13 = distinct !DILexicalBlock(scope: !8, file: !1, line: 20, column: 3)
+!14 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A2", scope: !8, file: !1, line: 18, size: 32, flags: DIFlagTypePassByValue, elements: !15)
+!15 = !{!16}
+!16 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !14, file: !1, line: 18, baseType: !17, size: 32)
+!17 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!18 = !DILocation(line: 23, column: 8, scope: !13)
+!19 = !DILocation(line: 23, column: 15, scope: !13)
+!20 = !DILocation(line: 23, column: 16, scope: !13)
+!21 = !DILocalVariable(name: "i2", scope: !22, file: !1, line: 25, type: !23)
+!22 = distinct !DILexicalBlock(scope: !13, file: !1, line: 24, column: 5)
+!23 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "I2", scope: !13, file: !1, line: 21, size: 32, flags: DIFlagTypePassByValue, elements: !24)
+!24 = !{!25}
+!25 = !DIDerivedType(tag: DW_TAG_member, name: "j", scope: !23, file: !1, line: 21, baseType: !26, size: 32)
+!26 = !DIDerivedType(tag: DW_TAG_typedef, name: "Int2", scope: !8, file: !1, line: 19, baseType: !17)
+!27 = !DILocation(line: 25, column: 10, scope: !22)
+!28 = !DILocation(line: 25, column: 17, scope: !22)
+!29 = !DILocation(line: 25, column: 18, scope: !22)
+!30 = !DILocalVariable(name: "c2", scope: !22, file: !1, line: 26, type: !31)
+!31 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "C2", scope: !13, file: !1, line: 22, size: 8, flags: DIFlagTypePassByValue, elements: !32)
+!32 = !{!33}
+!33 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !31, file: !1, line: 22, baseType: !34, size: 8)
+!34 = !DIDerivedType(tag: DW_TAG_typedef, name: "Char2", scope: !31, file: !1, line: 22, baseType: !35)
+!35 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!36 = !DILocation(line: 26, column: 10, scope: !22)
+!37 = !DILocation(line: 26, column: 17, scope: !22)
+!38 = !DILocation(line: 26, column: 18, scope: !22)
+!39 = !DILocation(line: 29, column: 1, scope: !8)
+!40 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !1, file: !1, line: 31, type: !9, scopeLine: 31, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !107)
+!41 = !DILocalVariable(name: "a3", scope: !42, file: !1, line: 38, type: !44)
+!42 = distinct !DILexicalBlock(scope: !43, file: !1, line: 36, column: 5)
+!43 = distinct !DILexicalBlock(scope: !40, file: !1, line: 34, column: 3)
+!44 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A3", scope: !40, file: !1, line: 32, size: 32, flags: DIFlagTypePassByValue, elements: !45)
+!45 = !{!46}
+!46 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !44, file: !1, line: 32, baseType: !17, size: 32)
+!47 = !DILocation(line: 38, column: 10, scope: !42)
+!48 = !DILocation(line: 38, column: 17, scope: !42)
+!49 = !DILocation(line: 38, column: 18, scope: !42)
+!50 = !DILocalVariable(name: "i3", scope: !51, file: !1, line: 40, type: !52)
+!51 = distinct !DILexicalBlock(scope: !42, file: !1, line: 39, column: 7)
+!52 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "I3", scope: !43, file: !1, line: 35, size: 32, flags: DIFlagTypePassByValue, elements: !53)
+!53 = !{!54}
+!54 = !DIDerivedType(tag: DW_TAG_member, name: "j", scope: !52, file: !1, line: 35, baseType: !55, size: 32)
+!55 = !DIDerivedType(tag: DW_TAG_typedef, name: "Int3", scope: !40, file: !1, line: 33, baseType: !17)
+!56 = !DILocation(line: 40, column: 12, scope: !51)
+!57 = !DILocation(line: 40, column: 19, scope: !51)
+!58 = !DILocation(line: 40, column: 20, scope: !51)
+!59 = !DILocalVariable(name: "c3", scope: !51, file: !1, line: 41, type: !60)
+!60 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "C3", scope: !42, file: !1, line: 37, size: 8, flags: DIFlagTypePassByValue, elements: !61)
+!61 = !{!62}
+!62 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !60, file: !1, line: 37, baseType: !63, size: 8)
+!63 = !DIDerivedType(tag: DW_TAG_typedef, name: "Char3", scope: !60, file: !1, line: 37, baseType: !35)
+!64 = !DILocation(line: 41, column: 12, scope: !51)
+!65 = !DILocation(line: 41, column: 19, scope: !51)
+!66 = !DILocation(line: 41, column: 20, scope: !51)
+!67 = !DILocalVariable(name: "a1", scope: !68, file: !1, line: 8, type: !70)
+!68 = distinct !DILexicalBlock(scope: !69, file: !1, line: 5, column: 3)
+!69 = distinct !DISubprogram(name: "removed", linkageName: "_Z7removedv", scope: !1, file: !1, line: 2, type: !9, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !110)
+!70 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A1", scope: !69, file: !1, line: 3, size: 32, flags: DIFlagTypePassByValue, elements: !71, identifier: "_ZTSZ7removedvE2A1")
+!71 = !{!72}
+!72 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !70, file: !1, line: 3, baseType: !17, size: 32)
+!73 = !DILocation(line: 8, column: 8, scope: !68, inlinedAt: !74)
+!74 = distinct !DILocation(line: 45, column: 3, scope: !40)
+!75 = !DILocation(line: 8, column: 15, scope: !68, inlinedAt: !74)
+!76 = !DILocation(line: 8, column: 16, scope: !68, inlinedAt: !74)
+!77 = !DILocalVariable(name: "i1", scope: !78, file: !1, line: 10, type: !79)
+!78 = distinct !DILexicalBlock(scope: !68, file: !1, line: 9, column: 5)
+!79 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "I1", scope: !68, file: !1, line: 6, size: 32, flags: DIFlagTypePassByValue, elements: !80, identifier: "_ZTSZ7removedvE2I1")
+!80 = !{!81}
+!81 = !DIDerivedType(tag: DW_TAG_member, name: "j", scope: !79, file: !1, line: 6, baseType: !82, size: 32)
+!82 = !DIDerivedType(tag: DW_TAG_typedef, name: "Int1", scope: !69, file: !1, line: 4, baseType: !17)
+!83 = !DILocation(line: 10, column: 10, scope: !78, inlinedAt: !74)
+!84 = !DILocation(line: 10, column: 17, scope: !78, inlinedAt: !74)
+!85 = !DILocation(line: 10, column: 18, scope: !78, inlinedAt: !74)
+!86 = !DILocalVariable(name: "c1", scope: !78, file: !1, line: 11, type: !87)
+!87 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "C1", scope: !68, file: !1, line: 7, size: 8, flags: DIFlagTypePassByValue, elements: !88, identifier: "_ZTSZ7removedvE2C1")
+!88 = !{!89}
+!89 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !87, file: !1, line: 7, baseType: !90, size: 8)
+!90 = !DIDerivedType(tag: DW_TAG_typedef, name: "Char1", scope: !87, file: !1, line: 7, baseType: !35)
+!91 = !DILocation(line: 11, column: 10, scope: !78, inlinedAt: !74)
+!92 = !DILocation(line: 11, column: 17, scope: !78, inlinedAt: !74)
+!93 = !DILocation(line: 11, column: 18, scope: !78, inlinedAt: !74)
+!94 = !DILocation(line: 23, column: 8, scope: !13, inlinedAt: !95)
+!95 = distinct !DILocation(line: 46, column: 3, scope: !40)
+!96 = !DILocation(line: 23, column: 15, scope: !13, inlinedAt: !95)
+!97 = !DILocation(line: 23, column: 16, scope: !13, inlinedAt: !95)
+!98 = !DILocation(line: 25, column: 10, scope: !22, inlinedAt: !95)
+!99 = !DILocation(line: 25, column: 17, scope: !22, inlinedAt: !95)
+!100 = !DILocation(line: 25, column: 18, scope: !22, inlinedAt: !95)
+!101 = !DILocation(line: 26, column: 10, scope: !22, inlinedAt: !95)
+!102 = !DILocation(line: 26, column: 17, scope: !22, inlinedAt: !95)
+!103 = !DILocation(line: 26, column: 18, scope: !22, inlinedAt: !95)
+!104 = !DILocation(line: 47, column: 1, scope: !40)
+!105 = !{!14, !23, !26, !31}
+!107 = !{!44, !52, !55, !60}
+!110 = !{!70, !79, !82, !87}
diff --git a/llvm/test/DebugInfo/Generic/local-type-as-template-parameter.ll b/llvm/test/DebugInfo/Generic/local-type-as-template-parameter.ll
new file mode 100644
index 0000000000000..7999e434d139e
--- /dev/null
+++ b/llvm/test/DebugInfo/Generic/local-type-as-template-parameter.ll
@@ -0,0 +1,160 @@
+; RUN: %llc_dwarf -O0 -filetype=obj < %s              \
+; RUN:  | llvm-dwarfdump --show-children --name=foo - \
+; RUN:  | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s
+
+; The test ensures that AsmPrinter doesn't crashed compiling this.
+; It also demostrates misplacement for a local type (see PR55680 for details).
+
+; The test compiled from:
+
+; template<typename T>
+; struct A {
+;   A(T &in) : a(in) {}
+;   T a;
+; };
+;
+; __attribute__((always_inline))
+; void foo() {
+;   struct B { int i; };
+;   B objB;
+;   A<B> objA(objB);
+; }
+;
+; int main() {
+;   foo();
+; }
+
+; Concrete out-of-line tree of foo().
+; CHECK: DW_TAG_subprogram
+; CHECK:   DW_AT_abstract_origin {{.*}} "_Z3foov"
+
+; FIXME: 'struct B' should be in the abstract tree below, not here.
+; CHECK:   DW_TAG_structure_type
+; CHECK:     DW_AT_name	("B")
+; CHECK:     DW_TAG_member
+; CHECK:     NULL
+;
+; CHECK:   DW_TAG_variable
+; CHECK:     DW_AT_abstract_origin {{.*}} "objB"
+; CHECK:   DW_TAG_variable
+; CHECK:     DW_AT_abstract_origin {{.*}} "objA"
+
+; CHECK:   NULL
+
+; Abstract tree of foo().
+; CHECK: DW_TAG_subprogram
+; CHECK:   DW_AT_name	("foo")
+; CHECK:   DW_AT_inline	(DW_INL_inlined)
+
+; CHECK:   DW_TAG_variable
+; CHECK:     DW_AT_name	("objB")
+; CHECK:   DW_TAG_variable
+; CHECK:     DW_AT_name	("objA")
+
+; CHECK:   NULL
+
+; CHECK: DW_TAG_inlined_subroutine
+; CHECK:   DW_AT_abstract_origin {{.*}} "_Z3foov"
+; CHECK:   DW_TAG_variable
+; CHECK:     DW_AT_abstract_origin {{.*}} "objB"
+; CHECK:   DW_TAG_variable
+; CHECK:     DW_AT_abstract_origin {{.*}} "objA"
+; CHECK:   NULL
+
+%struct.B = type { i32 }
+%struct.A = type { %struct.B }
+
+define dso_local void @_Z3foov() !dbg !7 {
+entry:
+  %objB = alloca %struct.B, align 4
+  %objA = alloca %struct.A, align 4
+  call void @llvm.dbg.declare(metadata ptr %objB, metadata !30, metadata !DIExpression()), !dbg !31
+  call void @llvm.dbg.declare(metadata ptr %objA, metadata !32, metadata !DIExpression()), !dbg !33
+  call void @_ZN1AIZ3foovE1BEC2ERS0_(ptr noundef nonnull align 4 dereferenceable(4) %objA, ptr noundef nonnull align 4 dereferenceable(4) %objB), !dbg !33
+  ret void, !dbg !34
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+define internal void @_ZN1AIZ3foovE1BEC2ERS0_(ptr noundef nonnull align 4 dereferenceable(4) %this, ptr noundef nonnull align 4 dereferenceable(4) %in) unnamed_addr align 2 !dbg !35 {
+entry:
+  %this.addr = alloca ptr, align 8
+  %in.addr = alloca ptr, align 8
+  store ptr %this, ptr %this.addr, align 8
+  call void @llvm.dbg.declare(metadata ptr %this.addr, metadata !36, metadata !DIExpression()), !dbg !38
+  store ptr %in, ptr %in.addr, align 8
+  call void @llvm.dbg.declare(metadata ptr %in.addr, metadata !39, metadata !DIExpression()), !dbg !40
+  %this1 = load ptr, ptr %this.addr, align 8
+  %a = getelementptr inbounds %struct.A, ptr %this1, i32 0, i32 0, !dbg !41
+  %0 = load ptr, ptr %in.addr, align 8, !dbg !42
+  call void @llvm.memcpy.p0.p0.i64(ptr align 4 %a, ptr align 4 %0, i64 4, i1 false), !dbg !41
+  ret void, !dbg !43
+}
+
+define dso_local noundef i32 @main() !dbg !44 {
+entry:
+  %objB.i = alloca %struct.B, align 4
+  %objA.i = alloca %struct.A, align 4
+  call void @llvm.dbg.declare(metadata ptr %objB.i, metadata !30, metadata !DIExpression()), !dbg !47
+  call void @llvm.dbg.declare(metadata ptr %objA.i, metadata !32, metadata !DIExpression()), !dbg !49
+  call void @_ZN1AIZ3foovE1BEC2ERS0_(ptr noundef nonnull align 4 dereferenceable(4) %objA.i, ptr noundef nonnull align 4 dereferenceable(4) %objB.i), !dbg !49
+  ret i32 0, !dbg !50
+}
+
+declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!22, !23, !24, !25, !26, !27, !28}
+!llvm.ident = !{!29}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 15.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, retainedTypes: !2, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "test.cpp", directory: "/", checksumkind: CSK_MD5, checksum: "aec7fd397e86f8655ef7f4bb4233b849")
+!2 = !{!3}
+!3 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A<B>", file: !1, line: 2, size: 32, flags: DIFlagTypePassByValue | DIFlagNonTrivial, elements: !4, templateParams: !20)
+!4 = !{!5, !15}
+!5 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !3, file: !1, line: 4, baseType: !6, size: 32)
+!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "B", scope: !7, file: !1, line: 9, size: 32, flags: DIFlagTypePassByValue, elements: !12)
+!7 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !1, file: !1, line: 8, type: !8, scopeLine: 8, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{null}
+!10 = !{}
+!11 = !{!6}
+!12 = !{!13}
+!13 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !6, file: !1, line: 9, baseType: !14, size: 32)
+!14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!15 = !DISubprogram(name: "A", scope: !3, file: !1, line: 3, type: !16, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagLocalToUnit)
+!16 = !DISubroutineType(types: !17)
+!17 = !{null, !18, !19}
+!18 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !3, size: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
+!19 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !6, size: 64)
+!20 = !{!21}
+!21 = !DITemplateTypeParameter(name: "T", type: !6)
+!22 = !{i32 7, !"Dwarf Version", i32 5}
+!23 = !{i32 2, !"Debug Info Version", i32 3}
+!24 = !{i32 1, !"wchar_size", i32 4}
+!25 = !{i32 7, !"PIC Level", i32 2}
+!26 = !{i32 7, !"PIE Level", i32 2}
+!27 = !{i32 7, !"uwtable", i32 2}
+!28 = !{i32 7, !"frame-pointer", i32 2}
+!29 = !{!"clang version 15.0.0"}
+!30 = !DILocalVariable(name: "objB", scope: !7, file: !1, line: 10, type: !6)
+!31 = !DILocation(line: 10, column: 5, scope: !7)
+!32 = !DILocalVariable(name: "objA", scope: !7, file: !1, line: 11, type: !3)
+!33 = !DILocation(line: 11, column: 8, scope: !7)
+!34 = !DILocation(line: 12, column: 1, scope: !7)
+!35 = distinct !DISubprogram(name: "A", linkageName: "_ZN1AIZ3foovE1BEC2ERS0_", scope: !3, file: !1, line: 3, type: !16, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !0, declaration: !15, retainedNodes: !10)
+!36 = !DILocalVariable(name: "this", arg: 1, scope: !35, type: !37, flags: DIFlagArtificial | DIFlagObjectPointer)
+!37 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !3, size: 64)
+!38 = !DILocation(line: 0, scope: !35)
+!39 = !DILocalVariable(name: "in", arg: 2, scope: !35, file: !1, line: 3, type: !19)
+!40 = !DILocation(line: 3, column: 8, scope: !35)
+!41 = !DILocation(line: 3, column: 14, scope: !35)
+!42 = !DILocation(line: 3, column: 16, scope: !35)
+!43 = !DILocation(line: 3, column: 21, scope: !35)
+!44 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 14, type: !45, scopeLine: 14, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !10)
+!45 = !DISubroutineType(types: !46)
+!46 = !{!14}
+!47 = !DILocation(line: 10, column: 5, scope: !7, inlinedAt: !48)
+!48 = distinct !DILocation(line: 15, column: 3, scope: !44)
+!49 = !DILocation(line: 11, column: 8, scope: !7, inlinedAt: !48)
+!50 = !DILocation(line: 16, column: 1, scope: !44)
diff --git a/llvm/test/DebugInfo/Generic/verifier-invalid-disubprogram.ll b/llvm/test/DebugInfo/Generic/verifier-invalid-disubprogram.ll
index 6d4d0e93d38f9..54ce1c56c6b30 100644
--- a/llvm/test/DebugInfo/Generic/verifier-invalid-disubprogram.ll
+++ b/llvm/test/DebugInfo/Generic/verifier-invalid-disubprogram.ll
@@ -38,7 +38,7 @@ define void @invalid_subprogram_declaration() !dbg !9 { ret void }
 define void @invalid_retained_nodes_list() !dbg !10 { ret void }
 !10 = distinct !DISubprogram(retainedNodes: !0)
 
-; CHECK: invalid retained nodes, expected DILocalVariable, DILabel or DIImportedEntity
+; CHECK: invalid retained nodes, expected DILocalVariable, DILabel, DIImportedEntity or DIType
 define void @invalid_retained_nodes_expected() !dbg !11 { ret void }
 !11 = distinct !DISubprogram(retainedNodes: !{!0})
 
diff --git a/llvm/test/DebugInfo/X86/set.ll b/llvm/test/DebugInfo/X86/set.ll
index 292c7c6e4a577..7dbec4520ed6c 100644
--- a/llvm/test/DebugInfo/X86/set.ll
+++ b/llvm/test/DebugInfo/X86/set.ll
@@ -68,11 +68,11 @@ attributes #1 = { nofree nosync nounwind readnone speculatable willreturn }
 !llvm.module.flags = !{!18, !19, !20}
 
 !0 = !{!"versions- cm3: d5.10.0 llvm: 9.0"}
-!1 = distinct !DICompileUnit(language: DW_LANG_Modula3, file: !2, producer: "cm3", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !3)
+!1 = distinct !DICompileUnit(language: DW_LANG_Modula3, file: !2, producer: "cm3", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug)
 !2 = !DIFile(filename: "Main.m3", directory: "/home/cm3/settest/src")
 !3 = !{!4}
 !4 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum", scope: !5, file: !2, line: 11, size: 8, align: 8, elements: !9)
-!5 = distinct !DISubprogram(name: "Test", linkageName: "Main__Test", scope: !2, file: !2, line: 11, type: !6, scopeLine: 11, spFlags: DISPFlagDefinition, unit: !1, retainedNodes: !8)
+!5 = distinct !DISubprogram(name: "Test", linkageName: "Main__Test", scope: !2, file: !2, line: 11, type: !6, scopeLine: 11, spFlags: DISPFlagDefinition, unit: !1, retainedNodes: !3)
 !6 = !DISubroutineType(types: !7)
 !7 = !{null}
 !8 = !{}

From ab927a998397c30fd3426d9aa1e071206fa554b2 Mon Sep 17 00:00:00 2001
From: Joe Nash <Joseph.Nash@amd.com>
Date: Mon, 12 Jun 2023 17:21:29 -0400
Subject: [PATCH 033/130] [AMDGPU] Fix operand class of v_ldexp_f16 src1

Patch eece6ba283bd changed the src1 type of v_ldexp_f16 from i32 to
i16. Though semantically src1 is an i16, the hardware reads this operand as an
f16 type, which primarily enables floating point inline constants.
Therefore this patch changes the operand type to f16. It maintains the
current behavior where floating point source modifiers are not allowed
on src1. SDWA sext modifier continues to be allowed.
The test asm and disasm test changes in eece6ba283bd are reverted,
because the floating point inline constants are allowed.

Reviewed By: arsenm

Differential Revision: https://reviews.llvm.org/D153169
---
 llvm/lib/Target/AMDGPU/VOP2Instructions.td    | 34 ++++++++++++++++---
 llvm/test/MC/AMDGPU/gfx10_asm_vop2.s          |  6 ++--
 .../test/MC/AMDGPU/gfx11_asm_vop3_from_vop2.s |  2 +-
 llvm/test/MC/AMDGPU/gfx8_asm_vop3.s           |  7 ++--
 llvm/test/MC/AMDGPU/gfx9_asm_vop3.s           |  7 ++--
 .../MC/Disassembler/AMDGPU/gfx10_vop3.txt     |  4 +--
 .../AMDGPU/gfx11_dasm_vop3_from_vop2.txt      |  2 +-
 .../test/MC/Disassembler/AMDGPU/gfx8_vop3.txt |  4 +--
 .../test/MC/Disassembler/AMDGPU/gfx9_vop3.txt |  4 +--
 9 files changed, 46 insertions(+), 24 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/VOP2Instructions.td b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
index 35429a44c93e3..ea7ab163992ea 100644
--- a/llvm/lib/Target/AMDGPU/VOP2Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP2Instructions.td
@@ -863,9 +863,18 @@ def :  divergent_i64_BinOp <xor, V_XOR_B32_e64>;
 // 16-Bit Operand Instructions
 //===----------------------------------------------------------------------===//
 
-def LDEXP_F16_VOPProfile_True16 : VOPProfile_True16<VOP_F16_F16_I16> {
-  // The ldexp.f16 intrinsic expects a i32 src1 operand, though the hardware
-  // encoding treats src1 as an f16
+// The ldexp.f16 intrinsic expects a integer src1 operand, though the hardware
+// encoding treats src1 as an f16
+def LDEXP_F16_VOPProfile : VOPProfile <[f16, f16, f16, untyped]> {
+  let Src1Mod = Int32InputMods;
+  let Src1ModDPP = IntVRegInputMods;
+  let Src1ModVOP3DPP = IntVRegInputMods;
+  // SDWA sext is the only modifier allowed.
+  let HasSrc1IntMods = 1;
+  let HasSrc1FloatMods = 0;
+  let Src1ModSDWA = Int16SDWAInputMods;
+}
+def LDEXP_F16_VOPProfile_True16 : VOPProfile_True16<VOP_F16_F16_F16> {
   let Src1RC32 = RegisterOperand<VGPR_32_Lo128>;
   let Src1DPP = VGPR_32_Lo128;
   let Src1ModDPP = IntT16VRegInputMods;
@@ -874,9 +883,9 @@ def LDEXP_F16_VOPProfile_True16 : VOPProfile_True16<VOP_F16_F16_I16> {
 let isReMaterializable = 1 in {
 let FPDPRounding = 1 in {
   let SubtargetPredicate = NotHasTrue16BitInsts, OtherPredicates = [Has16BitInsts]  in
-    defm V_LDEXP_F16 : VOP2Inst <"v_ldexp_f16", VOP_F16_F16_I16, any_fldexp>;
+    defm V_LDEXP_F16 : VOP2Inst <"v_ldexp_f16", LDEXP_F16_VOPProfile>;
   let SubtargetPredicate = HasTrue16BitInsts in
-    defm V_LDEXP_F16_t16 : VOP2Inst <"v_ldexp_f16_t16", LDEXP_F16_VOPProfile_True16, any_fldexp>;
+    defm V_LDEXP_F16_t16 : VOP2Inst <"v_ldexp_f16_t16", LDEXP_F16_VOPProfile_True16>;
 } // End FPDPRounding = 1
 // FIXME VOP3 Only instructions. NFC using VOPProfile_True16 for these until a planned change to use a new register class for VOP3 encoded True16 instuctions
 defm V_LSHLREV_B16 : VOP2Inst_e64_t16 <"v_lshlrev_b16", VOP_I16_I16_I16, clshl_rev_16>;
@@ -899,6 +908,21 @@ defm V_MIN_I16 : VOP2Inst_e64_t16 <"v_min_i16", VOP_I16_I16_I16, smin>;
 } // End isCommutable = 1
 } // End isReMaterializable = 1
 
+class LDEXP_F16_Pat <SDPatternOperator op, VOP_Pseudo inst, VOPProfile P = inst.Pfl> : GCNPat <
+  (P.DstVT (op (P.Src0VT (VOP3Mods0 P.Src0VT:$src0, i32:$src0_modifiers, i1:$clamp, i32:$omod)),
+               (i16 (VOP3Mods0 P.Src1VT:$src1, i32:$src1_modifiers)))),
+  (inst $src0_modifiers, $src0,
+        $src1_modifiers, $src1,
+        $clamp, /* clamp */
+        $omod /* omod */)
+>;
+
+let OtherPredicates = [NotHasTrue16BitInsts] in
+def : LDEXP_F16_Pat<any_fldexp, V_LDEXP_F16_e64>;
+
+let OtherPredicates = [HasTrue16BitInsts] in
+def : LDEXP_F16_Pat<any_fldexp, V_LDEXP_F16_t16_e64>;
+
 let SubtargetPredicate = isGFX11Plus in {
   let isCommutable = 1 in {
     defm V_AND_B16_t16 : VOP2Inst_e64 <"v_and_b16_t16", VOPProfile_True16<VOP_I16_I16_I16>, and>;
diff --git a/llvm/test/MC/AMDGPU/gfx10_asm_vop2.s b/llvm/test/MC/AMDGPU/gfx10_asm_vop2.s
index 6ea135c95d1f0..b1b54005bee28 100644
--- a/llvm/test/MC/AMDGPU/gfx10_asm_vop2.s
+++ b/llvm/test/MC/AMDGPU/gfx10_asm_vop2.s
@@ -12931,11 +12931,11 @@ v_ldexp_f16_e64 v5, v1, 0
 v_ldexp_f16_e64 v5, v1, -1
 // GFX10: encoding: [0x05,0x00,0x3b,0xd5,0x01,0x83,0x01,0x00]
 
-v_ldexp_f16_e64 v5, v1, 0x3800
-// GFX10: encoding: [0x05,0x00,0x3b,0xd5,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+v_ldexp_f16_e64 v5, v1, 0.5
+// GFX10: encoding: [0x05,0x00,0x3b,0xd5,0x01,0xe1,0x01,0x00]
 
 v_ldexp_f16_e64 v5, v1, -4.0
-// GFX10: encoding: [0x05,0x00,0x3b,0xd5,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+// GFX10: encoding: [0x05,0x00,0x3b,0xd5,0x01,0xef,0x01,0x00]
 
 v_ldexp_f16_e64 v5, -v1, v2
 // GFX10: encoding: [0x05,0x00,0x3b,0xd5,0x01,0x05,0x02,0x20]
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vop2.s b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vop2.s
index ffb83100c08ce..43c71617bb385 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vop2.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_vop3_from_vop2.s
@@ -734,7 +734,7 @@ v_ldexp_f16_e64 v5, ttmp15, src_scc
 // GFX11: encoding: [0x05,0x00,0x3b,0xd5,0x7b,0xfa,0x01,0x00]
 
 v_ldexp_f16_e64 v5, m0, 0.5
-// GFX11: encoding: [0x05,0x00,0x3b,0xd5,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+// GFX11: encoding: [0x05,0x00,0x3b,0xd5,0x7d,0xe0,0x01,0x00]
 
 v_ldexp_f16_e64 v5, exec_lo, -1
 // GFX11: encoding: [0x05,0x00,0x3b,0xd5,0x7e,0x82,0x01,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx8_asm_vop3.s b/llvm/test/MC/AMDGPU/gfx8_asm_vop3.s
index d46a97f1cbd58..d4c31f14d3bfc 100644
--- a/llvm/test/MC/AMDGPU/gfx8_asm_vop3.s
+++ b/llvm/test/MC/AMDGPU/gfx8_asm_vop3.s
@@ -1,5 +1,4 @@
-// RUN: not llvm-mc -arch=amdgcn -mcpu=tonga -show-encoding %s | FileCheck %s
-// RUN: not llvm-mc -arch=amdgcn -mcpu=tonga -show-encoding -filetype=null 2>&1 %s | FileCheck -check-prefix=ERR --implicit-check-not=error %s
+// RUN: llvm-mc -arch=amdgcn -mcpu=tonga -show-encoding %s | FileCheck %s
 
 v_interp_p1_f32_e64 v5, v2, attr0.x
 // CHECK: [0x05,0x00,0x70,0xd2,0x00,0x04,0x02,0x00]
@@ -12827,10 +12826,10 @@ v_ldexp_f16_e64 v5, v1, -1
 // CHECK: [0x05,0x00,0x33,0xd1,0x01,0x83,0x01,0x00]
 
 v_ldexp_f16_e64 v5, v1, 0.5
-// ERR: [[@LINE-1]]:25: error: literal operands are not supported
+// CHECK: [0x05,0x00,0x33,0xd1,0x01,0xe1,0x01,0x00]
 
 v_ldexp_f16_e64 v5, v1, -4.0
-// ERR: [[@LINE-1]]:25: error: literal operands are not supported
+// CHECK: [0x05,0x00,0x33,0xd1,0x01,0xef,0x01,0x00]
 
 v_ldexp_f16_e64 v5, v1, src_vccz
 // CHECK: [0x05,0x00,0x33,0xd1,0x01,0xf7,0x01,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx9_asm_vop3.s b/llvm/test/MC/AMDGPU/gfx9_asm_vop3.s
index 34d0d77c07f36..8781a01f1eb8e 100644
--- a/llvm/test/MC/AMDGPU/gfx9_asm_vop3.s
+++ b/llvm/test/MC/AMDGPU/gfx9_asm_vop3.s
@@ -1,5 +1,4 @@
-// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx900 -show-encoding %s | FileCheck %s
-// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx900 -show-encoding 2>&1 %s | FileCheck -check-prefix=ERR --implicit-check-not=error %s
+// RUN: llvm-mc -arch=amdgcn -mcpu=gfx900 -show-encoding %s | FileCheck %s
 
 v_interp_p1_f32_e64 v5, v2, attr0.x
 // CHECK: [0x05,0x00,0x70,0xd2,0x00,0x04,0x02,0x00]
@@ -11240,10 +11239,10 @@ v_ldexp_f16_e64 v5, v1, -1
 // CHECK: [0x05,0x00,0x33,0xd1,0x01,0x83,0x01,0x00]
 
 v_ldexp_f16_e64 v5, v1, 0.5
-// ERR: [[@LINE-1]]:25: error: literal operands are not supported
+// CHECK: [0x05,0x00,0x33,0xd1,0x01,0xe1,0x01,0x00]
 
 v_ldexp_f16_e64 v5, v1, -4.0
-// ERR: [[@LINE-1]]:25: error: literal operands are not supported
+// CHECK: [0x05,0x00,0x33,0xd1,0x01,0xef,0x01,0x00]
 
 v_ldexp_f16_e64 v5, v1, src_vccz
 // CHECK: [0x05,0x00,0x33,0xd1,0x01,0xf7,0x01,0x00]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx10_vop3.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx10_vop3.txt
index c5cd6f6c4dcea..0785ba2ea2eb6 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx10_vop3.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx10_vop3.txt
@@ -7520,13 +7520,13 @@
 # GFX10: v_ldexp_f16_e64 v5, v1, -1              ; encoding: [0x05,0x00,0x3b,0xd5,0x01,0x83,0x01,0x00]
 0x05,0x00,0x3b,0xd5,0x01,0x83,0x01,0x00
 
-# GFX10: v_ldexp_f16_e64 v5, v1, 0xc400          ; encoding: [0x05,0x00,0x3b,0xd5,0x01,0xff,0x01,0x00,0x00,0xc4,0x00,0x00]
+# GFX10: v_ldexp_f16_e64 v5, v1, -4.0            ; encoding: [0x05,0x00,0x3b,0xd5,0x01,0xef,0x01,0x00]
 0x05,0x00,0x3b,0xd5,0x01,0xef,0x01,0x00
 
 # GFX10: v_ldexp_f16_e64 v5, v1, 0               ; encoding: [0x05,0x00,0x3b,0xd5,0x01,0x01,0x01,0x00]
 0x05,0x00,0x3b,0xd5,0x01,0x01,0x01,0x00
 
-# GFX10: v_ldexp_f16_e64 v5, v1, 0x3800          ; encoding: [0x05,0x00,0x3b,0xd5,0x01,0xff,0x01,0x00,0x00,0x38,0x00,0x00]
+# GFX10: v_ldexp_f16_e64 v5, v1, 0.5             ; encoding: [0x05,0x00,0x3b,0xd5,0x01,0xe1,0x01,0x00]
 0x05,0x00,0x3b,0xd5,0x01,0xe1,0x01,0x00
 
 # GFX10: v_ldexp_f16_e64 v5, v1, exec_hi         ; encoding: [0x05,0x00,0x3b,0xd5,0x01,0xff,0x00,0x00]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_from_vop2.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_from_vop2.txt
index 4e430a88a1cb4..3141e8f4b2cbb 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_from_vop2.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx11_dasm_vop3_from_vop2.txt
@@ -543,7 +543,7 @@
 # GFX11: v_ldexp_f16_e64 v5, ttmp15, src_scc     ; encoding: [0x05,0x00,0x3b,0xd5,0x7b,0xfa,0x01,0x00]
 0x05,0x00,0x3b,0xd5,0x7b,0xfa,0x01,0x00
 
-# GFX11: v_ldexp_f16_e64 v5, m0, 0x3800          ; encoding: [0x05,0x00,0x3b,0xd5,0x7d,0xfe,0x01,0x00,0x00,0x38,0x00,0x00]
+# GFX11: v_ldexp_f16_e64 v5, m0, 0.5             ; encoding: [0x05,0x00,0x3b,0xd5,0x7d,0xe0,0x01,0x00]
 0x05,0x00,0x3b,0xd5,0x7d,0xe0,0x01,0x00
 
 # GFX11: v_ldexp_f16_e64 v5, exec_lo, -1         ; encoding: [0x05,0x00,0x3b,0xd5,0x7e,0x82,0x01,0x00]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx8_vop3.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx8_vop3.txt
index a0277c706ee23..2b07d620fdad2 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx8_vop3.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx8_vop3.txt
@@ -11178,10 +11178,10 @@
 # CHECK: v_ldexp_f16_e64 v5, v1, -1              ; encoding: [0x05,0x00,0x33,0xd1,0x01,0x83,0x01,0x00]
 0x05,0x00,0x33,0xd1,0x01,0x83,0x01,0x00
 
-# CHECK: v_ldexp_f16_e64 v5, v1, 0x3800          ; encoding: [0x05,0x00,0x33,0xd1,0x01,0xff,0x01,0x00]
+# CHECK: v_ldexp_f16_e64 v5, v1, 0.5             ; encoding: [0x05,0x00,0x33,0xd1,0x01,0xe1,0x01,0x00]
 0x05,0x00,0x33,0xd1,0x01,0xe1,0x01,0x00
 
-# CHECK: v_ldexp_f16_e64 v5, v1, 0xc400          ; encoding: [0x05,0x00,0x33,0xd1,0x01,0xff,0x01,0x00]
+# CHECK: v_ldexp_f16_e64 v5, v1, -4.0            ; encoding: [0x05,0x00,0x33,0xd1,0x01,0xef,0x01,0x00]
 0x05,0x00,0x33,0xd1,0x01,0xef,0x01,0x00
 
 # CHECK: v_ldexp_f16_e64 v5, -v1, v2             ; encoding: [0x05,0x00,0x33,0xd1,0x01,0x05,0x02,0x20]
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx9_vop3.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx9_vop3.txt
index c2ac84b44e0ca..e3ed9778f6fb4 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx9_vop3.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx9_vop3.txt
@@ -8814,10 +8814,10 @@
 # CHECK: v_ldexp_f16_e64 v5, v1, -1              ; encoding: [0x05,0x00,0x33,0xd1,0x01,0x83,0x01,0x00]
 0x05,0x00,0x33,0xd1,0x01,0x83,0x01,0x00
 
-# CHECK: v_ldexp_f16_e64 v5, v1, 0x3800          ; encoding: [0x05,0x00,0x33,0xd1,0x01,0xff,0x01,0x00]
+# CHECK: v_ldexp_f16_e64 v5, v1, 0.5             ; encoding: [0x05,0x00,0x33,0xd1,0x01,0xe1,0x01,0x00]
 0x05,0x00,0x33,0xd1,0x01,0xe1,0x01,0x00
 
-# CHECK: v_ldexp_f16_e64 v5, v1, 0xc400          ; encoding: [0x05,0x00,0x33,0xd1,0x01,0xff,0x01,0x00]
+# CHECK: v_ldexp_f16_e64 v5, v1, -4.0            ; encoding: [0x05,0x00,0x33,0xd1,0x01,0xef,0x01,0x00]
 0x05,0x00,0x33,0xd1,0x01,0xef,0x01,0x00
 
 # CHECK: v_ldexp_f16_e64 v5, -v1, v2             ; encoding: [0x05,0x00,0x33,0xd1,0x01,0x05,0x02,0x20]

From d0f56c3e5cc0ead069280d59c41fe12567ed0619 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Mon, 19 Jun 2023 15:47:21 +0100
Subject: [PATCH 034/130] [AArch64] Add and expand the testing of fmin/fmax
 reduction. NFC

For both CodeGen and CostModelling, this adds extran testing for the new
lvm.vector.reduce.fmaximum and lvm.vector.reduce.fminimum intrinsics, as well
as making sure there is test coverage for all the various cases.
---
 .../CostModel/AArch64/reduce-minmax.ll        |  64 +++++
 llvm/test/CodeGen/AArch64/vecreduce-fadd.ll   |   2 +-
 .../AArch64/vecreduce-fmax-legalization.ll    |  18 +-
 .../CodeGen/AArch64/vecreduce-fmaximum.ll     |  11 +
 .../AArch64/vecreduce-fmin-legalization.ll    |  18 +-
 .../CodeGen/AArch64/vecreduce-fminimum.ll     | 235 ++++++++++++++++++
 6 files changed, 339 insertions(+), 9 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/vecreduce-fminimum.ll

diff --git a/llvm/test/Analysis/CostModel/AArch64/reduce-minmax.ll b/llvm/test/Analysis/CostModel/AArch64/reduce-minmax.ll
index cf813bae839c6..c10f87fcef658 100644
--- a/llvm/test/Analysis/CostModel/AArch64/reduce-minmax.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/reduce-minmax.ll
@@ -166,6 +166,10 @@ define void @reduce_fmin16() {
 ; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V4f16 = call half @llvm.vector.reduce.fmin.v4f16(<4 x half> undef)
 ; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V8f16 = call half @llvm.vector.reduce.fmin.v8f16(<8 x half> undef)
 ; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 298 for instruction: %V16f16 = call half @llvm.vector.reduce.fmin.v16f16(<16 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2f16m = call half @llvm.vector.reduce.fminimum.v2f16(<2 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4f16m = call half @llvm.vector.reduce.fminimum.v4f16(<4 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V8f16m = call half @llvm.vector.reduce.fminimum.v8f16(<8 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V16f16m = call half @llvm.vector.reduce.fminimum.v16f16(<16 x half> undef)
 ; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-F16-LABEL: 'reduce_fmin16'
@@ -173,12 +177,20 @@ define void @reduce_fmin16() {
 ; CHECK-F16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4f16 = call half @llvm.vector.reduce.fmin.v4f16(<4 x half> undef)
 ; CHECK-F16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8f16 = call half @llvm.vector.reduce.fmin.v8f16(<8 x half> undef)
 ; CHECK-F16-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16f16 = call half @llvm.vector.reduce.fmin.v16f16(<16 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2f16m = call half @llvm.vector.reduce.fminimum.v2f16(<2 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4f16m = call half @llvm.vector.reduce.fminimum.v4f16(<4 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V8f16m = call half @llvm.vector.reduce.fminimum.v8f16(<8 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V16f16m = call half @llvm.vector.reduce.fminimum.v16f16(<16 x half> undef)
 ; CHECK-F16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %V2f16 = call half @llvm.vector.reduce.fmin.v2f16(<2 x half> undef)
   %V4f16 = call half @llvm.vector.reduce.fmin.v4f16(<4 x half> undef)
   %V8f16 = call half @llvm.vector.reduce.fmin.v8f16(<8 x half> undef)
   %V16f16 = call half @llvm.vector.reduce.fmin.v16f16(<16 x half> undef)
+  %V2f16m = call half @llvm.vector.reduce.fminimum.v2f16(<2 x half> undef)
+  %V4f16m = call half @llvm.vector.reduce.fminimum.v4f16(<4 x half> undef)
+  %V8f16m = call half @llvm.vector.reduce.fminimum.v8f16(<8 x half> undef)
+  %V16f16m = call half @llvm.vector.reduce.fminimum.v16f16(<16 x half> undef)
   ret void
 }
 
@@ -188,6 +200,10 @@ define void @reduce_fmax16() {
 ; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V4f16 = call half @llvm.vector.reduce.fmax.v4f16(<4 x half> undef)
 ; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V8f16 = call half @llvm.vector.reduce.fmax.v8f16(<8 x half> undef)
 ; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 298 for instruction: %V16f16 = call half @llvm.vector.reduce.fmax.v16f16(<16 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2f16m = call half @llvm.vector.reduce.fmaximum.v2f16(<2 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4f16m = call half @llvm.vector.reduce.fmaximum.v4f16(<4 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V8f16m = call half @llvm.vector.reduce.fmaximum.v8f16(<8 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V16f16m = call half @llvm.vector.reduce.fmaximum.v16f16(<16 x half> undef)
 ; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-F16-LABEL: 'reduce_fmax16'
@@ -195,12 +211,20 @@ define void @reduce_fmax16() {
 ; CHECK-F16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V4f16 = call half @llvm.vector.reduce.fmax.v4f16(<4 x half> undef)
 ; CHECK-F16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V8f16 = call half @llvm.vector.reduce.fmax.v8f16(<8 x half> undef)
 ; CHECK-F16-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V16f16 = call half @llvm.vector.reduce.fmax.v16f16(<16 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2f16m = call half @llvm.vector.reduce.fmaximum.v2f16(<2 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4f16m = call half @llvm.vector.reduce.fmaximum.v4f16(<4 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V8f16m = call half @llvm.vector.reduce.fmaximum.v8f16(<8 x half> undef)
+; CHECK-F16-NEXT:  Cost Model: Found an estimated cost of 58 for instruction: %V16f16m = call half @llvm.vector.reduce.fmaximum.v16f16(<16 x half> undef)
 ; CHECK-F16-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %V2f16 = call half @llvm.vector.reduce.fmax.v2f16(<2 x half> undef)
   %V4f16 = call half @llvm.vector.reduce.fmax.v4f16(<4 x half> undef)
   %V8f16 = call half @llvm.vector.reduce.fmax.v8f16(<8 x half> undef)
   %V16f16 = call half @llvm.vector.reduce.fmax.v16f16(<16 x half> undef)
+  %V2f16m = call half @llvm.vector.reduce.fmaximum.v2f16(<2 x half> undef)
+  %V4f16m = call half @llvm.vector.reduce.fmaximum.v4f16(<4 x half> undef)
+  %V8f16m = call half @llvm.vector.reduce.fmaximum.v8f16(<8 x half> undef)
+  %V16f16m = call half @llvm.vector.reduce.fmaximum.v16f16(<16 x half> undef)
   ret void
 }
 
@@ -211,6 +235,11 @@ define void @reduce_fmin() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8f32 = call float @llvm.vector.reduce.fmin.v8f32(<8 x float> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2f64 = call double @llvm.vector.reduce.fmin.v2f64(<2 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4f64 = call double @llvm.vector.reduce.fmin.v4f64(<4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2f32m = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4f32m = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8f32m = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2f64m = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4f64m = call double @llvm.vector.reduce.fminimum.v4f64(<4 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %V2f32 = call float @llvm.vector.reduce.fmin.v2f32(<2 x float> undef)
@@ -218,6 +247,11 @@ define void @reduce_fmin() {
   %V8f32 = call float @llvm.vector.reduce.fmin.v8f32(<8 x float> undef)
   %V2f64 = call double @llvm.vector.reduce.fmin.v2f64(<2 x double> undef)
   %V4f64 = call double @llvm.vector.reduce.fmin.v4f64(<4 x double> undef)
+  %V2f32m = call float @llvm.vector.reduce.fminimum.v2f32(<2 x float> undef)
+  %V4f32m = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> undef)
+  %V8f32m = call float @llvm.vector.reduce.fminimum.v8f32(<8 x float> undef)
+  %V2f64m = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> undef)
+  %V4f64m = call double @llvm.vector.reduce.fminimum.v4f64(<4 x double> undef)
   ret void
 }
 
@@ -228,6 +262,11 @@ define void @reduce_fmax() {
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V8f32 = call float @llvm.vector.reduce.fmax.v8f32(<8 x float> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %V2f64 = call double @llvm.vector.reduce.fmax.v2f64(<2 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 3 for instruction: %V4f64 = call double @llvm.vector.reduce.fmax.v4f64(<4 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2f32m = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4f32m = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 26 for instruction: %V8f32m = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2f64m = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef)
+; CHECK-NEXT:  Cost Model: Found an estimated cost of 10 for instruction: %V4f64m = call double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef)
 ; CHECK-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
   %V2f32 = call float @llvm.vector.reduce.fmax.v2f32(<2 x float> undef)
@@ -235,6 +274,11 @@ define void @reduce_fmax() {
   %V8f32 = call float @llvm.vector.reduce.fmax.v8f32(<8 x float> undef)
   %V2f64 = call double @llvm.vector.reduce.fmax.v2f64(<2 x double> undef)
   %V4f64 = call double @llvm.vector.reduce.fmax.v4f64(<4 x double> undef)
+  %V2f32m = call float @llvm.vector.reduce.fmaximum.v2f32(<2 x float> undef)
+  %V4f32m = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> undef)
+  %V8f32m = call float @llvm.vector.reduce.fmaximum.v8f32(<8 x float> undef)
+  %V2f64m = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> undef)
+  %V4f64m = call double @llvm.vector.reduce.fmaximum.v4f64(<4 x double> undef)
   ret void
 }
 
@@ -325,3 +369,23 @@ declare float @llvm.vector.reduce.fmax.v4f32(<4 x float>)
 declare float @llvm.vector.reduce.fmax.v8f32(<8 x float>)
 declare double @llvm.vector.reduce.fmax.v2f64(<2 x double>)
 declare double @llvm.vector.reduce.fmax.v4f64(<4 x double>)
+
+declare half @llvm.vector.reduce.fminimum.v2f16(<2 x half>)
+declare half @llvm.vector.reduce.fminimum.v4f16(<4 x half>)
+declare half @llvm.vector.reduce.fminimum.v8f16(<8 x half>)
+declare half @llvm.vector.reduce.fminimum.v16f16(<16 x half>)
+declare float @llvm.vector.reduce.fminimum.v2f32(<2 x float>)
+declare float @llvm.vector.reduce.fminimum.v4f32(<4 x float>)
+declare float @llvm.vector.reduce.fminimum.v8f32(<8 x float>)
+declare double @llvm.vector.reduce.fminimum.v2f64(<2 x double>)
+declare double @llvm.vector.reduce.fminimum.v4f64(<4 x double>)
+
+declare half @llvm.vector.reduce.fmaximum.v2f16(<2 x half>)
+declare half @llvm.vector.reduce.fmaximum.v4f16(<4 x half>)
+declare half @llvm.vector.reduce.fmaximum.v8f16(<8 x half>)
+declare half @llvm.vector.reduce.fmaximum.v16f16(<16 x half>)
+declare float @llvm.vector.reduce.fmaximum.v2f32(<2 x float>)
+declare float @llvm.vector.reduce.fmaximum.v4f32(<4 x float>)
+declare float @llvm.vector.reduce.fmaximum.v8f32(<8 x float>)
+declare double @llvm.vector.reduce.fmaximum.v2f64(<2 x double>)
+declare double @llvm.vector.reduce.fmaximum.v4f64(<4 x double>)
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll b/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll
index 55b140b28bd49..b43b01027dda2 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fadd.ll
@@ -227,7 +227,7 @@ define float @add_S_init_42(<4 x float> %bin.rdx)  {
 ; CHECK-LABEL: add_S_init_42:
 ; CHECK:       // %bb.0:
 ; CHECK-NEXT:    faddp v0.4s, v0.4s, v0.4s
-; CHECK-NEXT:    mov w8, #1109917696
+; CHECK-NEXT:    mov w8, #1109917696 // =0x42280000
 ; CHECK-NEXT:    fmov s1, w8
 ; CHECK-NEXT:    faddp s0, v0.2s
 ; CHECK-NEXT:    fadd s0, s0, s1
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fmax-legalization.ll b/llvm/test/CodeGen/AArch64/vecreduce-fmax-legalization.ll
index 51b60332bf5ab..3df8749d8978c 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-fmax-legalization.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fmax-legalization.ll
@@ -12,6 +12,7 @@ declare half @llvm.vector.reduce.fmax.v11f16(<11 x half> %a)
 declare float @llvm.vector.reduce.fmax.v3f32(<3 x float> %a)
 declare fp128 @llvm.vector.reduce.fmax.v2f128(<2 x fp128> %a)
 declare float @llvm.vector.reduce.fmax.v16f32(<16 x float> %a)
+declare double @llvm.vector.reduce.fmax.v2f64(<2 x double> %a)
 
 define half @test_v1f16(<1 x half> %a) nounwind {
 ; CHECK-LABEL: test_v1f16:
@@ -131,7 +132,7 @@ define half @test_v11f16(<11 x half> %a) nounwind {
 ; CHECK-NOFP-NEXT:    fmaxnm s0, s0, s1
 ; CHECK-NOFP-NEXT:    fcsel s1, s2, s16, gt
 ; CHECK-NOFP-NEXT:    ldr h2, [x8, :lo12:.LCPI6_0]
-; CHECK-NOFP-NEXT:    mov w8, #-8388608
+; CHECK-NOFP-NEXT:    mov w8, #-8388608 // =0xff800000
 ; CHECK-NOFP-NEXT:    fcvt h0, s0
 ; CHECK-NOFP-NEXT:    fcvt h1, s1
 ; CHECK-NOFP-NEXT:    fcvt s2, h2
@@ -238,7 +239,7 @@ define half @test_v11f16_ninf(<11 x half> %a) nounwind {
 ; CHECK-NOFP-NEXT:    fmaxnm s0, s0, s1
 ; CHECK-NOFP-NEXT:    fcsel s1, s2, s16, gt
 ; CHECK-NOFP-NEXT:    ldr h2, [x8, :lo12:.LCPI7_0]
-; CHECK-NOFP-NEXT:    mov w8, #57344
+; CHECK-NOFP-NEXT:    mov w8, #57344 // =0xe000
 ; CHECK-NOFP-NEXT:    movk w8, #51071, lsl #16
 ; CHECK-NOFP-NEXT:    fcvt h0, s0
 ; CHECK-NOFP-NEXT:    fcvt h1, s1
@@ -323,7 +324,7 @@ define half @test_v11f16_ninf(<11 x half> %a) nounwind {
 define float @test_v3f32(<3 x float> %a) nounwind {
 ; CHECK-LABEL: test_v3f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #-8388608
+; CHECK-NEXT:    mov w8, #-8388608 // =0xff800000
 ; CHECK-NEXT:    fmov s1, w8
 ; CHECK-NEXT:    mov v0.s[3], v1.s[0]
 ; CHECK-NEXT:    fmaxnmv s0, v0.4s
@@ -335,7 +336,7 @@ define float @test_v3f32(<3 x float> %a) nounwind {
 define float @test_v3f32_ninf(<3 x float> %a) nounwind {
 ; CHECK-LABEL: test_v3f32_ninf:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #-8388609
+; CHECK-NEXT:    mov w8, #-8388609 // =0xff7fffff
 ; CHECK-NEXT:    fmov s1, w8
 ; CHECK-NEXT:    mov v0.s[3], v1.s[0]
 ; CHECK-NEXT:    fmaxnmv s0, v0.4s
@@ -363,3 +364,12 @@ define float @test_v16f32(<16 x float> %a) nounwind {
   %b = call nnan float @llvm.vector.reduce.fmax.v16f32(<16 x float> %a)
   ret float %b
 }
+
+define double @test_v2f64(<2 x double> %a) nounwind {
+; CHECK-LABEL: test_v2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmaxnmp d0, v0.2d
+; CHECK-NEXT:    ret
+  %b = call double @llvm.vector.reduce.fmax.v2f64(<2 x double> %a)
+  ret double %b
+}
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fmaximum.ll b/llvm/test/CodeGen/AArch64/vecreduce-fmaximum.ll
index 82454564df68d..1204554294fd1 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-fmaximum.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fmaximum.ll
@@ -12,6 +12,7 @@ declare half @llvm.vector.reduce.fmaximum.v11f16(<11 x half> %a)
 declare float @llvm.vector.reduce.fmaximum.v3f32(<3 x float> %a)
 declare fp128 @llvm.vector.reduce.fmaximum.v2f128(<2 x fp128> %a)
 declare float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> %a)
+declare double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> %a)
 
 define half @test_v1f16(<1 x half> %a) nounwind {
 ; CHECK-LABEL: test_v1f16:
@@ -222,3 +223,13 @@ define float @test_v16f32(<16 x float> %a) nounwind {
   %b = call float @llvm.vector.reduce.fmaximum.v16f32(<16 x float> %a)
   ret float %b
 }
+
+define double @test_v2f64(<2 x double> %a) nounwind {
+; CHECK-LABEL: test_v2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov d1, v0.d[1]
+; CHECK-NEXT:    fmax d0, d0, d1
+; CHECK-NEXT:    ret
+  %b = call double @llvm.vector.reduce.fmaximum.v2f64(<2 x double> %a)
+  ret double %b
+}
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fmin-legalization.ll b/llvm/test/CodeGen/AArch64/vecreduce-fmin-legalization.ll
index fade974b07dc7..5e4205855707b 100644
--- a/llvm/test/CodeGen/AArch64/vecreduce-fmin-legalization.ll
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fmin-legalization.ll
@@ -12,6 +12,7 @@ declare half @llvm.vector.reduce.fmin.v11f16(<11 x half> %a)
 declare float @llvm.vector.reduce.fmin.v3f32(<3 x float> %a)
 declare fp128 @llvm.vector.reduce.fmin.v2f128(<2 x fp128> %a)
 declare float @llvm.vector.reduce.fmin.v16f32(<16 x float> %a)
+declare double @llvm.vector.reduce.fmin.v2f64(<2 x double> %a)
 
 define half @test_v1f16(<1 x half> %a) nounwind {
 ; CHECK-LABEL: test_v1f16:
@@ -131,7 +132,7 @@ define half @test_v11f16(<11 x half> %a) nounwind {
 ; CHECK-NOFP-NEXT:    fminnm s0, s0, s1
 ; CHECK-NOFP-NEXT:    fcsel s1, s2, s16, lt
 ; CHECK-NOFP-NEXT:    ldr h2, [x8, :lo12:.LCPI6_0]
-; CHECK-NOFP-NEXT:    mov w8, #2139095040
+; CHECK-NOFP-NEXT:    mov w8, #2139095040 // =0x7f800000
 ; CHECK-NOFP-NEXT:    fcvt h0, s0
 ; CHECK-NOFP-NEXT:    fcvt h1, s1
 ; CHECK-NOFP-NEXT:    fcvt s2, h2
@@ -238,7 +239,7 @@ define half @test_v11f16_ninf(<11 x half> %a) nounwind {
 ; CHECK-NOFP-NEXT:    fminnm s0, s0, s1
 ; CHECK-NOFP-NEXT:    fcsel s1, s2, s16, lt
 ; CHECK-NOFP-NEXT:    ldr h2, [x8, :lo12:.LCPI7_0]
-; CHECK-NOFP-NEXT:    mov w8, #57344
+; CHECK-NOFP-NEXT:    mov w8, #57344 // =0xe000
 ; CHECK-NOFP-NEXT:    movk w8, #18303, lsl #16
 ; CHECK-NOFP-NEXT:    fcvt h0, s0
 ; CHECK-NOFP-NEXT:    fcvt h1, s1
@@ -323,7 +324,7 @@ define half @test_v11f16_ninf(<11 x half> %a) nounwind {
 define float @test_v3f32(<3 x float> %a) nounwind {
 ; CHECK-LABEL: test_v3f32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #2139095040
+; CHECK-NEXT:    mov w8, #2139095040 // =0x7f800000
 ; CHECK-NEXT:    fmov s1, w8
 ; CHECK-NEXT:    mov v0.s[3], v1.s[0]
 ; CHECK-NEXT:    fminnmv s0, v0.4s
@@ -335,7 +336,7 @@ define float @test_v3f32(<3 x float> %a) nounwind {
 define float @test_v3f32_ninf(<3 x float> %a) nounwind {
 ; CHECK-LABEL: test_v3f32_ninf:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #2139095039
+; CHECK-NEXT:    mov w8, #2139095039 // =0x7f7fffff
 ; CHECK-NEXT:    fmov s1, w8
 ; CHECK-NEXT:    mov v0.s[3], v1.s[0]
 ; CHECK-NEXT:    fminnmv s0, v0.4s
@@ -363,3 +364,12 @@ define float @test_v16f32(<16 x float> %a) nounwind {
   %b = call nnan float @llvm.vector.reduce.fmin.v16f32(<16 x float> %a)
   ret float %b
 }
+
+define double @test_v2f64(<2 x double> %a) nounwind {
+; CHECK-LABEL: test_v2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fminnmp d0, v0.2d
+; CHECK-NEXT:    ret
+  %b = call double @llvm.vector.reduce.fmin.v2f64(<2 x double> %a)
+  ret double %b
+}
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-fminimum.ll b/llvm/test/CodeGen/AArch64/vecreduce-fminimum.ll
new file mode 100644
index 0000000000000..3cf48c9a8ab33
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/vecreduce-fminimum.ll
@@ -0,0 +1,235 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mattr=+neon | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-NOFP
+; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mattr=+neon,+fullfp16 | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK-FP
+
+declare half @llvm.vector.reduce.fminimum.v1f16(<1 x half> %a)
+declare float @llvm.vector.reduce.fminimum.v1f32(<1 x float> %a)
+declare double @llvm.vector.reduce.fminimum.v1f64(<1 x double> %a)
+declare fp128 @llvm.vector.reduce.fminimum.v1f128(<1 x fp128> %a)
+
+declare half @llvm.vector.reduce.fminimum.v4f16(<4 x half> %a)
+declare half @llvm.vector.reduce.fminimum.v11f16(<11 x half> %a)
+declare float @llvm.vector.reduce.fminimum.v3f32(<3 x float> %a)
+declare fp128 @llvm.vector.reduce.fminimum.v2f128(<2 x fp128> %a)
+declare float @llvm.vector.reduce.fminimum.v16f32(<16 x float> %a)
+declare double @llvm.vector.reduce.fminimum.v2f64(<2 x double> %a)
+
+define half @test_v1f16(<1 x half> %a) nounwind {
+; CHECK-LABEL: test_v1f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %b = call half @llvm.vector.reduce.fminimum.v1f16(<1 x half> %a)
+  ret half %b
+}
+
+define float @test_v1f32(<1 x float> %a) nounwind {
+; CHECK-LABEL: test_v1f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    // kill: def $s0 killed $s0 killed $q0
+; CHECK-NEXT:    ret
+  %b = call float @llvm.vector.reduce.fminimum.v1f32(<1 x float> %a)
+  ret float %b
+}
+
+define double @test_v1f64(<1 x double> %a) nounwind {
+; CHECK-LABEL: test_v1f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %b = call double @llvm.vector.reduce.fminimum.v1f64(<1 x double> %a)
+  ret double %b
+}
+
+define fp128 @test_v1f128(<1 x fp128> %a) nounwind {
+; CHECK-LABEL: test_v1f128:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    ret
+  %b = call fp128 @llvm.vector.reduce.fminimum.v1f128(<1 x fp128> %a)
+  ret fp128 %b
+}
+
+define half @test_v4f16(<4 x half> %a) nounwind {
+; CHECK-NOFP-LABEL: test_v4f16:
+; CHECK-NOFP:       // %bb.0:
+; CHECK-NOFP-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NOFP-NEXT:    mov h1, v0.h[1]
+; CHECK-NOFP-NEXT:    fcvt s2, h0
+; CHECK-NOFP-NEXT:    fcvt s1, h1
+; CHECK-NOFP-NEXT:    fmin s1, s2, s1
+; CHECK-NOFP-NEXT:    mov h2, v0.h[2]
+; CHECK-NOFP-NEXT:    mov h0, v0.h[3]
+; CHECK-NOFP-NEXT:    fcvt h1, s1
+; CHECK-NOFP-NEXT:    fcvt s2, h2
+; CHECK-NOFP-NEXT:    fcvt s0, h0
+; CHECK-NOFP-NEXT:    fcvt s1, h1
+; CHECK-NOFP-NEXT:    fmin s1, s1, s2
+; CHECK-NOFP-NEXT:    fcvt h1, s1
+; CHECK-NOFP-NEXT:    fcvt s1, h1
+; CHECK-NOFP-NEXT:    fmin s0, s1, s0
+; CHECK-NOFP-NEXT:    fcvt h0, s0
+; CHECK-NOFP-NEXT:    ret
+;
+; CHECK-FP-LABEL: test_v4f16:
+; CHECK-FP:       // %bb.0:
+; CHECK-FP-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-FP-NEXT:    mov h1, v0.h[1]
+; CHECK-FP-NEXT:    mov h2, v0.h[2]
+; CHECK-FP-NEXT:    fmin h1, h0, h1
+; CHECK-FP-NEXT:    mov h0, v0.h[3]
+; CHECK-FP-NEXT:    fmin h1, h1, h2
+; CHECK-FP-NEXT:    fmin h0, h1, h0
+; CHECK-FP-NEXT:    ret
+  %b = call half @llvm.vector.reduce.fminimum.v4f16(<4 x half> %a)
+  ret half %b
+}
+
+define half @test_v11f16(<11 x half> %a) nounwind {
+; CHECK-NOFP-LABEL: test_v11f16:
+; CHECK-NOFP:       // %bb.0:
+; CHECK-NOFP-NEXT:    ldr h16, [sp, #8]
+; CHECK-NOFP-NEXT:    fcvt s1, h1
+; CHECK-NOFP-NEXT:    ldr h17, [sp]
+; CHECK-NOFP-NEXT:    fcvt s0, h0
+; CHECK-NOFP-NEXT:    fcvt s2, h2
+; CHECK-NOFP-NEXT:    fcvt s16, h16
+; CHECK-NOFP-NEXT:    fcvt s17, h17
+; CHECK-NOFP-NEXT:    fmin s1, s1, s16
+; CHECK-NOFP-NEXT:    ldr h16, [sp, #16]
+; CHECK-NOFP-NEXT:    fmin s0, s0, s17
+; CHECK-NOFP-NEXT:    fcvt s16, h16
+; CHECK-NOFP-NEXT:    fcvt h1, s1
+; CHECK-NOFP-NEXT:    fcvt h0, s0
+; CHECK-NOFP-NEXT:    fcvt s1, h1
+; CHECK-NOFP-NEXT:    fcvt s0, h0
+; CHECK-NOFP-NEXT:    fmin s0, s0, s1
+; CHECK-NOFP-NEXT:    fmin s1, s2, s16
+; CHECK-NOFP-NEXT:    fcvt h0, s0
+; CHECK-NOFP-NEXT:    fcvt h1, s1
+; CHECK-NOFP-NEXT:    fcvt s0, h0
+; CHECK-NOFP-NEXT:    fcvt s1, h1
+; CHECK-NOFP-NEXT:    fmin s0, s0, s1
+; CHECK-NOFP-NEXT:    fcvt s1, h3
+; CHECK-NOFP-NEXT:    fcvt h0, s0
+; CHECK-NOFP-NEXT:    fcvt s0, h0
+; CHECK-NOFP-NEXT:    fmin s0, s0, s1
+; CHECK-NOFP-NEXT:    fcvt s1, h4
+; CHECK-NOFP-NEXT:    fcvt h0, s0
+; CHECK-NOFP-NEXT:    fcvt s0, h0
+; CHECK-NOFP-NEXT:    fmin s0, s0, s1
+; CHECK-NOFP-NEXT:    fcvt s1, h5
+; CHECK-NOFP-NEXT:    fcvt h0, s0
+; CHECK-NOFP-NEXT:    fcvt s0, h0
+; CHECK-NOFP-NEXT:    fmin s0, s0, s1
+; CHECK-NOFP-NEXT:    fcvt s1, h6
+; CHECK-NOFP-NEXT:    fcvt h0, s0
+; CHECK-NOFP-NEXT:    fcvt s0, h0
+; CHECK-NOFP-NEXT:    fmin s0, s0, s1
+; CHECK-NOFP-NEXT:    fcvt s1, h7
+; CHECK-NOFP-NEXT:    fcvt h0, s0
+; CHECK-NOFP-NEXT:    fcvt s0, h0
+; CHECK-NOFP-NEXT:    fmin s0, s0, s1
+; CHECK-NOFP-NEXT:    fcvt h0, s0
+; CHECK-NOFP-NEXT:    ret
+;
+; CHECK-FP-LABEL: test_v11f16:
+; CHECK-FP:       // %bb.0:
+; CHECK-FP-NEXT:    // kill: def $h0 killed $h0 def $q0
+; CHECK-FP-NEXT:    // kill: def $h1 killed $h1 def $q1
+; CHECK-FP-NEXT:    // kill: def $h2 killed $h2 def $q2
+; CHECK-FP-NEXT:    // kill: def $h3 killed $h3 def $q3
+; CHECK-FP-NEXT:    // kill: def $h4 killed $h4 def $q4
+; CHECK-FP-NEXT:    mov x8, sp
+; CHECK-FP-NEXT:    // kill: def $h5 killed $h5 def $q5
+; CHECK-FP-NEXT:    // kill: def $h6 killed $h6 def $q6
+; CHECK-FP-NEXT:    // kill: def $h7 killed $h7 def $q7
+; CHECK-FP-NEXT:    mov v0.h[1], v1.h[0]
+; CHECK-FP-NEXT:    movi v1.8h, #124, lsl #8
+; CHECK-FP-NEXT:    mov v0.h[2], v2.h[0]
+; CHECK-FP-NEXT:    ld1 { v1.h }[0], [x8]
+; CHECK-FP-NEXT:    add x8, sp, #8
+; CHECK-FP-NEXT:    mov v0.h[3], v3.h[0]
+; CHECK-FP-NEXT:    ld1 { v1.h }[1], [x8]
+; CHECK-FP-NEXT:    add x8, sp, #16
+; CHECK-FP-NEXT:    mov v0.h[4], v4.h[0]
+; CHECK-FP-NEXT:    ld1 { v1.h }[2], [x8]
+; CHECK-FP-NEXT:    mov v0.h[5], v5.h[0]
+; CHECK-FP-NEXT:    mov v0.h[6], v6.h[0]
+; CHECK-FP-NEXT:    mov v0.h[7], v7.h[0]
+; CHECK-FP-NEXT:    fmin v0.8h, v0.8h, v1.8h
+; CHECK-FP-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-FP-NEXT:    fmin v0.4h, v0.4h, v1.4h
+; CHECK-FP-NEXT:    mov h1, v0.h[1]
+; CHECK-FP-NEXT:    mov h2, v0.h[2]
+; CHECK-FP-NEXT:    fmin h1, h0, h1
+; CHECK-FP-NEXT:    mov h0, v0.h[3]
+; CHECK-FP-NEXT:    fmin h1, h1, h2
+; CHECK-FP-NEXT:    fmin h0, h1, h0
+; CHECK-FP-NEXT:    ret
+  %b = call half @llvm.vector.reduce.fminimum.v11f16(<11 x half> %a)
+  ret half %b
+}
+
+; Neutral element is negative infinity which is chosen for padding the widened
+; vector.
+define float @test_v3f32(<3 x float> %a) nounwind {
+; CHECK-LABEL: test_v3f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #2139095040 // =0x7f800000
+; CHECK-NEXT:    fmov s1, w8
+; CHECK-NEXT:    mov v0.s[3], v1.s[0]
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    fmin v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    mov s1, v0.s[1]
+; CHECK-NEXT:    fmin s0, s0, s1
+; CHECK-NEXT:    ret
+  %b = call float @llvm.vector.reduce.fminimum.v3f32(<3 x float> %a)
+  ret float %b
+}
+
+; Neutral element chosen for padding the widened vector is not negative infinity.
+define float @test_v3f32_ninf(<3 x float> %a) nounwind {
+; CHECK-LABEL: test_v3f32_ninf:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov w8, #2139095039 // =0x7f7fffff
+; CHECK-NEXT:    fmov s1, w8
+; CHECK-NEXT:    mov v0.s[3], v1.s[0]
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    fmin v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    mov s1, v0.s[1]
+; CHECK-NEXT:    fmin s0, s0, s1
+; CHECK-NEXT:    ret
+  %b = call ninf float @llvm.vector.reduce.fminimum.v3f32(<3 x float> %a)
+  ret float %b
+}
+
+; Cannot legalize f128. See PR63267 - The underlying fminimum has no default
+; expansion and no libcalls.
+;define fp128 @test_v2f128(<2 x fp128> %a) nounwind {
+;  %b = call fp128 @llvm.vector.reduce.fminimum.v2f128(<2 x fp128> %a)
+;  ret fp128 %b
+;}
+
+define float @test_v16f32(<16 x float> %a) nounwind {
+; CHECK-LABEL: test_v16f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmin v1.4s, v1.4s, v3.4s
+; CHECK-NEXT:    fmin v0.4s, v0.4s, v2.4s
+; CHECK-NEXT:    fmin v0.4s, v0.4s, v1.4s
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    fmin v0.2s, v0.2s, v1.2s
+; CHECK-NEXT:    mov s1, v0.s[1]
+; CHECK-NEXT:    fmin s0, s0, s1
+; CHECK-NEXT:    ret
+  %b = call float @llvm.vector.reduce.fminimum.v16f32(<16 x float> %a)
+  ret float %b
+}
+
+define double @test_v2f64(<2 x double> %a) nounwind {
+; CHECK-LABEL: test_v2f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    mov d1, v0.d[1]
+; CHECK-NEXT:    fmin d0, d0, d1
+; CHECK-NEXT:    ret
+  %b = call double @llvm.vector.reduce.fminimum.v2f64(<2 x double> %a)
+  ret double %b
+}

From 3fa96d699256c52b0e2df856e3fd7bec3e33fb73 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Mon, 19 Jun 2023 10:47:20 -0400
Subject: [PATCH 035/130] Reland "[gn build] Port 2700da5fe28d (lld/unittests
 etc)"

The lld CL relanded in 6f2e92c10cebca5.

This reverts commit d76b37e6954ad0cf66f1f3c6a9c70328c45859f3.
---
 llvm/utils/gn/secondary/lld/Common/BUILD.gn   |  1 +
 llvm/utils/gn/secondary/lld/test/BUILD.gn     | 81 +++++++++++++------
 .../secondary/lld/unittests/AsLibAll/BUILD.gn | 14 ++++
 .../secondary/lld/unittests/AsLibELF/BUILD.gn | 13 +++
 .../utils/gn/secondary/lld/unittests/BUILD.gn |  8 ++
 .../gn/secondary/llvm/utils/llvm-lit/BUILD.gn |  1 +
 6 files changed, 94 insertions(+), 24 deletions(-)
 create mode 100644 llvm/utils/gn/secondary/lld/unittests/AsLibAll/BUILD.gn
 create mode 100644 llvm/utils/gn/secondary/lld/unittests/AsLibELF/BUILD.gn
 create mode 100644 llvm/utils/gn/secondary/lld/unittests/BUILD.gn

diff --git a/llvm/utils/gn/secondary/lld/Common/BUILD.gn b/llvm/utils/gn/secondary/lld/Common/BUILD.gn
index 0d5713b6fd3f9..b5286545f867d 100644
--- a/llvm/utils/gn/secondary/lld/Common/BUILD.gn
+++ b/llvm/utils/gn/secondary/lld/Common/BUILD.gn
@@ -35,6 +35,7 @@ static_library("Common") {
     "Args.cpp",
     "CommonLinkerContext.cpp",
     "DWARF.cpp",
+    "DriverDispatcher.cpp",
     "ErrorHandler.cpp",
     "Filesystem.cpp",
     "Memory.cpp",
diff --git a/llvm/utils/gn/secondary/lld/test/BUILD.gn b/llvm/utils/gn/secondary/lld/test/BUILD.gn
index 01ee607e582eb..06256e1ba161b 100644
--- a/llvm/utils/gn/secondary/lld/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/lld/test/BUILD.gn
@@ -3,28 +3,42 @@ import("//llvm/triples.gni")
 import("//llvm/utils/gn/build/libs/xml/enable.gni")
 import("//llvm/utils/gn/build/libs/zlib/enable.gni")
 import("//llvm/utils/gn/build/write_cmake_config.gni")
+import("//llvm/utils/llvm-lit/lit_path_function.gni")
 import("lld_lit_site_cfg_files.gni")
 
-write_cmake_config("lit_site_cfg") {
+# The bits common to writing lit.site.cfg.py.in and Unit/lit.site.cfg.py.in.
+template("write_lit_cfg") {
+  write_cmake_config(target_name) {
+    input = invoker.input
+    output = invoker.output
+    values = [
+      "LIT_SITE_CFG_IN_HEADER=" +
+          "## Autogenerated from $input, do not edit\n\n" + lit_path_function,
+      "LLD_BINARY_DIR=" +
+          rebase_path(get_label_info("//lld", "target_out_dir")),
+      "CURRENT_LIBS_DIR=",  # FIXME: for shared builds only (?)
+      "CURRENT_TOOLS_DIR=" + rebase_path("$root_out_dir/bin"),
+      "LLD_SOURCE_DIR=" + rebase_path("//lld"),
+      "LLVM_BINARY_DIR=" +
+          rebase_path(get_label_info("//llvm", "target_out_dir")),
+      "LLVM_SOURCE_DIR=" + rebase_path("//llvm"),
+      "LLVM_TOOLS_DIR=" + rebase_path("$root_out_dir/bin"),
+    ]
+    values += invoker.extra_values
+  }
+}
+
+write_lit_cfg("lit_site_cfg") {
   # Fully-qualified instead of relative for LIT_SITE_CFG_IN_HEADER.
   input = "//lld/test/lit.site.cfg.py.in"
   output = lld_lit_site_cfg_file
   dir = get_path_info(output, "dir")
 
-  values = [
-    "LIT_SITE_CFG_IN_HEADER=## Autogenerated from $input, do not edit",
-    "LLD_BINARY_DIR=" + rebase_path(get_label_info("//lld", "target_out_dir")),
-    "CURRENT_LIBS_DIR=",  # FIXME: for shared builds only (?)
-    "CURRENT_TOOLS_DIR=" + rebase_path("$root_out_dir/bin"),
+  extra_values = [
     "ENABLE_BACKTRACES=1",
-    "LLD_SOURCE_DIR=" + rebase_path("//lld"),
-    "LLVM_BINARY_DIR=" +
-        rebase_path(get_label_info("//llvm", "target_out_dir")),
     "LLVM_HOST_TRIPLE=$llvm_current_triple",
     "LLVM_LIBS_DIR=",  # needed only for shared builds
     "LLVM_LIT_TOOLS_DIR=",  # Intentionally empty, matches cmake build.
-    "LLVM_SOURCE_DIR=" + rebase_path("//llvm"),
-    "LLVM_TOOLS_DIR=" + rebase_path("$root_out_dir/bin"),
     "Python3_EXECUTABLE=$python_path",
     "LLVM_TARGET_TRIPLE=$llvm_target_triple",
 
@@ -35,7 +49,7 @@ write_cmake_config("lit_site_cfg") {
   ]
 
   if (host_os == "win") {
-    values += [
+    extra_values += [
       "LLVM_LIT_ERRC_MESSAGES=no such file or directory;is a directory;" +
           "invalid argument;permission denied",
       "LLVM_ENABLE_PLUGINS=0",
@@ -44,7 +58,7 @@ write_cmake_config("lit_site_cfg") {
       "SHLIBDIR=" + rebase_path("$root_out_dir/bin", dir),
     ]
   } else {
-    values += [
+    extra_values += [
       "LLVM_LIT_ERRC_MESSAGES=",
       "LLVM_ENABLE_PLUGINS=1",
       "SHLIBDIR=" + rebase_path("$root_out_dir/lib", dir),
@@ -52,36 +66,53 @@ write_cmake_config("lit_site_cfg") {
   }
 
   if (host_os == "mac") {
-    values += [ "SHLIBEXT=.dylib" ]
+    extra_values += [ "SHLIBEXT=.dylib" ]
   } else if (host_os == "win") {
-    values += [ "SHLIBEXT=.dll" ]
+    extra_values += [ "SHLIBEXT=.dll" ]
   } else {
-    values += [ "SHLIBEXT=.so" ]
+    extra_values += [ "SHLIBEXT=.so" ]
   }
 
   if (llvm_enable_dia_sdk) {
-    values += [ "LLVM_ENABLE_DIA_SDK=1" ]
+    extra_values += [ "LLVM_ENABLE_DIA_SDK=1" ]
   } else {
-    values += [ "LLVM_ENABLE_DIA_SDK=0" ]  # Must be 0.
+    extra_values += [ "LLVM_ENABLE_DIA_SDK=0" ]  # Must be 0.
   }
 
   if (llvm_enable_libxml2) {
-    values += [ "LLVM_ENABLE_LIBXML2=1" ]
+    extra_values += [ "LLVM_ENABLE_LIBXML2=1" ]
   } else {
-    values += [ "LLVM_ENABLE_LIBXML2=0" ]  # Must be 0.
+    extra_values += [ "LLVM_ENABLE_LIBXML2=0" ]  # Must be 0.
   }
 
   if (llvm_enable_zlib) {
-    values += [ "LLVM_ENABLE_ZLIB=1" ]
+    extra_values += [ "LLVM_ENABLE_ZLIB=1" ]
   } else {
-    values += [ "LLVM_ENABLE_ZLIB=0" ]  # Must be 0.
+    extra_values += [ "LLVM_ENABLE_ZLIB=0" ]  # Must be 0.
   }
 
   if (current_cpu == "x64" || current_cpu == "arm64" ||
       current_cpu == "ppc64") {
-    values += [ "CMAKE_SIZEOF_VOID_P=8" ]
+    extra_values += [ "CMAKE_SIZEOF_VOID_P=8" ]
+  } else {
+    extra_values += [ "CMAKE_SIZEOF_VOID_P=4" ]
+  }
+}
+
+write_lit_cfg("lit_unit_site_cfg") {
+  # Fully-qualified instead of relative for LIT_SITE_CFG_IN_HEADER.
+  input = "//lld/test/Unit/lit.site.cfg.py.in"
+  output = lld_lit_unit_site_cfg_file
+  extra_values = [
+    "ENABLE_SHARED=0",
+    "LLVM_BUILD_MODE=.",
+  ]
+  dir = get_path_info(output, "dir")
+  if (host_os == "win") {
+    # See comment for Windows solink in llvm/utils/gn/build/toolchain/BUILD.gn
+    extra_values += [ "SHLIBDIR=" + rebase_path("$root_out_dir/bin", dir) ]
   } else {
-    values += [ "CMAKE_SIZEOF_VOID_P=4" ]
+    extra_values += [ "SHLIBDIR=" + rebase_path("$root_out_dir/lib", dir) ]
   }
 }
 
@@ -91,7 +122,9 @@ write_cmake_config("lit_site_cfg") {
 group("test") {
   deps = [
     ":lit_site_cfg",
+    ":lit_unit_site_cfg",
     "//lld/tools/lld:symlinks",
+    "//lld/unittests",
     "//llvm/tools/dsymutil",
     "//llvm/tools/llc",
     "//llvm/tools/llvm-ar:symlinks",
diff --git a/llvm/utils/gn/secondary/lld/unittests/AsLibAll/BUILD.gn b/llvm/utils/gn/secondary/lld/unittests/AsLibAll/BUILD.gn
new file mode 100644
index 0000000000000..d6af6a1c73792
--- /dev/null
+++ b/llvm/utils/gn/secondary/lld/unittests/AsLibAll/BUILD.gn
@@ -0,0 +1,14 @@
+import("//third-party/unittest/unittest.gni")
+
+unittest("LLDAsLibAllTests") {
+  configs += [ "//llvm/utils/gn/build:lld_code" ]
+  deps = [
+    "//lld/Common",
+    "//lld/COFF",
+    "//lld/ELF",
+    "//lld/MachO",
+    "//lld/MinGW",
+    "//lld/wasm",
+  ]
+  sources = [ "AllDrivers.cpp" ]
+}
diff --git a/llvm/utils/gn/secondary/lld/unittests/AsLibELF/BUILD.gn b/llvm/utils/gn/secondary/lld/unittests/AsLibELF/BUILD.gn
new file mode 100644
index 0000000000000..e22a0eec1ff02
--- /dev/null
+++ b/llvm/utils/gn/secondary/lld/unittests/AsLibELF/BUILD.gn
@@ -0,0 +1,13 @@
+import("//third-party/unittest/unittest.gni")
+
+unittest("LLDAsLibELFTests") {
+  configs += [ "//llvm/utils/gn/build:lld_code" ]
+  deps = [
+    "//lld/Common",
+    "//lld/ELF",
+  ]
+  sources = [
+    "ROCm.cpp",
+    "SomeDrivers.cpp",
+  ]
+}
diff --git a/llvm/utils/gn/secondary/lld/unittests/BUILD.gn b/llvm/utils/gn/secondary/lld/unittests/BUILD.gn
new file mode 100644
index 0000000000000..c909670f4b1ba
--- /dev/null
+++ b/llvm/utils/gn/secondary/lld/unittests/BUILD.gn
@@ -0,0 +1,8 @@
+group("unittests") {
+  deps = [
+    "AsLibAll:LLDAsLibAllTests",
+    "AsLibELF:LLDAsLibELFTests",
+  ]
+  testonly = true
+}
+
diff --git a/llvm/utils/gn/secondary/llvm/utils/llvm-lit/BUILD.gn b/llvm/utils/gn/secondary/llvm/utils/llvm-lit/BUILD.gn
index f9b71bf33cd04..5d3497e7e0044 100644
--- a/llvm/utils/gn/secondary/llvm/utils/llvm-lit/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/utils/llvm-lit/BUILD.gn
@@ -31,6 +31,7 @@ write_cmake_config("llvm-lit") {
     "//clang/test:lit_site_cfg",
     "//clang/test:lit_unit_site_cfg",
     "//lld/test:lit_site_cfg",
+    "//lld/test:lit_unit_site_cfg",
     "//lldb/test:lit_api_site_cfg",
     "//lldb/test:lit_shell_site_cfg",
     "//lldb/test:lit_site_cfg",

From f25003dfc28c0dae77b51782ae9e1b8d9fa69337 Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Mon, 19 Jun 2023 10:48:35 -0400
Subject: [PATCH 036/130] [gn] port 3956a34e4fc6

---
 llvm/utils/gn/secondary/libcxx/src/BUILD.gn | 1 +
 1 file changed, 1 insertion(+)

diff --git a/llvm/utils/gn/secondary/libcxx/src/BUILD.gn b/llvm/utils/gn/secondary/libcxx/src/BUILD.gn
index fbdf26f50ec2a..330c280416f6d 100644
--- a/llvm/utils/gn/secondary/libcxx/src/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/src/BUILD.gn
@@ -149,6 +149,7 @@ cxx_sources = [
   "mutex_destructor.cpp",
   "new.cpp",
   "new_handler.cpp",
+  "new_helpers.cpp",
   "optional.cpp",
   "random.cpp",
   "random_shuffle.cpp",

From ce02978bd45086890eef0c407b456bcfcf7d4aac Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Mon, 19 Jun 2023 10:49:44 -0400
Subject: [PATCH 037/130] [gn] port c7d3c84449f4

---
 llvm/utils/gn/secondary/libcxx/src/BUILD.gn | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/libcxx/src/BUILD.gn b/llvm/utils/gn/secondary/libcxx/src/BUILD.gn
index 330c280416f6d..50ffd388a0d0f 100644
--- a/llvm/utils/gn/secondary/libcxx/src/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/src/BUILD.gn
@@ -203,10 +203,16 @@ if (libcxx_enable_debug_mode) {
 }
 if (libcxx_enable_filesystem) {
   cxx_sources += [
+    "filesystem/directory_entry.cpp",
     "filesystem/directory_iterator.cpp",
-    "filesystem/filesystem_common.h",
+    "filesystem/file_descriptor.h",
+    "filesystem/filesystem_clock.cpp",
+    "filesystem/filesystem_error.cpp",
     "filesystem/operations.cpp",
+    "filesystem/path.cpp",
+    "filesystem/path_parser.h",
     "filesystem/posix_compat.h",
+    "filesystem/time_utils.h",
   ]
   if (libcxx_use_compiler_rt) {
     cxx_sources += [ "filesystem/int128_builtins.cpp" ]

From 3f1e45005fd19314abd4565be0c75e15091f1245 Mon Sep 17 00:00:00 2001
From: Nicolas Vasilache <nicolasvasilache@users.noreply.github.com>
Date: Mon, 19 Jun 2023 14:51:12 +0000
Subject: [PATCH 038/130] Revert "[mlir][Vector] Let VectorToLLVM operate on
 non-ModuleOp"

This reverts commit aabea3d320c87561fe98b56c9f53cca1c6d18869.

That commit had mistakenly squashed spurious changes in.
---
 mlir/include/mlir/Conversion/Passes.td | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index 1e16d7b1dbdf8..ac53af4709098 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -361,6 +361,10 @@ def GpuToLLVMConversionPass : Pass<"gpu-to-llvm", "ModuleOp"> {
   }];
 
   let options = [
+    Option<"hostBarePtrCallConv", "use-bare-pointers-for-host", "bool",
+           /*default=*/"false",
+           "Use bare pointers to pass memref arguments to host functions. "
+           "All memrefs must have static shape.">,
     Option<"kernelBarePtrCallConv", "use-bare-pointers-for-kernels", "bool",
            /*default=*/"false",
              "Use bare pointers to pass memref arguments to kernels. "
@@ -422,6 +426,10 @@ def ConvertGpuOpsToNVVMOps : Pass<"convert-gpu-to-nvvm", "gpu::GPUModuleOp"> {
            "Bitwidth of the index type, 0 to use size of machine word">,
     Option<"hasRedux", "has-redux", "bool", /*default=*/"false",
            "Target gpu supports redux">,
+    Option<"useBarePtrCallConv", "use-bare-ptr-memref-call-conv", "bool",
+           /*default=*/"false",
+           "Replace memref arguments in GPU functions with bare pointers. "
+           "All memrefs must have static shape.">,
     Option<"useOpaquePointers", "use-opaque-pointers", "bool",
                    /*default=*/"true", "Generate LLVM IR using opaque pointers "
                    "instead of typed pointers">,
@@ -1057,7 +1065,7 @@ def ConvertVectorToSCF : Pass<"convert-vector-to-scf"> {
 // VectorToLLVM
 //===----------------------------------------------------------------------===//
 
-def ConvertVectorToLLVMPass : Pass<"convert-vector-to-llvm"> {
+def ConvertVectorToLLVMPass : Pass<"convert-vector-to-llvm", "ModuleOp"> {
   let summary = "Lower the operations from the vector dialect into the LLVM "
                 "dialect";
   let description = [{
@@ -1092,6 +1100,10 @@ def ConvertVectorToLLVMPass : Pass<"convert-vector-to-llvm"> {
            "bool", /*default=*/"false",
            "Enables the use of ArmSVE dialect while lowering the vector "
        "dialect.">,
+    Option<"armSME", "enable-arm-sme",
+           "bool", /*default=*/"false",
+           "Enables the use of ArmSME dialect while lowering the vector "
+       "dialect.">,
     Option<"x86Vector", "enable-x86vector",
            "bool", /*default=*/"false",
            "Enables the use of X86Vector dialect while lowering the vector "

From 72e598957c63c507f1d2dd63c365b64c11cf704f Mon Sep 17 00:00:00 2001
From: Nicolas Vasilache <nicolasvasilache@users.noreply.github.com>
Date: Mon, 19 Jun 2023 14:54:14 +0000
Subject: [PATCH 039/130] [mlir][Vector] Let VectorToLLVM operate on
 non-ModuleOp

---
 mlir/include/mlir/Conversion/Passes.td | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index ac53af4709098..287386411f935 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -1065,7 +1065,7 @@ def ConvertVectorToSCF : Pass<"convert-vector-to-scf"> {
 // VectorToLLVM
 //===----------------------------------------------------------------------===//
 
-def ConvertVectorToLLVMPass : Pass<"convert-vector-to-llvm", "ModuleOp"> {
+def ConvertVectorToLLVMPass : Pass<"convert-vector-to-llvm"> {
   let summary = "Lower the operations from the vector dialect into the LLVM "
                 "dialect";
   let description = [{

From 3f05d044f491b97596fdaab7614f81cfb70ba54d Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Sat, 17 Jun 2023 12:00:23 +0200
Subject: [PATCH 040/130] [libc++] Update status after Varna meeting.

This updates:
- The status tables
- Feature test macros
- New headers for modules
The latter avoids forgetting about modules when implementing the feature
in a new header.

Reviewed By: #libc, philnik

Differential Revision: https://reviews.llvm.org/D153192
---
 libcxx/docs/FeatureTestMacroTable.rst         | 732 +++++++--------
 libcxx/docs/Status/Cxx2cIssues.csv            |  19 +-
 libcxx/docs/Status/Cxx2cPapers.csv            |  27 +
 libcxx/docs/Status/FormatIssues.csv           |   3 +
 libcxx/include/version                        |  40 +-
 libcxx/modules/CMakeLists.txt                 |   3 +
 libcxx/modules/std.cppm                       |   3 +
 libcxx/modules/std/hazard_pointer.cppm        |  30 +
 libcxx/modules/std/rcu.cppm                   |  29 +
 libcxx/modules/std/text_encoding.cppm         |  26 +
 .../bitset.version.compile.pass.cpp           |  34 +
 .../fstream.version.compile.pass.cpp          |  73 ++
 .../functional.version.compile.pass.cpp       |  78 +-
 .../map.version.compile.pass.cpp              |  52 +-
 .../mdspan.version.compile.pass.cpp           |  38 +-
 .../memory.version.compile.pass.cpp           |  34 +
 .../ratio.version.compile.pass.cpp            |  71 ++
 .../set.version.compile.pass.cpp              |  50 +-
 .../sstream.version.compile.pass.cpp          |  73 ++
 .../string.version.compile.pass.cpp           |  43 +
 .../type_traits.version.compile.pass.cpp      |  34 +
 .../unordered_map.version.compile.pass.cpp    |  52 +-
 .../unordered_set.version.compile.pass.cpp    |  50 +-
 .../version.version.compile.pass.cpp          | 839 ++++++++++++++----
 .../generate_feature_test_macro_components.py | 126 ++-
 25 files changed, 1982 insertions(+), 577 deletions(-)
 create mode 100644 libcxx/modules/std/hazard_pointer.cppm
 create mode 100644 libcxx/modules/std/rcu.cppm
 create mode 100644 libcxx/modules/std/text_encoding.cppm
 create mode 100644 libcxx/test/std/language.support/support.limits/support.limits.general/fstream.version.compile.pass.cpp
 create mode 100644 libcxx/test/std/language.support/support.limits/support.limits.general/ratio.version.compile.pass.cpp
 create mode 100644 libcxx/test/std/language.support/support.limits/support.limits.general/sstream.version.compile.pass.cpp

diff --git a/libcxx/docs/FeatureTestMacroTable.rst b/libcxx/docs/FeatureTestMacroTable.rst
index f24e210926473..45e580f397756 100644
--- a/libcxx/docs/FeatureTestMacroTable.rst
+++ b/libcxx/docs/FeatureTestMacroTable.rst
@@ -21,359 +21,391 @@ Status
     :name: feature-status-table
     :widths: auto
 
-    ================================================= =================
-    Macro Name                                        Value
-    ================================================= =================
+    =================================================== =================
+    Macro Name                                          Value
+    =================================================== =================
     **C++ 14**
-    -------------------------------------------------------------------
-    ``__cpp_lib_chrono_udls``                         ``201304L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_complex_udls``                        ``201309L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_exchange_function``                   ``201304L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_generic_associative_lookup``          ``201304L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_integer_sequence``                    ``201304L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_integral_constant_callable``          ``201304L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_is_final``                            ``201402L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_is_null_pointer``                     ``201309L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_make_reverse_iterator``               ``201402L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_make_unique``                         ``201304L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_null_iterators``                      ``201304L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_quoted_string_io``                    ``201304L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_result_of_sfinae``                    ``201210L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_robust_nonmodifying_seq_ops``         ``201304L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_shared_timed_mutex``                  ``201402L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_string_udls``                         ``201304L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_transformation_trait_aliases``        ``201304L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_transparent_operators``               ``201210L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_tuple_element_t``                     ``201402L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_tuples_by_type``                      ``201304L``
-    ------------------------------------------------- -----------------
+    ---------------------------------------------------------------------
+    ``__cpp_lib_chrono_udls``                           ``201304L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_complex_udls``                          ``201309L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_exchange_function``                     ``201304L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_generic_associative_lookup``            ``201304L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_integer_sequence``                      ``201304L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_integral_constant_callable``            ``201304L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_is_final``                              ``201402L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_is_null_pointer``                       ``201309L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_make_reverse_iterator``                 ``201402L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_make_unique``                           ``201304L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_null_iterators``                        ``201304L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_quoted_string_io``                      ``201304L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_result_of_sfinae``                      ``201210L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_robust_nonmodifying_seq_ops``           ``201304L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_shared_timed_mutex``                    ``201402L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_string_udls``                           ``201304L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_transformation_trait_aliases``          ``201304L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_transparent_operators``                 ``201210L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_tuple_element_t``                       ``201402L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_tuples_by_type``                        ``201304L``
+    --------------------------------------------------- -----------------
     **C++ 17**
-    -------------------------------------------------------------------
-    ``__cpp_lib_addressof_constexpr``                 ``201603L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_allocator_traits_is_always_equal``    ``201411L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_any``                                 ``201606L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_apply``                               ``201603L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_array_constexpr``                     ``201603L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_as_const``                            ``201510L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_atomic_is_always_lock_free``          ``201603L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_bool_constant``                       ``201505L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_boyer_moore_searcher``                ``201603L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_byte``                                ``201603L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_chrono``                              ``201611L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_clamp``                               ``201603L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_enable_shared_from_this``             ``201603L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_execution``                           *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_filesystem``                          ``201703L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_gcd_lcm``                             ``201606L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_hardware_interference_size``          ``201703L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_has_unique_object_representations``   ``201606L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_hypot``                               ``201603L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_incomplete_container_elements``       ``201505L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_invoke``                              ``201411L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_is_aggregate``                        ``201703L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_is_invocable``                        ``201703L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_is_swappable``                        ``201603L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_launder``                             ``201606L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_logical_traits``                      ``201510L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_make_from_tuple``                     ``201606L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_map_try_emplace``                     ``201411L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_math_special_functions``              *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_memory_resource``                     ``201603L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_node_extract``                        ``201606L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_nonmember_container_access``          ``201411L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_not_fn``                              ``201603L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_optional``                            ``201606L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_parallel_algorithm``                  *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_raw_memory_algorithms``               ``201606L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_sample``                              ``201603L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_scoped_lock``                         ``201703L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_shared_mutex``                        ``201505L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_shared_ptr_arrays``                   ``201611L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_shared_ptr_weak_type``                ``201606L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_string_view``                         ``201606L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_to_chars``                            *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_transparent_operators``               ``201510L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_type_trait_variable_templates``       ``201510L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_uncaught_exceptions``                 ``201411L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_unordered_map_try_emplace``           ``201411L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_variant``                             ``202102L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_void_t``                              ``201411L``
-    ------------------------------------------------- -----------------
+    ---------------------------------------------------------------------
+    ``__cpp_lib_addressof_constexpr``                   ``201603L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_allocator_traits_is_always_equal``      ``201411L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_any``                                   ``201606L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_apply``                                 ``201603L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_array_constexpr``                       ``201603L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_as_const``                              ``201510L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_atomic_is_always_lock_free``            ``201603L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_bool_constant``                         ``201505L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_boyer_moore_searcher``                  ``201603L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_byte``                                  ``201603L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_chrono``                                ``201611L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_clamp``                                 ``201603L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_enable_shared_from_this``               ``201603L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_execution``                             *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_filesystem``                            ``201703L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_gcd_lcm``                               ``201606L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_hardware_interference_size``            ``201703L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_has_unique_object_representations``     ``201606L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_hypot``                                 ``201603L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_incomplete_container_elements``         ``201505L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_invoke``                                ``201411L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_is_aggregate``                          ``201703L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_is_invocable``                          ``201703L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_is_swappable``                          ``201603L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_launder``                               ``201606L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_logical_traits``                        ``201510L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_make_from_tuple``                       ``201606L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_map_try_emplace``                       ``201411L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_math_special_functions``                *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_memory_resource``                       ``201603L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_node_extract``                          ``201606L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_nonmember_container_access``            ``201411L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_not_fn``                                ``201603L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_optional``                              ``201606L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_parallel_algorithm``                    *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_raw_memory_algorithms``                 ``201606L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_sample``                                ``201603L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_scoped_lock``                           ``201703L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_shared_mutex``                          ``201505L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_shared_ptr_arrays``                     ``201611L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_shared_ptr_weak_type``                  ``201606L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_string_view``                           ``201606L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_to_chars``                              *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_transparent_operators``                 ``201510L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_type_trait_variable_templates``         ``201510L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_uncaught_exceptions``                   ``201411L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_unordered_map_try_emplace``             ``201411L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_variant``                               ``202102L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_void_t``                                ``201411L``
+    --------------------------------------------------- -----------------
     **C++ 20**
-    -------------------------------------------------------------------
-    ``__cpp_lib_array_constexpr``                     ``201811L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_assume_aligned``                      ``201811L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_atomic_flag_test``                    ``201907L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_atomic_float``                        *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_atomic_lock_free_type_aliases``       ``201907L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_atomic_ref``                          *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_atomic_shared_ptr``                   *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_atomic_value_initialization``         ``201911L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_atomic_wait``                         ``201907L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_barrier``                             ``201907L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_bind_front``                          ``201907L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_bit_cast``                            ``201806L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_bitops``                              *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_bounded_array_traits``                ``201902L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_char8_t``                             ``201907L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_concepts``                            ``202002L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_constexpr_algorithms``                ``201806L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_constexpr_complex``                   ``201711L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_constexpr_dynamic_alloc``             ``201907L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_constexpr_functional``                ``201907L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_constexpr_iterator``                  ``201811L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_constexpr_memory``                    ``201811L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_constexpr_numeric``                   ``201911L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_constexpr_string``                    ``201907L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_constexpr_string_view``               ``201811L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_constexpr_tuple``                     ``201811L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_constexpr_utility``                   ``201811L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_constexpr_vector``                    ``201907L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_coroutine``                           ``201902L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_destroying_delete``                   ``201806L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_endian``                              ``201907L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_erase_if``                            ``202002L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_execution``                           *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_format``                              *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_generic_unordered_lookup``            ``201811L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_int_pow2``                            ``202002L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_integer_comparison_functions``        ``202002L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_interpolate``                         ``201902L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_is_constant_evaluated``               ``201811L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_is_layout_compatible``                *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_is_nothrow_convertible``              ``201806L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_is_pointer_interconvertible``         *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_jthread``                             *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_latch``                               ``201907L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_list_remove_return_type``             ``201806L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_math_constants``                      ``201907L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_move_iterator_concept``               ``202207L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_polymorphic_allocator``               ``201902L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_ranges``                              ``202106L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_remove_cvref``                        ``201711L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_semaphore``                           ``201907L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_shared_ptr_arrays``                   ``201707L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_shift``                               ``201806L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_smart_ptr_for_overwrite``             *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_source_location``                     ``201907L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_span``                                ``202002L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_ssize``                               ``201902L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_starts_ends_with``                    ``201711L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_string_view``                         ``201803L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_syncbuf``                             *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_three_way_comparison``                *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_to_address``                          ``201711L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_to_array``                            ``201907L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_type_identity``                       ``201806L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_unwrap_ref``                          ``201811L``
-    ------------------------------------------------- -----------------
+    ---------------------------------------------------------------------
+    ``__cpp_lib_array_constexpr``                       ``201811L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_assume_aligned``                        ``201811L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_atomic_flag_test``                      ``201907L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_atomic_float``                          *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_atomic_lock_free_type_aliases``         ``201907L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_atomic_ref``                            *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_atomic_shared_ptr``                     *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_atomic_value_initialization``           ``201911L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_atomic_wait``                           ``201907L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_barrier``                               ``201907L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_bind_front``                            ``201907L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_bit_cast``                              ``201806L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_bitops``                                *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_bounded_array_traits``                  ``201902L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_char8_t``                               ``201907L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_concepts``                              ``202002L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_constexpr_algorithms``                  ``201806L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_constexpr_complex``                     ``201711L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_constexpr_dynamic_alloc``               ``201907L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_constexpr_functional``                  ``201907L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_constexpr_iterator``                    ``201811L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_constexpr_memory``                      ``201811L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_constexpr_numeric``                     ``201911L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_constexpr_string``                      ``201907L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_constexpr_string_view``                 ``201811L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_constexpr_tuple``                       ``201811L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_constexpr_utility``                     ``201811L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_constexpr_vector``                      ``201907L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_coroutine``                             ``201902L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_destroying_delete``                     ``201806L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_endian``                                ``201907L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_erase_if``                              ``202002L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_execution``                             *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_format``                                *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_generic_unordered_lookup``              ``201811L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_int_pow2``                              ``202002L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_integer_comparison_functions``          ``202002L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_interpolate``                           ``201902L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_is_constant_evaluated``                 ``201811L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_is_layout_compatible``                  *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_is_nothrow_convertible``                ``201806L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_is_pointer_interconvertible``           *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_jthread``                               *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_latch``                                 ``201907L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_list_remove_return_type``               ``201806L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_math_constants``                        ``201907L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_move_iterator_concept``                 ``202207L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_polymorphic_allocator``                 ``201902L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_ranges``                                ``202106L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_remove_cvref``                          ``201711L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_semaphore``                             ``201907L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_shared_ptr_arrays``                     ``201707L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_shift``                                 ``201806L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_smart_ptr_for_overwrite``               *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_source_location``                       ``201907L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_span``                                  ``202002L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_ssize``                                 ``201902L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_starts_ends_with``                      ``201711L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_string_view``                           ``201803L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_syncbuf``                               *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_three_way_comparison``                  *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_to_address``                            ``201711L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_to_array``                              ``201907L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_type_identity``                         ``201806L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_unwrap_ref``                            ``201811L``
+    --------------------------------------------------- -----------------
     **C++ 23**
-    -------------------------------------------------------------------
-    ``__cpp_lib_adaptor_iterator_pair_constructor``   ``202106L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_allocate_at_least``                   ``202106L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_associative_heterogeneous_erasure``   *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_bind_back``                           *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_byteswap``                            ``202110L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_constexpr_bitset``                    ``202207L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_constexpr_charconv``                  ``202207L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_constexpr_cmath``                     *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_constexpr_memory``                    ``202202L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_constexpr_typeinfo``                  ``202106L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_expected``                            ``202211L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_format_ranges``                       ``202207L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_formatters``                          *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_forward_like``                        ``202207L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_invoke_r``                            ``202106L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_is_scoped_enum``                      ``202011L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_mdspan``                              *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_move_only_function``                  *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_optional``                            ``202110L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_out_ptr``                             *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_ranges_as_rvalue``                    ``202207L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_ranges_chunk``                        *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_ranges_chunk_by``                     *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_ranges_iota``                         *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_ranges_join_with``                    *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_ranges_slide``                        *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_ranges_starts_ends_with``             *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_ranges_to_container``                 *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_ranges_zip``                          *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_reference_from_temporary``            *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_spanstream``                          *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_stacktrace``                          *unimplemented*
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_stdatomic_h``                         ``202011L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_string_contains``                     ``202011L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_string_resize_and_overwrite``         ``202110L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_to_underlying``                       ``202102L``
-    ------------------------------------------------- -----------------
-    ``__cpp_lib_unreachable``                         ``202202L``
-    ------------------------------------------------- -----------------
+    ---------------------------------------------------------------------
+    ``__cpp_lib_adaptor_iterator_pair_constructor``     ``202106L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_allocate_at_least``                     ``202106L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_associative_heterogeneous_erasure``     *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_bind_back``                             *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_byteswap``                              ``202110L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_constexpr_bitset``                      ``202207L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_constexpr_charconv``                    ``202207L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_constexpr_cmath``                       *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_constexpr_memory``                      ``202202L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_constexpr_typeinfo``                    ``202106L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_expected``                              ``202211L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_format_ranges``                         ``202207L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_formatters``                            *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_forward_like``                          ``202207L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_invoke_r``                              ``202106L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_is_scoped_enum``                        ``202011L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_mdspan``                                *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_move_only_function``                    *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_optional``                              ``202110L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_out_ptr``                               *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_ranges_as_rvalue``                      ``202207L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_ranges_chunk``                          *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_ranges_chunk_by``                       *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_ranges_iota``                           *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_ranges_join_with``                      *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_ranges_slide``                          *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_ranges_starts_ends_with``               *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_ranges_to_container``                   *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_ranges_zip``                            *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_reference_from_temporary``              *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_spanstream``                            *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_stacktrace``                            *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_stdatomic_h``                           ``202011L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_string_contains``                       ``202011L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_string_resize_and_overwrite``           ``202110L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_to_string``                             *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_to_underlying``                         ``202102L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_unreachable``                           ``202202L``
+    --------------------------------------------------- -----------------
     **C++ 26**
-    ===================================================================
+    ---------------------------------------------------------------------
+    ``__cpp_lib_associative_heterogeneous_insertion``   *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_bind_back``                             *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_bind_front``                            ``202306L``
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_bitset``                                *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_copyable_function``                     *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_fstream_native_handle``                 *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_function_ref``                          *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_hazard_pointer``                        *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_ratio``                                 *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_rcu``                                   *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_smart_ptr_owner_equality``              *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_sstream_from_string_view``              *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_submdspan``                             *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_text_encoding``                         *unimplemented*
+    --------------------------------------------------- -----------------
+    ``__cpp_lib_within_lifetime``                       *unimplemented*
+    =================================================== =================
 
diff --git a/libcxx/docs/Status/Cxx2cIssues.csv b/libcxx/docs/Status/Cxx2cIssues.csv
index b1d423e127f26..00da9135e0aae 100644
--- a/libcxx/docs/Status/Cxx2cIssues.csv
+++ b/libcxx/docs/Status/Cxx2cIssues.csv
@@ -1,5 +1,22 @@
 "Issue #","Issue Name","Meeting","Status","First released version","Labels"
+"`2994 <https://wg21.link/LWG2994>`__","Needless UB for ``basic_string`` and ``basic_string_view``","Varna June 2023","","",""
+"`3884 <https://wg21.link/LWG3884>`__","``flat_foo`` is missing allocator-extended copy/move constructors","Varna June 2023","","","|flat_containers|"
+"`3885 <https://wg21.link/LWG3885>`__","``op`` should be in [zombie.names]","Varna June 2023","","",""
+"`3887 <https://wg21.link/LWG3887>`__","Version macro for ``allocate_at_least``","Varna June 2023","","",""
+"`3893 <https://wg21.link/LWG3893>`__","LWG 3661 broke ``atomic<shared_ptr<T>> a; a = nullptr;``","Varna June 2023","","",""
+"`3894 <https://wg21.link/LWG3894>`__","``generator::promise_type::yield_value(ranges::elements_of<Rng, Alloc>)`` should not be ``noexcept``","Varna June 2023","","",""
+"`3903 <https://wg21.link/LWG3903>`__","span destructor is redundantly noexcept","Varna June 2023","","",""
+"`3904 <https://wg21.link/LWG3904>`__","``lazy_split_view::outer-iterator``'s const-converting constructor isn't setting ``trailing_empty_``","Varna June 2023","","","|ranges|"
+"`3905 <https://wg21.link/LWG3905>`__","Type of ``std::fexcept_t``","Varna June 2023","","",""
+"`3912 <https://wg21.link/LWG3912>`__","``enumerate_view::iterator::operator-`` should be ``noexcept``","Varna June 2023","","","|ranges|"
+"`3914 <https://wg21.link/LWG3914>`__","Inconsistent template-head of ``ranges::enumerate_view``","Varna June 2023","","","|ranges|"
+"`3915 <https://wg21.link/LWG3915>`__","Redundant paragraph about expression variations","Varna June 2023","","","|ranges|"
+"`3925 <https://wg21.link/LWG3925>`__","Concept ``formattable``'s definition is incorrect","Varna June 2023","|In Progress|","","|format|"
+"`3927 <https://wg21.link/LWG3927>`__","Unclear preconditions for ``operator[]`` for sequence containers","Varna June 2023","","",""
+"`3935 <https://wg21.link/LWG3935>`__","``template<class X> constexpr complex& operator=(const complex<X>&)`` has no specification","Varna June 2023","","",""
+"`3938 <https://wg21.link/LWG3938>`__","Cannot use ``std::expected`` monadic ops with move-only ``error_type``","Varna June 2023","","",""
+"`3940 <https://wg21.link/LWG3940>`__","``std::expected<void, E>::value()`` also needs ``E`` to be copy constructible","Varna June 2023","","",""
 "","","","","",""
 "`3343 <https://wg21.link/LWG3343>`__","Ordering of calls to ``unlock()`` and ``notify_all()`` in Effects element of ``notify_all_at_thread_exit()`` should be reversed","Not Yet Adopted","|Complete|","16.0",""
-"`3892 <https://wg21.link/LWG3892>`__","Incorrect formatting of nested ranges and tuples","Not Yet Adopted","|Complete|","17.0",""
+"`3892 <https://wg21.link/LWG3892>`__","Incorrect formatting of nested ranges and tuples","Not Yet Adopted","|Complete|","17.0","|format|"
 "","","","","",""
diff --git a/libcxx/docs/Status/Cxx2cPapers.csv b/libcxx/docs/Status/Cxx2cPapers.csv
index 8b6942024d865..494e70f22a356 100644
--- a/libcxx/docs/Status/Cxx2cPapers.csv
+++ b/libcxx/docs/Status/Cxx2cPapers.csv
@@ -1,2 +1,29 @@
 "Paper #","Group","Paper Name","Meeting","Status","First released version","Labels"
+"`P2497R0 <https://wg21.link/P2497R0>`__","LWG","Testing for success or failure of ``<charconv>`` functions","Varna June 2023","","",""
+"`P2592R3 <https://wg21.link/P2592R3>`__","LWG","Hashing support for ``std::chrono`` value classes","Varna June 2023","","",""
+"`P2587R3 <https://wg21.link/P2587R3>`__","LWG","``to_string`` or not ``to_string``","Varna June 2023","","","|format|"
+"`P2562R1 <https://wg21.link/P2562R1>`__","LWG","``constexpr`` Stable Sorting","Varna June 2023","","",""
+"`P2545R4 <https://wg21.link/P2545R4>`__","LWG","Read-Copy Update (RCU)","Varna June 2023","","",""
+"`P2530R3 <https://wg21.link/P2530R3>`__","LWG","Hazard Pointers for C++26","Varna June 2023","","",""
+"`P2538R1 <https://wg21.link/P2538R1>`__","LWG","ADL-proof ``std::projected``","Varna June 2023","","","|ranges|"
+"`P2495R3 <https://wg21.link/P2495R3>`__","LWG","Interfacing ``stringstreams`` with ``string_view``","Varna June 2023","","",""
+"`P2510R3 <https://wg21.link/P2510R3>`__","LWG","Formatting pointers","Varna June 2023","|In Progress|","","|format|"
+"`P2198R7 <https://wg21.link/P2198R7>`__","LWG","Freestanding Feature-Test Macros and Implementation-Defined Extensions","Varna June 2023","","",""
+"`P2338R4 <https://wg21.link/P2338R4>`__","LWG","Freestanding Library: Character primitives and the C library","Varna June 2023","","",""
+"`P2013R5 <https://wg21.link/P2013R5>`__","LWG","Freestanding Language: Optional ``::operator new``","Varna June 2023","","",""
+"`P2363R5 <https://wg21.link/P2363R5>`__","LWG","Extending associative containers with the remaining heterogeneous overloads","Varna June 2023","","",""
+"`P1901R2 <https://wg21.link/P1901R2>`__","LWG","Enabling the Use of ``weak_ptr`` as Keys in Unordered Associative Containers","Varna June 2023","","",""
+"`P1885R12 <https://wg21.link/P1885R12>`__","LWG","Naming Text Encodings to Demystify Them","Varna June 2023","","",""
+"`P0792R14 <https://wg21.link/P0792R14>`__","LWG","``function_ref``: a type-erased callable reference","Varna June 2023","","",""
+"`P2874R2 <https://wg21.link/P2874R2>`__","LWG","Mandating Annex D Require No More","Varna June 2023","","",""
+"`P2757R3 <https://wg21.link/P2757R3>`__","LWG","Type-checking format args","Varna June 2023","","","|format|"
+"`P2637R3 <https://wg21.link/P2637R3>`__","LWG","Member ``visit``","Varna June 2023","","","|format|"
+"`P2641R4 <https://wg21.link/P2641R4>`__","CWG, LWG","Checking if a ``union`` alternative is active","Varna June 2023","","",""
+"`P1759R6 <https://wg21.link/P1759R6>`__","LWG","Native handles and file streams","Varna June 2023","","",""
+"`P2697R1 <https://wg21.link/P2697R1>`__","LWG","Interfacing ``bitset`` with ``string_view``","Varna June 2023","","",""
+"`P1383R2 <https://wg21.link/P1383R2>`__","LWG","More ``constexpr`` for ``<cmath>`` and ``<complex>``","Varna June 2023","","",""
+"`P2734R0 <https://wg21.link/P2734R0>`__","LWG","Adding the new SI prefixes","Varna June 2023","","",""
+"`P2548R6 <https://wg21.link/P2548R6>`__","LWG","``copyable_function``","Varna June 2023","","",""
+"`P2714R1 <https://wg21.link/P2714R1>`__","LWG","Bind front and back to NTTP callables","Varna June 2023","","",""
+"`P2630R4 <https://wg21.link/P2630R4>`__","LWG","``submdspan``","Varna June 2023","","",""
 "","","","","","",""
diff --git a/libcxx/docs/Status/FormatIssues.csv b/libcxx/docs/Status/FormatIssues.csv
index 20a0be7fc7bb2..0ac1fa6108750 100644
--- a/libcxx/docs/Status/FormatIssues.csv
+++ b/libcxx/docs/Status/FormatIssues.csv
@@ -14,6 +14,9 @@ Number,Name,Standard,Assignee,Status,First released version
 "`P2675R1 <https://wg21.link/P2675R1>`__","``format``'s width estimation is too approximate and not forward compatible","C++23","Mark de Wever","|Complete|", Clang 17
 "`P2572R1 <https://wg21.link/P2572R1>`__","``std::format`` fill character allowances","C++23","Mark de Wever","|Complete|", Clang 17
 "`P2693R1 <https://wg21.link/P2693R1>`__","Formatting ``thread::id`` and ``stacktrace``","C++23","Mark de Wever","|In progress|"
+"`P2510R3 <https://wg21.link/P2510R3>`__","Formatting pointers","C++26","Mark de Wever","|In Progress|",
+"`P2757R3 <https://wg21.link/P2757R3>`__","Type-checking format args","C++26","","",
+"`P2637R3 <https://wg21.link/P2637R3>`__","Member ``visit``","C++26","","",
 `P1361 <https://wg21.link/P1361>`_,"Integration of chrono with text formatting","C++20",Mark de Wever,|In Progress|,
 `P2372 <https://wg21.link/P2372>`__,"Fixing locale handling in chrono formatters","C++20",Mark de Wever,|In Progress|,
 "`P2419R2 <https://wg21.link/P2419R2>`__","Clarify handling of encodings in localized formatting of chrono types","C++23",
diff --git a/libcxx/include/version b/libcxx/include/version
index a1525397b988a..5f69eff42b0a1 100644
--- a/libcxx/include/version
+++ b/libcxx/include/version
@@ -28,6 +28,8 @@ __cpp_lib_array_constexpr                               201811L <array> <iterato
 __cpp_lib_as_const                                      201510L <utility>
 __cpp_lib_associative_heterogeneous_erasure             202110L <map> <set> <unordered_map>
                                                                 <unordered_set>
+__cpp_lib_associative_heterogeneous_insertion           202306L <map> <set> <unordered_map>
+                                                                <unordered_set>
 __cpp_lib_assume_aligned                                201811L <memory>
 __cpp_lib_atomic_flag_test                              201907L <atomic>
 __cpp_lib_atomic_float                                  201711L <atomic>
@@ -38,10 +40,13 @@ __cpp_lib_atomic_shared_ptr                             201711L <atomic>
 __cpp_lib_atomic_value_initialization                   201911L <atomic> <memory>
 __cpp_lib_atomic_wait                                   201907L <atomic>
 __cpp_lib_barrier                                       201907L <barrier>
-__cpp_lib_bind_back                                     202202L <functional>
-__cpp_lib_bind_front                                    201907L <functional>
+__cpp_lib_bind_back                                     202306L <functional>
+                                                        202202L // C++23
+__cpp_lib_bind_front                                    202306L <functional>
+                                                        201907L // C++20
 __cpp_lib_bit_cast                                      201806L <bit>
 __cpp_lib_bitops                                        201907L <bit>
+__cpp_lib_bitset                                        202306L <bitset>
 __cpp_lib_bool_constant                                 201505L <type_traits>
 __cpp_lib_bounded_array_traits                          201902L <type_traits>
 __cpp_lib_boyer_moore_searcher                          201603L <functional>
@@ -72,6 +77,7 @@ __cpp_lib_constexpr_tuple                               201811L <tuple>
 __cpp_lib_constexpr_typeinfo                            202106L <typeinfo>
 __cpp_lib_constexpr_utility                             201811L <utility>
 __cpp_lib_constexpr_vector                              201907L <vector>
+__cpp_lib_copyable_function                             202306L <functional>
 __cpp_lib_coroutine                                     201902L <coroutine>
 __cpp_lib_destroying_delete                             201806L <new>
 __cpp_lib_enable_shared_from_this                       201603L <memory>
@@ -88,11 +94,14 @@ __cpp_lib_format                                        202106L <format>
 __cpp_lib_format_ranges                                 202207L <format>
 __cpp_lib_formatters                                    202302L <stacktrace> <thread>
 __cpp_lib_forward_like                                  202207L <utility>
+__cpp_lib_fstream_native_handle                         202306L <fstream>
+__cpp_lib_function_ref                                  202306L <functional>
 __cpp_lib_gcd_lcm                                       201606L <numeric>
 __cpp_lib_generic_associative_lookup                    201304L <map> <set>
 __cpp_lib_generic_unordered_lookup                      201811L <unordered_map> <unordered_set>
 __cpp_lib_hardware_interference_size                    201703L <new>
 __cpp_lib_has_unique_object_representations             201606L <type_traits>
+__cpp_lib_hazard_pointer                                202306L <hazard_pointer>
 __cpp_lib_hypot                                         201603L <cmath>
 __cpp_lib_incomplete_container_elements                 201505L <forward_list> <list> <vector>
 __cpp_lib_int_pow2                                      202002L <bit>
@@ -155,7 +164,9 @@ __cpp_lib_ranges_to_container                           202202L <deque> <forward
                                                                 <set> <stack> <string>
                                                                 <unordered_map> <unordered_set> <vector>
 __cpp_lib_ranges_zip                                    202110L <ranges> <tuple> <utility>
+__cpp_lib_ratio                                         202306L <ratio>
 __cpp_lib_raw_memory_algorithms                         201606L <memory>
+__cpp_lib_rcu                                           202306L <rcu>
 __cpp_lib_reference_from_temporary                      202202L <type_traits>
 __cpp_lib_remove_cvref                                  201711L <type_traits>
 __cpp_lib_result_of_sfinae                              201210L <functional> <type_traits>
@@ -170,10 +181,12 @@ __cpp_lib_shared_ptr_weak_type                          201606L <memory>
 __cpp_lib_shared_timed_mutex                            201402L <shared_mutex>
 __cpp_lib_shift                                         201806L <algorithm>
 __cpp_lib_smart_ptr_for_overwrite                       202002L <memory>
+__cpp_lib_smart_ptr_owner_equality                      202306L <memory>
 __cpp_lib_source_location                               201907L <source_location>
 __cpp_lib_span                                          202002L <span>
 __cpp_lib_spanstream                                    202106L <spanstream>
 __cpp_lib_ssize                                         201902L <iterator>
+__cpp_lib_sstream_from_string_view                      202306L <sstream>
 __cpp_lib_stacktrace                                    202011L <stacktrace>
 __cpp_lib_starts_ends_with                              201711L <string> <string_view>
 __cpp_lib_stdatomic_h                                   202011L <stdatomic.h>
@@ -182,11 +195,14 @@ __cpp_lib_string_resize_and_overwrite                   202110L <string>
 __cpp_lib_string_udls                                   201304L <string>
 __cpp_lib_string_view                                   201803L <string> <string_view>
                                                         201606L // C++17
+__cpp_lib_submdspan                                     202306L <mdspan>
 __cpp_lib_syncbuf                                       201803L <syncstream>
+__cpp_lib_text_encoding                                 202306L <text_encoding>
 __cpp_lib_three_way_comparison                          201907L <compare>
 __cpp_lib_to_address                                    201711L <memory>
 __cpp_lib_to_array                                      201907L <array>
 __cpp_lib_to_chars                                      201611L <charconv>
+__cpp_lib_to_string                                     202306L <string>
 __cpp_lib_to_underlying                                 202102L <utility>
 __cpp_lib_transformation_trait_aliases                  201304L <type_traits>
 __cpp_lib_transparent_operators                         201510L <functional> <memory>
@@ -201,6 +217,7 @@ __cpp_lib_unreachable                                   202202L <utility>
 __cpp_lib_unwrap_ref                                    201811L <functional>
 __cpp_lib_variant                                       202102L <variant>
 __cpp_lib_void_t                                        201411L <type_traits>
+__cpp_lib_within_lifetime                               202306L <type_traits>
 
 */
 
@@ -423,12 +440,29 @@ __cpp_lib_void_t                                        201411L <type_traits>
 # define __cpp_lib_stdatomic_h                          202011L
 # define __cpp_lib_string_contains                      202011L
 # define __cpp_lib_string_resize_and_overwrite          202110L
+// # define __cpp_lib_to_string                            202306L
 # define __cpp_lib_to_underlying                        202102L
 # define __cpp_lib_unreachable                          202202L
 #endif
 
 #if _LIBCPP_STD_VER >= 26
-
+// # define __cpp_lib_associative_heterogeneous_insertion  202306L
+# undef  __cpp_lib_bind_back
+// # define __cpp_lib_bind_back                            202306L
+# undef  __cpp_lib_bind_front
+# define __cpp_lib_bind_front                           202306L
+// # define __cpp_lib_bitset                               202306L
+// # define __cpp_lib_copyable_function                    202306L
+// # define __cpp_lib_fstream_native_handle                202306L
+// # define __cpp_lib_function_ref                         202306L
+// # define __cpp_lib_hazard_pointer                       202306L
+// # define __cpp_lib_ratio                                202306L
+// # define __cpp_lib_rcu                                  202306L
+// # define __cpp_lib_smart_ptr_owner_equality             202306L
+// # define __cpp_lib_sstream_from_string_view             202306L
+// # define __cpp_lib_submdspan                            202306L
+// # define __cpp_lib_text_encoding                        202306L
+// # define __cpp_lib_within_lifetime                      202306L
 #endif
 
 // clang-format on
diff --git a/libcxx/modules/CMakeLists.txt b/libcxx/modules/CMakeLists.txt
index 0070780a65dd1..d24ddb0efc7fa 100644
--- a/libcxx/modules/CMakeLists.txt
+++ b/libcxx/modules/CMakeLists.txt
@@ -56,6 +56,7 @@ set(LIBCXX_SOURCES_MODULE_STD
   std/functional.cppm
   std/future.cppm
   std/generator.cppm
+  std/hazard_pointer.cppm
   std/initializer_list.cppm
   std/iomanip.cppm
   std/ios.cppm
@@ -83,6 +84,7 @@ set(LIBCXX_SOURCES_MODULE_STD
   std/random.cppm
   std/ranges.cppm
   std/ratio.cppm
+  std/rcu.cppm
   std/regex.cppm
   std/scoped_allocator.cppm
   std/semaphore.cppm
@@ -104,6 +106,7 @@ set(LIBCXX_SOURCES_MODULE_STD
   std/strstream.cppm
   std/syncstream.cppm
   std/system_error.cppm
+  std/text_encoding.cppm
   std/thread.cppm
   std/tuple.cppm
   std/type_traits.cppm
diff --git a/libcxx/modules/std.cppm b/libcxx/modules/std.cppm
index d01e167169e53..fcd7d55bbed0a 100644
--- a/libcxx/modules/std.cppm
+++ b/libcxx/modules/std.cppm
@@ -61,6 +61,7 @@ export import :fstream;
 export import :functional;
 export import :future;
 export import :generator;
+export import :hazard_pointer;
 export import :initializer_list;
 export import :iomanip;
 export import :ios;
@@ -86,6 +87,7 @@ export import :queue;
 export import :random;
 export import :ranges;
 export import :ratio;
+export import :rcu;
 export import :regex;
 export import :scoped_allocator;
 export import :semaphore;
@@ -106,6 +108,7 @@ export import :string_view;
 export import :strstream;
 export import :syncstream;
 export import :system_error;
+export import :text_encoding;
 export import :thread;
 export import :tuple;
 export import :type_traits;
diff --git a/libcxx/modules/std/hazard_pointer.cppm b/libcxx/modules/std/hazard_pointer.cppm
new file mode 100644
index 0000000000000..84b83f55b31fb
--- /dev/null
+++ b/libcxx/modules/std/hazard_pointer.cppm
@@ -0,0 +1,30 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+module;
+#if __has_include(<hazard_pointer>)
+#  error "include this header unconditionally and uncomment the exported symbols"
+#  include <hazard_pointer>
+#endif
+
+export module std:hazard_pointer;
+export namespace std {
+#if 0
+#  if _LIBCPP_STD_VER >= 23
+  // 4.1.3, class template hazard_pointer_obj_base
+  using std::hazard_pointer_obj_base;
+  // 4.1.4, class hazard_pointer
+  using std::hazard_pointer;
+  // 4.1.5, Construct non-empty hazard_pointer
+  using std::make_hazard_pointer;
+  // 4.1.6, Hazard pointer swap
+  using std::swap;
+#  endif // _LIBCPP_STD_VER >= 23
+#endif
+} // namespace std
diff --git a/libcxx/modules/std/rcu.cppm b/libcxx/modules/std/rcu.cppm
new file mode 100644
index 0000000000000..fb5e5e1a5066a
--- /dev/null
+++ b/libcxx/modules/std/rcu.cppm
@@ -0,0 +1,29 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+module;
+#if __has_include(<rcu>)
+#  error "include this header unconditionally and uncomment the exported symbols"
+#  include <rcu>
+#endif
+
+export module std:rcu;
+export namespace std {
+#if 0
+#  if _LIBCPP_STD_VER >= 23
+  // 2.2.3, class template rcu_obj_base using std::rcu_obj_base;
+  // 2.2.4, class rcu_domain
+  using std::rcu_domain;
+  using std::rcu_default_domain();
+  using std::rcu_barrier;
+  using std::rcu_retire;
+  using std::rcu_synchronize;
+#  endif // _LIBCPP_STD_VER >= 23
+#endif
+} // namespace std
diff --git a/libcxx/modules/std/text_encoding.cppm b/libcxx/modules/std/text_encoding.cppm
new file mode 100644
index 0000000000000..dcbaa734a6856
--- /dev/null
+++ b/libcxx/modules/std/text_encoding.cppm
@@ -0,0 +1,26 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+module;
+#if __has_include(<text_encoding>)
+#  error "include this header unconditionally and uncomment the exported symbols"
+#  include <text_encoding>
+#endif
+
+export module std:text_encoding;
+export namespace std {
+#if 0
+#  if _LIBCPP_STD_VER >= 23
+  using std::text_encoding;
+
+  // hash support
+  using std::hash;
+#  endif // _LIBCPP_STD_VER >= 23
+#endif
+} // namespace std
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/bitset.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/bitset.version.compile.pass.cpp
index bd1f5feddac0d..e850479d99e1c 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/bitset.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/bitset.version.compile.pass.cpp
@@ -16,6 +16,7 @@
 // Test the feature test macros defined by <bitset>
 
 /*  Constant                      Value
+    __cpp_lib_bitset              202306L [C++26]
     __cpp_lib_constexpr_bitset    202207L [C++23]
 */
 
@@ -24,30 +25,50 @@
 
 #if TEST_STD_VER < 14
 
+# ifdef __cpp_lib_bitset
+#   error "__cpp_lib_bitset should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_constexpr_bitset
 #   error "__cpp_lib_constexpr_bitset should not be defined before c++23"
 # endif
 
 #elif TEST_STD_VER == 14
 
+# ifdef __cpp_lib_bitset
+#   error "__cpp_lib_bitset should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_constexpr_bitset
 #   error "__cpp_lib_constexpr_bitset should not be defined before c++23"
 # endif
 
 #elif TEST_STD_VER == 17
 
+# ifdef __cpp_lib_bitset
+#   error "__cpp_lib_bitset should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_constexpr_bitset
 #   error "__cpp_lib_constexpr_bitset should not be defined before c++23"
 # endif
 
 #elif TEST_STD_VER == 20
 
+# ifdef __cpp_lib_bitset
+#   error "__cpp_lib_bitset should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_constexpr_bitset
 #   error "__cpp_lib_constexpr_bitset should not be defined before c++23"
 # endif
 
 #elif TEST_STD_VER == 23
 
+# ifdef __cpp_lib_bitset
+#   error "__cpp_lib_bitset should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_constexpr_bitset
 #   error "__cpp_lib_constexpr_bitset should be defined in c++23"
 # endif
@@ -57,6 +78,19 @@
 
 #elif TEST_STD_VER > 23
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_bitset
+#     error "__cpp_lib_bitset should be defined in c++26"
+#   endif
+#   if __cpp_lib_bitset != 202306L
+#     error "__cpp_lib_bitset should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_bitset
+#     error "__cpp_lib_bitset should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 # ifndef __cpp_lib_constexpr_bitset
 #   error "__cpp_lib_constexpr_bitset should be defined in c++26"
 # endif
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/fstream.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/fstream.version.compile.pass.cpp
new file mode 100644
index 0000000000000..981f5420ff7e0
--- /dev/null
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/fstream.version.compile.pass.cpp
@@ -0,0 +1,73 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// WARNING: This test was generated by generate_feature_test_macro_components.py
+// and should not be edited manually.
+//
+// clang-format off
+
+// UNSUPPORTED: no-localization
+
+// <fstream>
+
+// Test the feature test macros defined by <fstream>
+
+/*  Constant                           Value
+    __cpp_lib_fstream_native_handle    202306L [C++26]
+*/
+
+#include <fstream>
+#include "test_macros.h"
+
+#if TEST_STD_VER < 14
+
+# ifdef __cpp_lib_fstream_native_handle
+#   error "__cpp_lib_fstream_native_handle should not be defined before c++26"
+# endif
+
+#elif TEST_STD_VER == 14
+
+# ifdef __cpp_lib_fstream_native_handle
+#   error "__cpp_lib_fstream_native_handle should not be defined before c++26"
+# endif
+
+#elif TEST_STD_VER == 17
+
+# ifdef __cpp_lib_fstream_native_handle
+#   error "__cpp_lib_fstream_native_handle should not be defined before c++26"
+# endif
+
+#elif TEST_STD_VER == 20
+
+# ifdef __cpp_lib_fstream_native_handle
+#   error "__cpp_lib_fstream_native_handle should not be defined before c++26"
+# endif
+
+#elif TEST_STD_VER == 23
+
+# ifdef __cpp_lib_fstream_native_handle
+#   error "__cpp_lib_fstream_native_handle should not be defined before c++26"
+# endif
+
+#elif TEST_STD_VER > 23
+
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_fstream_native_handle
+#     error "__cpp_lib_fstream_native_handle should be defined in c++26"
+#   endif
+#   if __cpp_lib_fstream_native_handle != 202306L
+#     error "__cpp_lib_fstream_native_handle should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_fstream_native_handle
+#     error "__cpp_lib_fstream_native_handle should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
+#endif // TEST_STD_VER > 23
+
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/functional.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/functional.version.compile.pass.cpp
index 79cb615c518f3..f14b127e51951 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/functional.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/functional.version.compile.pass.cpp
@@ -17,9 +17,13 @@
 
 /*  Constant                           Value
     __cpp_lib_bind_back                202202L [C++23]
+                                       202306L [C++26]
     __cpp_lib_bind_front               201907L [C++20]
+                                       202306L [C++26]
     __cpp_lib_boyer_moore_searcher     201603L [C++17]
     __cpp_lib_constexpr_functional     201907L [C++20]
+    __cpp_lib_copyable_function        202306L [C++26]
+    __cpp_lib_function_ref             202306L [C++26]
     __cpp_lib_invoke                   201411L [C++17]
     __cpp_lib_invoke_r                 202106L [C++23]
     __cpp_lib_move_only_function       202110L [C++23]
@@ -52,6 +56,14 @@
 #   error "__cpp_lib_constexpr_functional should not be defined before c++20"
 # endif
 
+# ifdef __cpp_lib_copyable_function
+#   error "__cpp_lib_copyable_function should not be defined before c++26"
+# endif
+
+# ifdef __cpp_lib_function_ref
+#   error "__cpp_lib_function_ref should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_invoke
 #   error "__cpp_lib_invoke should not be defined before c++17"
 # endif
@@ -102,6 +114,14 @@
 #   error "__cpp_lib_constexpr_functional should not be defined before c++20"
 # endif
 
+# ifdef __cpp_lib_copyable_function
+#   error "__cpp_lib_copyable_function should not be defined before c++26"
+# endif
+
+# ifdef __cpp_lib_function_ref
+#   error "__cpp_lib_function_ref should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_invoke
 #   error "__cpp_lib_invoke should not be defined before c++17"
 # endif
@@ -161,6 +181,14 @@
 #   error "__cpp_lib_constexpr_functional should not be defined before c++20"
 # endif
 
+# ifdef __cpp_lib_copyable_function
+#   error "__cpp_lib_copyable_function should not be defined before c++26"
+# endif
+
+# ifdef __cpp_lib_function_ref
+#   error "__cpp_lib_function_ref should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_invoke
 #   error "__cpp_lib_invoke should be defined in c++17"
 # endif
@@ -232,6 +260,14 @@
 #   error "__cpp_lib_constexpr_functional should have the value 201907L in c++20"
 # endif
 
+# ifdef __cpp_lib_copyable_function
+#   error "__cpp_lib_copyable_function should not be defined before c++26"
+# endif
+
+# ifdef __cpp_lib_function_ref
+#   error "__cpp_lib_function_ref should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_invoke
 #   error "__cpp_lib_invoke should be defined in c++20"
 # endif
@@ -318,6 +354,14 @@
 #   error "__cpp_lib_constexpr_functional should have the value 201907L in c++23"
 # endif
 
+# ifdef __cpp_lib_copyable_function
+#   error "__cpp_lib_copyable_function should not be defined before c++26"
+# endif
+
+# ifdef __cpp_lib_function_ref
+#   error "__cpp_lib_function_ref should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_invoke
 #   error "__cpp_lib_invoke should be defined in c++23"
 # endif
@@ -386,8 +430,8 @@
 #   ifndef __cpp_lib_bind_back
 #     error "__cpp_lib_bind_back should be defined in c++26"
 #   endif
-#   if __cpp_lib_bind_back != 202202L
-#     error "__cpp_lib_bind_back should have the value 202202L in c++26"
+#   if __cpp_lib_bind_back != 202306L
+#     error "__cpp_lib_bind_back should have the value 202306L in c++26"
 #   endif
 # else // _LIBCPP_VERSION
 #   ifdef __cpp_lib_bind_back
@@ -398,8 +442,8 @@
 # ifndef __cpp_lib_bind_front
 #   error "__cpp_lib_bind_front should be defined in c++26"
 # endif
-# if __cpp_lib_bind_front != 201907L
-#   error "__cpp_lib_bind_front should have the value 201907L in c++26"
+# if __cpp_lib_bind_front != 202306L
+#   error "__cpp_lib_bind_front should have the value 202306L in c++26"
 # endif
 
 # ifndef __cpp_lib_boyer_moore_searcher
@@ -416,6 +460,32 @@
 #   error "__cpp_lib_constexpr_functional should have the value 201907L in c++26"
 # endif
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_copyable_function
+#     error "__cpp_lib_copyable_function should be defined in c++26"
+#   endif
+#   if __cpp_lib_copyable_function != 202306L
+#     error "__cpp_lib_copyable_function should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_copyable_function
+#     error "__cpp_lib_copyable_function should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_function_ref
+#     error "__cpp_lib_function_ref should be defined in c++26"
+#   endif
+#   if __cpp_lib_function_ref != 202306L
+#     error "__cpp_lib_function_ref should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_function_ref
+#     error "__cpp_lib_function_ref should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 # ifndef __cpp_lib_invoke
 #   error "__cpp_lib_invoke should be defined in c++26"
 # endif
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/map.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/map.version.compile.pass.cpp
index 28d10cd3e7f1f..8599c54e24923 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/map.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/map.version.compile.pass.cpp
@@ -15,15 +15,16 @@
 
 // Test the feature test macros defined by <map>
 
-/*  Constant                                       Value
-    __cpp_lib_allocator_traits_is_always_equal     201411L [C++17]
-    __cpp_lib_associative_heterogeneous_erasure    202110L [C++23]
-    __cpp_lib_erase_if                             202002L [C++20]
-    __cpp_lib_generic_associative_lookup           201304L [C++14]
-    __cpp_lib_map_try_emplace                      201411L [C++17]
-    __cpp_lib_node_extract                         201606L [C++17]
-    __cpp_lib_nonmember_container_access           201411L [C++17]
-    __cpp_lib_ranges_to_container                  202202L [C++23]
+/*  Constant                                         Value
+    __cpp_lib_allocator_traits_is_always_equal       201411L [C++17]
+    __cpp_lib_associative_heterogeneous_erasure      202110L [C++23]
+    __cpp_lib_associative_heterogeneous_insertion    202306L [C++26]
+    __cpp_lib_erase_if                               202002L [C++20]
+    __cpp_lib_generic_associative_lookup             201304L [C++14]
+    __cpp_lib_map_try_emplace                        201411L [C++17]
+    __cpp_lib_node_extract                           201606L [C++17]
+    __cpp_lib_nonmember_container_access             201411L [C++17]
+    __cpp_lib_ranges_to_container                    202202L [C++23]
 */
 
 #include <map>
@@ -39,6 +40,10 @@
 #   error "__cpp_lib_associative_heterogeneous_erasure should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_associative_heterogeneous_insertion
+#   error "__cpp_lib_associative_heterogeneous_insertion should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_erase_if
 #   error "__cpp_lib_erase_if should not be defined before c++20"
 # endif
@@ -73,6 +78,10 @@
 #   error "__cpp_lib_associative_heterogeneous_erasure should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_associative_heterogeneous_insertion
+#   error "__cpp_lib_associative_heterogeneous_insertion should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_erase_if
 #   error "__cpp_lib_erase_if should not be defined before c++20"
 # endif
@@ -113,6 +122,10 @@
 #   error "__cpp_lib_associative_heterogeneous_erasure should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_associative_heterogeneous_insertion
+#   error "__cpp_lib_associative_heterogeneous_insertion should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_erase_if
 #   error "__cpp_lib_erase_if should not be defined before c++20"
 # endif
@@ -162,6 +175,10 @@
 #   error "__cpp_lib_associative_heterogeneous_erasure should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_associative_heterogeneous_insertion
+#   error "__cpp_lib_associative_heterogeneous_insertion should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_erase_if
 #   error "__cpp_lib_erase_if should be defined in c++20"
 # endif
@@ -223,6 +240,10 @@
 #   endif
 # endif
 
+# ifdef __cpp_lib_associative_heterogeneous_insertion
+#   error "__cpp_lib_associative_heterogeneous_insertion should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_erase_if
 #   error "__cpp_lib_erase_if should be defined in c++23"
 # endif
@@ -293,6 +314,19 @@
 #   endif
 # endif
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_associative_heterogeneous_insertion
+#     error "__cpp_lib_associative_heterogeneous_insertion should be defined in c++26"
+#   endif
+#   if __cpp_lib_associative_heterogeneous_insertion != 202306L
+#     error "__cpp_lib_associative_heterogeneous_insertion should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_associative_heterogeneous_insertion
+#     error "__cpp_lib_associative_heterogeneous_insertion should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 # ifndef __cpp_lib_erase_if
 #   error "__cpp_lib_erase_if should be defined in c++26"
 # endif
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/mdspan.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/mdspan.version.compile.pass.cpp
index c2da617f42cc1..e47d678b254ef 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/mdspan.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/mdspan.version.compile.pass.cpp
@@ -15,8 +15,9 @@
 
 // Test the feature test macros defined by <mdspan>
 
-/*  Constant            Value
-    __cpp_lib_mdspan    202207L [C++23]
+/*  Constant               Value
+    __cpp_lib_mdspan       202207L [C++23]
+    __cpp_lib_submdspan    202306L [C++26]
 */
 
 #include <mdspan>
@@ -28,24 +29,40 @@
 #   error "__cpp_lib_mdspan should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_submdspan
+#   error "__cpp_lib_submdspan should not be defined before c++26"
+# endif
+
 #elif TEST_STD_VER == 14
 
 # ifdef __cpp_lib_mdspan
 #   error "__cpp_lib_mdspan should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_submdspan
+#   error "__cpp_lib_submdspan should not be defined before c++26"
+# endif
+
 #elif TEST_STD_VER == 17
 
 # ifdef __cpp_lib_mdspan
 #   error "__cpp_lib_mdspan should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_submdspan
+#   error "__cpp_lib_submdspan should not be defined before c++26"
+# endif
+
 #elif TEST_STD_VER == 20
 
 # ifdef __cpp_lib_mdspan
 #   error "__cpp_lib_mdspan should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_submdspan
+#   error "__cpp_lib_submdspan should not be defined before c++26"
+# endif
+
 #elif TEST_STD_VER == 23
 
 # if !defined(_LIBCPP_VERSION)
@@ -61,6 +78,10 @@
 #   endif
 # endif
 
+# ifdef __cpp_lib_submdspan
+#   error "__cpp_lib_submdspan should not be defined before c++26"
+# endif
+
 #elif TEST_STD_VER > 23
 
 # if !defined(_LIBCPP_VERSION)
@@ -76,5 +97,18 @@
 #   endif
 # endif
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_submdspan
+#     error "__cpp_lib_submdspan should be defined in c++26"
+#   endif
+#   if __cpp_lib_submdspan != 202306L
+#     error "__cpp_lib_submdspan should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_submdspan
+#     error "__cpp_lib_submdspan should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 #endif // TEST_STD_VER > 23
 
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/memory.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/memory.version.compile.pass.cpp
index 7db7c956d3751..02559d1340531 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/memory.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/memory.version.compile.pass.cpp
@@ -33,6 +33,7 @@
                                                   201707L [C++20]
     __cpp_lib_shared_ptr_weak_type                201606L [C++17]
     __cpp_lib_smart_ptr_for_overwrite             202002L [C++20]
+    __cpp_lib_smart_ptr_owner_equality            202306L [C++26]
     __cpp_lib_to_address                          201711L [C++20]
     __cpp_lib_transparent_operators               201210L [C++14]
                                                   201510L [C++17]
@@ -103,6 +104,10 @@
 #   error "__cpp_lib_smart_ptr_for_overwrite should not be defined before c++20"
 # endif
 
+# ifdef __cpp_lib_smart_ptr_owner_equality
+#   error "__cpp_lib_smart_ptr_owner_equality should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_to_address
 #   error "__cpp_lib_to_address should not be defined before c++20"
 # endif
@@ -176,6 +181,10 @@
 #   error "__cpp_lib_smart_ptr_for_overwrite should not be defined before c++20"
 # endif
 
+# ifdef __cpp_lib_smart_ptr_owner_equality
+#   error "__cpp_lib_smart_ptr_owner_equality should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_to_address
 #   error "__cpp_lib_to_address should not be defined before c++20"
 # endif
@@ -270,6 +279,10 @@
 #   error "__cpp_lib_smart_ptr_for_overwrite should not be defined before c++20"
 # endif
 
+# ifdef __cpp_lib_smart_ptr_owner_equality
+#   error "__cpp_lib_smart_ptr_owner_equality should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_to_address
 #   error "__cpp_lib_to_address should not be defined before c++20"
 # endif
@@ -388,6 +401,10 @@
 #   endif
 # endif
 
+# ifdef __cpp_lib_smart_ptr_owner_equality
+#   error "__cpp_lib_smart_ptr_owner_equality should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_to_address
 #   error "__cpp_lib_to_address should be defined in c++20"
 # endif
@@ -521,6 +538,10 @@
 #   endif
 # endif
 
+# ifdef __cpp_lib_smart_ptr_owner_equality
+#   error "__cpp_lib_smart_ptr_owner_equality should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_to_address
 #   error "__cpp_lib_to_address should be defined in c++23"
 # endif
@@ -654,6 +675,19 @@
 #   endif
 # endif
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_smart_ptr_owner_equality
+#     error "__cpp_lib_smart_ptr_owner_equality should be defined in c++26"
+#   endif
+#   if __cpp_lib_smart_ptr_owner_equality != 202306L
+#     error "__cpp_lib_smart_ptr_owner_equality should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_smart_ptr_owner_equality
+#     error "__cpp_lib_smart_ptr_owner_equality should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 # ifndef __cpp_lib_to_address
 #   error "__cpp_lib_to_address should be defined in c++26"
 # endif
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/ratio.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/ratio.version.compile.pass.cpp
new file mode 100644
index 0000000000000..f782c24898b43
--- /dev/null
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/ratio.version.compile.pass.cpp
@@ -0,0 +1,71 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// WARNING: This test was generated by generate_feature_test_macro_components.py
+// and should not be edited manually.
+//
+// clang-format off
+
+// <ratio>
+
+// Test the feature test macros defined by <ratio>
+
+/*  Constant           Value
+    __cpp_lib_ratio    202306L [C++26]
+*/
+
+#include <ratio>
+#include "test_macros.h"
+
+#if TEST_STD_VER < 14
+
+# ifdef __cpp_lib_ratio
+#   error "__cpp_lib_ratio should not be defined before c++26"
+# endif
+
+#elif TEST_STD_VER == 14
+
+# ifdef __cpp_lib_ratio
+#   error "__cpp_lib_ratio should not be defined before c++26"
+# endif
+
+#elif TEST_STD_VER == 17
+
+# ifdef __cpp_lib_ratio
+#   error "__cpp_lib_ratio should not be defined before c++26"
+# endif
+
+#elif TEST_STD_VER == 20
+
+# ifdef __cpp_lib_ratio
+#   error "__cpp_lib_ratio should not be defined before c++26"
+# endif
+
+#elif TEST_STD_VER == 23
+
+# ifdef __cpp_lib_ratio
+#   error "__cpp_lib_ratio should not be defined before c++26"
+# endif
+
+#elif TEST_STD_VER > 23
+
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_ratio
+#     error "__cpp_lib_ratio should be defined in c++26"
+#   endif
+#   if __cpp_lib_ratio != 202306L
+#     error "__cpp_lib_ratio should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_ratio
+#     error "__cpp_lib_ratio should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
+#endif // TEST_STD_VER > 23
+
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/set.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/set.version.compile.pass.cpp
index 23f93805cc169..241b66b2bfc31 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/set.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/set.version.compile.pass.cpp
@@ -15,14 +15,15 @@
 
 // Test the feature test macros defined by <set>
 
-/*  Constant                                       Value
-    __cpp_lib_allocator_traits_is_always_equal     201411L [C++17]
-    __cpp_lib_associative_heterogeneous_erasure    202110L [C++23]
-    __cpp_lib_erase_if                             202002L [C++20]
-    __cpp_lib_generic_associative_lookup           201304L [C++14]
-    __cpp_lib_node_extract                         201606L [C++17]
-    __cpp_lib_nonmember_container_access           201411L [C++17]
-    __cpp_lib_ranges_to_container                  202202L [C++23]
+/*  Constant                                         Value
+    __cpp_lib_allocator_traits_is_always_equal       201411L [C++17]
+    __cpp_lib_associative_heterogeneous_erasure      202110L [C++23]
+    __cpp_lib_associative_heterogeneous_insertion    202306L [C++26]
+    __cpp_lib_erase_if                               202002L [C++20]
+    __cpp_lib_generic_associative_lookup             201304L [C++14]
+    __cpp_lib_node_extract                           201606L [C++17]
+    __cpp_lib_nonmember_container_access             201411L [C++17]
+    __cpp_lib_ranges_to_container                    202202L [C++23]
 */
 
 #include <set>
@@ -38,6 +39,10 @@
 #   error "__cpp_lib_associative_heterogeneous_erasure should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_associative_heterogeneous_insertion
+#   error "__cpp_lib_associative_heterogeneous_insertion should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_erase_if
 #   error "__cpp_lib_erase_if should not be defined before c++20"
 # endif
@@ -68,6 +73,10 @@
 #   error "__cpp_lib_associative_heterogeneous_erasure should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_associative_heterogeneous_insertion
+#   error "__cpp_lib_associative_heterogeneous_insertion should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_erase_if
 #   error "__cpp_lib_erase_if should not be defined before c++20"
 # endif
@@ -104,6 +113,10 @@
 #   error "__cpp_lib_associative_heterogeneous_erasure should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_associative_heterogeneous_insertion
+#   error "__cpp_lib_associative_heterogeneous_insertion should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_erase_if
 #   error "__cpp_lib_erase_if should not be defined before c++20"
 # endif
@@ -146,6 +159,10 @@
 #   error "__cpp_lib_associative_heterogeneous_erasure should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_associative_heterogeneous_insertion
+#   error "__cpp_lib_associative_heterogeneous_insertion should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_erase_if
 #   error "__cpp_lib_erase_if should be defined in c++20"
 # endif
@@ -200,6 +217,10 @@
 #   endif
 # endif
 
+# ifdef __cpp_lib_associative_heterogeneous_insertion
+#   error "__cpp_lib_associative_heterogeneous_insertion should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_erase_if
 #   error "__cpp_lib_erase_if should be defined in c++23"
 # endif
@@ -263,6 +284,19 @@
 #   endif
 # endif
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_associative_heterogeneous_insertion
+#     error "__cpp_lib_associative_heterogeneous_insertion should be defined in c++26"
+#   endif
+#   if __cpp_lib_associative_heterogeneous_insertion != 202306L
+#     error "__cpp_lib_associative_heterogeneous_insertion should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_associative_heterogeneous_insertion
+#     error "__cpp_lib_associative_heterogeneous_insertion should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 # ifndef __cpp_lib_erase_if
 #   error "__cpp_lib_erase_if should be defined in c++26"
 # endif
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/sstream.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/sstream.version.compile.pass.cpp
new file mode 100644
index 0000000000000..78e17e5f1ddb5
--- /dev/null
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/sstream.version.compile.pass.cpp
@@ -0,0 +1,73 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// WARNING: This test was generated by generate_feature_test_macro_components.py
+// and should not be edited manually.
+//
+// clang-format off
+
+// UNSUPPORTED: no-localization
+
+// <sstream>
+
+// Test the feature test macros defined by <sstream>
+
+/*  Constant                              Value
+    __cpp_lib_sstream_from_string_view    202306L [C++26]
+*/
+
+#include <sstream>
+#include "test_macros.h"
+
+#if TEST_STD_VER < 14
+
+# ifdef __cpp_lib_sstream_from_string_view
+#   error "__cpp_lib_sstream_from_string_view should not be defined before c++26"
+# endif
+
+#elif TEST_STD_VER == 14
+
+# ifdef __cpp_lib_sstream_from_string_view
+#   error "__cpp_lib_sstream_from_string_view should not be defined before c++26"
+# endif
+
+#elif TEST_STD_VER == 17
+
+# ifdef __cpp_lib_sstream_from_string_view
+#   error "__cpp_lib_sstream_from_string_view should not be defined before c++26"
+# endif
+
+#elif TEST_STD_VER == 20
+
+# ifdef __cpp_lib_sstream_from_string_view
+#   error "__cpp_lib_sstream_from_string_view should not be defined before c++26"
+# endif
+
+#elif TEST_STD_VER == 23
+
+# ifdef __cpp_lib_sstream_from_string_view
+#   error "__cpp_lib_sstream_from_string_view should not be defined before c++26"
+# endif
+
+#elif TEST_STD_VER > 23
+
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_sstream_from_string_view
+#     error "__cpp_lib_sstream_from_string_view should be defined in c++26"
+#   endif
+#   if __cpp_lib_sstream_from_string_view != 202306L
+#     error "__cpp_lib_sstream_from_string_view should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_sstream_from_string_view
+#     error "__cpp_lib_sstream_from_string_view should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
+#endif // TEST_STD_VER > 23
+
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/string.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/string.version.compile.pass.cpp
index e869fedbdaaf2..5902bdbd15521 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/string.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/string.version.compile.pass.cpp
@@ -28,6 +28,7 @@
     __cpp_lib_string_udls                         201304L [C++14]
     __cpp_lib_string_view                         201606L [C++17]
                                                   201803L [C++20]
+    __cpp_lib_to_string                           202306L [C++23]
 */
 
 #include <string>
@@ -79,6 +80,10 @@
 #   error "__cpp_lib_string_view should not be defined before c++17"
 # endif
 
+# ifdef __cpp_lib_to_string
+#   error "__cpp_lib_to_string should not be defined before c++23"
+# endif
+
 #elif TEST_STD_VER == 14
 
 # ifdef __cpp_lib_allocator_traits_is_always_equal
@@ -128,6 +133,10 @@
 #   error "__cpp_lib_string_view should not be defined before c++17"
 # endif
 
+# ifdef __cpp_lib_to_string
+#   error "__cpp_lib_to_string should not be defined before c++23"
+# endif
+
 #elif TEST_STD_VER == 17
 
 # ifndef __cpp_lib_allocator_traits_is_always_equal
@@ -186,6 +195,10 @@
 #   error "__cpp_lib_string_view should have the value 201606L in c++17"
 # endif
 
+# ifdef __cpp_lib_to_string
+#   error "__cpp_lib_to_string should not be defined before c++23"
+# endif
+
 #elif TEST_STD_VER == 20
 
 # ifndef __cpp_lib_allocator_traits_is_always_equal
@@ -262,6 +275,10 @@
 #   error "__cpp_lib_string_view should have the value 201803L in c++20"
 # endif
 
+# ifdef __cpp_lib_to_string
+#   error "__cpp_lib_to_string should not be defined before c++23"
+# endif
+
 #elif TEST_STD_VER == 23
 
 # ifndef __cpp_lib_allocator_traits_is_always_equal
@@ -353,6 +370,19 @@
 #   error "__cpp_lib_string_view should have the value 201803L in c++23"
 # endif
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_to_string
+#     error "__cpp_lib_to_string should be defined in c++23"
+#   endif
+#   if __cpp_lib_to_string != 202306L
+#     error "__cpp_lib_to_string should have the value 202306L in c++23"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_to_string
+#     error "__cpp_lib_to_string should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 #elif TEST_STD_VER > 23
 
 # ifndef __cpp_lib_allocator_traits_is_always_equal
@@ -444,5 +474,18 @@
 #   error "__cpp_lib_string_view should have the value 201803L in c++26"
 # endif
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_to_string
+#     error "__cpp_lib_to_string should be defined in c++26"
+#   endif
+#   if __cpp_lib_to_string != 202306L
+#     error "__cpp_lib_to_string should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_to_string
+#     error "__cpp_lib_to_string should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 #endif // TEST_STD_VER > 23
 
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/type_traits.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/type_traits.version.compile.pass.cpp
index a851b1bbb19d4..b1dd76b98720d 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/type_traits.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/type_traits.version.compile.pass.cpp
@@ -38,6 +38,7 @@
     __cpp_lib_type_identity                        201806L [C++20]
     __cpp_lib_type_trait_variable_templates        201510L [C++17]
     __cpp_lib_void_t                               201411L [C++17]
+    __cpp_lib_within_lifetime                      202306L [C++26]
 */
 
 #include <type_traits>
@@ -133,6 +134,10 @@
 #   error "__cpp_lib_void_t should not be defined before c++17"
 # endif
 
+# ifdef __cpp_lib_within_lifetime
+#   error "__cpp_lib_within_lifetime should not be defined before c++26"
+# endif
+
 #elif TEST_STD_VER == 14
 
 # ifdef __cpp_lib_bool_constant
@@ -238,6 +243,10 @@
 #   error "__cpp_lib_void_t should not be defined before c++17"
 # endif
 
+# ifdef __cpp_lib_within_lifetime
+#   error "__cpp_lib_within_lifetime should not be defined before c++26"
+# endif
+
 #elif TEST_STD_VER == 17
 
 # ifndef __cpp_lib_bool_constant
@@ -367,6 +376,10 @@
 #   error "__cpp_lib_void_t should have the value 201411L in c++17"
 # endif
 
+# ifdef __cpp_lib_within_lifetime
+#   error "__cpp_lib_within_lifetime should not be defined before c++26"
+# endif
+
 #elif TEST_STD_VER == 20
 
 # ifndef __cpp_lib_bool_constant
@@ -529,6 +542,10 @@
 #   error "__cpp_lib_void_t should have the value 201411L in c++20"
 # endif
 
+# ifdef __cpp_lib_within_lifetime
+#   error "__cpp_lib_within_lifetime should not be defined before c++26"
+# endif
+
 #elif TEST_STD_VER == 23
 
 # ifndef __cpp_lib_bool_constant
@@ -703,6 +720,10 @@
 #   error "__cpp_lib_void_t should have the value 201411L in c++23"
 # endif
 
+# ifdef __cpp_lib_within_lifetime
+#   error "__cpp_lib_within_lifetime should not be defined before c++26"
+# endif
+
 #elif TEST_STD_VER > 23
 
 # ifndef __cpp_lib_bool_constant
@@ -877,5 +898,18 @@
 #   error "__cpp_lib_void_t should have the value 201411L in c++26"
 # endif
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_within_lifetime
+#     error "__cpp_lib_within_lifetime should be defined in c++26"
+#   endif
+#   if __cpp_lib_within_lifetime != 202306L
+#     error "__cpp_lib_within_lifetime should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_within_lifetime
+#     error "__cpp_lib_within_lifetime should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 #endif // TEST_STD_VER > 23
 
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/unordered_map.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/unordered_map.version.compile.pass.cpp
index 0e5b5985df60e..c3dfb78cfadd1 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/unordered_map.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/unordered_map.version.compile.pass.cpp
@@ -15,15 +15,16 @@
 
 // Test the feature test macros defined by <unordered_map>
 
-/*  Constant                                       Value
-    __cpp_lib_allocator_traits_is_always_equal     201411L [C++17]
-    __cpp_lib_associative_heterogeneous_erasure    202110L [C++23]
-    __cpp_lib_erase_if                             202002L [C++20]
-    __cpp_lib_generic_unordered_lookup             201811L [C++20]
-    __cpp_lib_node_extract                         201606L [C++17]
-    __cpp_lib_nonmember_container_access           201411L [C++17]
-    __cpp_lib_ranges_to_container                  202202L [C++23]
-    __cpp_lib_unordered_map_try_emplace            201411L [C++17]
+/*  Constant                                         Value
+    __cpp_lib_allocator_traits_is_always_equal       201411L [C++17]
+    __cpp_lib_associative_heterogeneous_erasure      202110L [C++23]
+    __cpp_lib_associative_heterogeneous_insertion    202306L [C++26]
+    __cpp_lib_erase_if                               202002L [C++20]
+    __cpp_lib_generic_unordered_lookup               201811L [C++20]
+    __cpp_lib_node_extract                           201606L [C++17]
+    __cpp_lib_nonmember_container_access             201411L [C++17]
+    __cpp_lib_ranges_to_container                    202202L [C++23]
+    __cpp_lib_unordered_map_try_emplace              201411L [C++17]
 */
 
 #include <unordered_map>
@@ -39,6 +40,10 @@
 #   error "__cpp_lib_associative_heterogeneous_erasure should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_associative_heterogeneous_insertion
+#   error "__cpp_lib_associative_heterogeneous_insertion should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_erase_if
 #   error "__cpp_lib_erase_if should not be defined before c++20"
 # endif
@@ -73,6 +78,10 @@
 #   error "__cpp_lib_associative_heterogeneous_erasure should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_associative_heterogeneous_insertion
+#   error "__cpp_lib_associative_heterogeneous_insertion should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_erase_if
 #   error "__cpp_lib_erase_if should not be defined before c++20"
 # endif
@@ -110,6 +119,10 @@
 #   error "__cpp_lib_associative_heterogeneous_erasure should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_associative_heterogeneous_insertion
+#   error "__cpp_lib_associative_heterogeneous_insertion should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_erase_if
 #   error "__cpp_lib_erase_if should not be defined before c++20"
 # endif
@@ -156,6 +169,10 @@
 #   error "__cpp_lib_associative_heterogeneous_erasure should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_associative_heterogeneous_insertion
+#   error "__cpp_lib_associative_heterogeneous_insertion should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_erase_if
 #   error "__cpp_lib_erase_if should be defined in c++20"
 # endif
@@ -217,6 +234,10 @@
 #   endif
 # endif
 
+# ifdef __cpp_lib_associative_heterogeneous_insertion
+#   error "__cpp_lib_associative_heterogeneous_insertion should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_erase_if
 #   error "__cpp_lib_erase_if should be defined in c++23"
 # endif
@@ -287,6 +308,19 @@
 #   endif
 # endif
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_associative_heterogeneous_insertion
+#     error "__cpp_lib_associative_heterogeneous_insertion should be defined in c++26"
+#   endif
+#   if __cpp_lib_associative_heterogeneous_insertion != 202306L
+#     error "__cpp_lib_associative_heterogeneous_insertion should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_associative_heterogeneous_insertion
+#     error "__cpp_lib_associative_heterogeneous_insertion should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 # ifndef __cpp_lib_erase_if
 #   error "__cpp_lib_erase_if should be defined in c++26"
 # endif
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/unordered_set.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/unordered_set.version.compile.pass.cpp
index 8a3a8afd5b473..d215aec861564 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/unordered_set.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/unordered_set.version.compile.pass.cpp
@@ -15,14 +15,15 @@
 
 // Test the feature test macros defined by <unordered_set>
 
-/*  Constant                                       Value
-    __cpp_lib_allocator_traits_is_always_equal     201411L [C++17]
-    __cpp_lib_associative_heterogeneous_erasure    202110L [C++23]
-    __cpp_lib_erase_if                             202002L [C++20]
-    __cpp_lib_generic_unordered_lookup             201811L [C++20]
-    __cpp_lib_node_extract                         201606L [C++17]
-    __cpp_lib_nonmember_container_access           201411L [C++17]
-    __cpp_lib_ranges_to_container                  202202L [C++23]
+/*  Constant                                         Value
+    __cpp_lib_allocator_traits_is_always_equal       201411L [C++17]
+    __cpp_lib_associative_heterogeneous_erasure      202110L [C++23]
+    __cpp_lib_associative_heterogeneous_insertion    202306L [C++26]
+    __cpp_lib_erase_if                               202002L [C++20]
+    __cpp_lib_generic_unordered_lookup               201811L [C++20]
+    __cpp_lib_node_extract                           201606L [C++17]
+    __cpp_lib_nonmember_container_access             201411L [C++17]
+    __cpp_lib_ranges_to_container                    202202L [C++23]
 */
 
 #include <unordered_set>
@@ -38,6 +39,10 @@
 #   error "__cpp_lib_associative_heterogeneous_erasure should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_associative_heterogeneous_insertion
+#   error "__cpp_lib_associative_heterogeneous_insertion should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_erase_if
 #   error "__cpp_lib_erase_if should not be defined before c++20"
 # endif
@@ -68,6 +73,10 @@
 #   error "__cpp_lib_associative_heterogeneous_erasure should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_associative_heterogeneous_insertion
+#   error "__cpp_lib_associative_heterogeneous_insertion should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_erase_if
 #   error "__cpp_lib_erase_if should not be defined before c++20"
 # endif
@@ -101,6 +110,10 @@
 #   error "__cpp_lib_associative_heterogeneous_erasure should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_associative_heterogeneous_insertion
+#   error "__cpp_lib_associative_heterogeneous_insertion should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_erase_if
 #   error "__cpp_lib_erase_if should not be defined before c++20"
 # endif
@@ -140,6 +153,10 @@
 #   error "__cpp_lib_associative_heterogeneous_erasure should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_associative_heterogeneous_insertion
+#   error "__cpp_lib_associative_heterogeneous_insertion should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_erase_if
 #   error "__cpp_lib_erase_if should be defined in c++20"
 # endif
@@ -194,6 +211,10 @@
 #   endif
 # endif
 
+# ifdef __cpp_lib_associative_heterogeneous_insertion
+#   error "__cpp_lib_associative_heterogeneous_insertion should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_erase_if
 #   error "__cpp_lib_erase_if should be defined in c++23"
 # endif
@@ -257,6 +278,19 @@
 #   endif
 # endif
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_associative_heterogeneous_insertion
+#     error "__cpp_lib_associative_heterogeneous_insertion should be defined in c++26"
+#   endif
+#   if __cpp_lib_associative_heterogeneous_insertion != 202306L
+#     error "__cpp_lib_associative_heterogeneous_insertion should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_associative_heterogeneous_insertion
+#     error "__cpp_lib_associative_heterogeneous_insertion should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 # ifndef __cpp_lib_erase_if
 #   error "__cpp_lib_erase_if should be defined in c++26"
 # endif
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
index 8443fb1d97513..dbad0e6b3cf0c 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
@@ -15,178 +15,194 @@
 
 // Test the feature test macros defined by <version>
 
-/*  Constant                                       Value
-    __cpp_lib_adaptor_iterator_pair_constructor    202106L [C++23]
-    __cpp_lib_addressof_constexpr                  201603L [C++17]
-    __cpp_lib_allocate_at_least                    202106L [C++23]
-    __cpp_lib_allocator_traits_is_always_equal     201411L [C++17]
-    __cpp_lib_any                                  201606L [C++17]
-    __cpp_lib_apply                                201603L [C++17]
-    __cpp_lib_array_constexpr                      201603L [C++17]
-                                                   201811L [C++20]
-    __cpp_lib_as_const                             201510L [C++17]
-    __cpp_lib_associative_heterogeneous_erasure    202110L [C++23]
-    __cpp_lib_assume_aligned                       201811L [C++20]
-    __cpp_lib_atomic_flag_test                     201907L [C++20]
-    __cpp_lib_atomic_float                         201711L [C++20]
-    __cpp_lib_atomic_is_always_lock_free           201603L [C++17]
-    __cpp_lib_atomic_lock_free_type_aliases        201907L [C++20]
-    __cpp_lib_atomic_ref                           201806L [C++20]
-    __cpp_lib_atomic_shared_ptr                    201711L [C++20]
-    __cpp_lib_atomic_value_initialization          201911L [C++20]
-    __cpp_lib_atomic_wait                          201907L [C++20]
-    __cpp_lib_barrier                              201907L [C++20]
-    __cpp_lib_bind_back                            202202L [C++23]
-    __cpp_lib_bind_front                           201907L [C++20]
-    __cpp_lib_bit_cast                             201806L [C++20]
-    __cpp_lib_bitops                               201907L [C++20]
-    __cpp_lib_bool_constant                        201505L [C++17]
-    __cpp_lib_bounded_array_traits                 201902L [C++20]
-    __cpp_lib_boyer_moore_searcher                 201603L [C++17]
-    __cpp_lib_byte                                 201603L [C++17]
-    __cpp_lib_byteswap                             202110L [C++23]
-    __cpp_lib_char8_t                              201907L [C++20]
-    __cpp_lib_chrono                               201611L [C++17]
-    __cpp_lib_chrono_udls                          201304L [C++14]
-    __cpp_lib_clamp                                201603L [C++17]
-    __cpp_lib_complex_udls                         201309L [C++14]
-    __cpp_lib_concepts                             202002L [C++20]
-    __cpp_lib_constexpr_algorithms                 201806L [C++20]
-    __cpp_lib_constexpr_bitset                     202207L [C++23]
-    __cpp_lib_constexpr_charconv                   202207L [C++23]
-    __cpp_lib_constexpr_cmath                      202202L [C++23]
-    __cpp_lib_constexpr_complex                    201711L [C++20]
-    __cpp_lib_constexpr_dynamic_alloc              201907L [C++20]
-    __cpp_lib_constexpr_functional                 201907L [C++20]
-    __cpp_lib_constexpr_iterator                   201811L [C++20]
-    __cpp_lib_constexpr_memory                     201811L [C++20]
-                                                   202202L [C++23]
-    __cpp_lib_constexpr_numeric                    201911L [C++20]
-    __cpp_lib_constexpr_string                     201907L [C++20]
-    __cpp_lib_constexpr_string_view                201811L [C++20]
-    __cpp_lib_constexpr_tuple                      201811L [C++20]
-    __cpp_lib_constexpr_typeinfo                   202106L [C++23]
-    __cpp_lib_constexpr_utility                    201811L [C++20]
-    __cpp_lib_constexpr_vector                     201907L [C++20]
-    __cpp_lib_coroutine                            201902L [C++20]
-    __cpp_lib_destroying_delete                    201806L [C++20]
-    __cpp_lib_enable_shared_from_this              201603L [C++17]
-    __cpp_lib_endian                               201907L [C++20]
-    __cpp_lib_erase_if                             202002L [C++20]
-    __cpp_lib_exchange_function                    201304L [C++14]
-    __cpp_lib_execution                            201603L [C++17]
-                                                   201902L [C++20]
-    __cpp_lib_expected                             202211L [C++23]
-    __cpp_lib_filesystem                           201703L [C++17]
-    __cpp_lib_format                               202106L [C++20]
-    __cpp_lib_format_ranges                        202207L [C++23]
-    __cpp_lib_formatters                           202302L [C++23]
-    __cpp_lib_forward_like                         202207L [C++23]
-    __cpp_lib_gcd_lcm                              201606L [C++17]
-    __cpp_lib_generic_associative_lookup           201304L [C++14]
-    __cpp_lib_generic_unordered_lookup             201811L [C++20]
-    __cpp_lib_hardware_interference_size           201703L [C++17]
-    __cpp_lib_has_unique_object_representations    201606L [C++17]
-    __cpp_lib_hypot                                201603L [C++17]
-    __cpp_lib_incomplete_container_elements        201505L [C++17]
-    __cpp_lib_int_pow2                             202002L [C++20]
-    __cpp_lib_integer_comparison_functions         202002L [C++20]
-    __cpp_lib_integer_sequence                     201304L [C++14]
-    __cpp_lib_integral_constant_callable           201304L [C++14]
-    __cpp_lib_interpolate                          201902L [C++20]
-    __cpp_lib_invoke                               201411L [C++17]
-    __cpp_lib_invoke_r                             202106L [C++23]
-    __cpp_lib_is_aggregate                         201703L [C++17]
-    __cpp_lib_is_constant_evaluated                201811L [C++20]
-    __cpp_lib_is_final                             201402L [C++14]
-    __cpp_lib_is_invocable                         201703L [C++17]
-    __cpp_lib_is_layout_compatible                 201907L [C++20]
-    __cpp_lib_is_nothrow_convertible               201806L [C++20]
-    __cpp_lib_is_null_pointer                      201309L [C++14]
-    __cpp_lib_is_pointer_interconvertible          201907L [C++20]
-    __cpp_lib_is_scoped_enum                       202011L [C++23]
-    __cpp_lib_is_swappable                         201603L [C++17]
-    __cpp_lib_jthread                              201911L [C++20]
-    __cpp_lib_latch                                201907L [C++20]
-    __cpp_lib_launder                              201606L [C++17]
-    __cpp_lib_list_remove_return_type              201806L [C++20]
-    __cpp_lib_logical_traits                       201510L [C++17]
-    __cpp_lib_make_from_tuple                      201606L [C++17]
-    __cpp_lib_make_reverse_iterator                201402L [C++14]
-    __cpp_lib_make_unique                          201304L [C++14]
-    __cpp_lib_map_try_emplace                      201411L [C++17]
-    __cpp_lib_math_constants                       201907L [C++20]
-    __cpp_lib_math_special_functions               201603L [C++17]
-    __cpp_lib_mdspan                               202207L [C++23]
-    __cpp_lib_memory_resource                      201603L [C++17]
-    __cpp_lib_move_iterator_concept                202207L [C++20]
-    __cpp_lib_move_only_function                   202110L [C++23]
-    __cpp_lib_node_extract                         201606L [C++17]
-    __cpp_lib_nonmember_container_access           201411L [C++17]
-    __cpp_lib_not_fn                               201603L [C++17]
-    __cpp_lib_null_iterators                       201304L [C++14]
-    __cpp_lib_optional                             201606L [C++17]
-                                                   202110L [C++23]
-    __cpp_lib_out_ptr                              202106L [C++23]
-    __cpp_lib_parallel_algorithm                   201603L [C++17]
-    __cpp_lib_polymorphic_allocator                201902L [C++20]
-    __cpp_lib_quoted_string_io                     201304L [C++14]
-    __cpp_lib_ranges                               202106L [C++20]
-    __cpp_lib_ranges_as_rvalue                     202207L [C++23]
-    __cpp_lib_ranges_chunk                         202202L [C++23]
-    __cpp_lib_ranges_chunk_by                      202202L [C++23]
-    __cpp_lib_ranges_iota                          202202L [C++23]
-    __cpp_lib_ranges_join_with                     202202L [C++23]
-    __cpp_lib_ranges_slide                         202202L [C++23]
-    __cpp_lib_ranges_starts_ends_with              202106L [C++23]
-    __cpp_lib_ranges_to_container                  202202L [C++23]
-    __cpp_lib_ranges_zip                           202110L [C++23]
-    __cpp_lib_raw_memory_algorithms                201606L [C++17]
-    __cpp_lib_reference_from_temporary             202202L [C++23]
-    __cpp_lib_remove_cvref                         201711L [C++20]
-    __cpp_lib_result_of_sfinae                     201210L [C++14]
-    __cpp_lib_robust_nonmodifying_seq_ops          201304L [C++14]
-    __cpp_lib_sample                               201603L [C++17]
-    __cpp_lib_scoped_lock                          201703L [C++17]
-    __cpp_lib_semaphore                            201907L [C++20]
-    __cpp_lib_shared_mutex                         201505L [C++17]
-    __cpp_lib_shared_ptr_arrays                    201611L [C++17]
-                                                   201707L [C++20]
-    __cpp_lib_shared_ptr_weak_type                 201606L [C++17]
-    __cpp_lib_shared_timed_mutex                   201402L [C++14]
-    __cpp_lib_shift                                201806L [C++20]
-    __cpp_lib_smart_ptr_for_overwrite              202002L [C++20]
-    __cpp_lib_source_location                      201907L [C++20]
-    __cpp_lib_span                                 202002L [C++20]
-    __cpp_lib_spanstream                           202106L [C++23]
-    __cpp_lib_ssize                                201902L [C++20]
-    __cpp_lib_stacktrace                           202011L [C++23]
-    __cpp_lib_starts_ends_with                     201711L [C++20]
-    __cpp_lib_stdatomic_h                          202011L [C++23]
-    __cpp_lib_string_contains                      202011L [C++23]
-    __cpp_lib_string_resize_and_overwrite          202110L [C++23]
-    __cpp_lib_string_udls                          201304L [C++14]
-    __cpp_lib_string_view                          201606L [C++17]
-                                                   201803L [C++20]
-    __cpp_lib_syncbuf                              201803L [C++20]
-    __cpp_lib_three_way_comparison                 201907L [C++20]
-    __cpp_lib_to_address                           201711L [C++20]
-    __cpp_lib_to_array                             201907L [C++20]
-    __cpp_lib_to_chars                             201611L [C++17]
-    __cpp_lib_to_underlying                        202102L [C++23]
-    __cpp_lib_transformation_trait_aliases         201304L [C++14]
-    __cpp_lib_transparent_operators                201210L [C++14]
-                                                   201510L [C++17]
-    __cpp_lib_tuple_element_t                      201402L [C++14]
-    __cpp_lib_tuples_by_type                       201304L [C++14]
-    __cpp_lib_type_identity                        201806L [C++20]
-    __cpp_lib_type_trait_variable_templates        201510L [C++17]
-    __cpp_lib_uncaught_exceptions                  201411L [C++17]
-    __cpp_lib_unordered_map_try_emplace            201411L [C++17]
-    __cpp_lib_unreachable                          202202L [C++23]
-    __cpp_lib_unwrap_ref                           201811L [C++20]
-    __cpp_lib_variant                              202102L [C++17]
-    __cpp_lib_void_t                               201411L [C++17]
+/*  Constant                                         Value
+    __cpp_lib_adaptor_iterator_pair_constructor      202106L [C++23]
+    __cpp_lib_addressof_constexpr                    201603L [C++17]
+    __cpp_lib_allocate_at_least                      202106L [C++23]
+    __cpp_lib_allocator_traits_is_always_equal       201411L [C++17]
+    __cpp_lib_any                                    201606L [C++17]
+    __cpp_lib_apply                                  201603L [C++17]
+    __cpp_lib_array_constexpr                        201603L [C++17]
+                                                     201811L [C++20]
+    __cpp_lib_as_const                               201510L [C++17]
+    __cpp_lib_associative_heterogeneous_erasure      202110L [C++23]
+    __cpp_lib_associative_heterogeneous_insertion    202306L [C++26]
+    __cpp_lib_assume_aligned                         201811L [C++20]
+    __cpp_lib_atomic_flag_test                       201907L [C++20]
+    __cpp_lib_atomic_float                           201711L [C++20]
+    __cpp_lib_atomic_is_always_lock_free             201603L [C++17]
+    __cpp_lib_atomic_lock_free_type_aliases          201907L [C++20]
+    __cpp_lib_atomic_ref                             201806L [C++20]
+    __cpp_lib_atomic_shared_ptr                      201711L [C++20]
+    __cpp_lib_atomic_value_initialization            201911L [C++20]
+    __cpp_lib_atomic_wait                            201907L [C++20]
+    __cpp_lib_barrier                                201907L [C++20]
+    __cpp_lib_bind_back                              202202L [C++23]
+                                                     202306L [C++26]
+    __cpp_lib_bind_front                             201907L [C++20]
+                                                     202306L [C++26]
+    __cpp_lib_bit_cast                               201806L [C++20]
+    __cpp_lib_bitops                                 201907L [C++20]
+    __cpp_lib_bitset                                 202306L [C++26]
+    __cpp_lib_bool_constant                          201505L [C++17]
+    __cpp_lib_bounded_array_traits                   201902L [C++20]
+    __cpp_lib_boyer_moore_searcher                   201603L [C++17]
+    __cpp_lib_byte                                   201603L [C++17]
+    __cpp_lib_byteswap                               202110L [C++23]
+    __cpp_lib_char8_t                                201907L [C++20]
+    __cpp_lib_chrono                                 201611L [C++17]
+    __cpp_lib_chrono_udls                            201304L [C++14]
+    __cpp_lib_clamp                                  201603L [C++17]
+    __cpp_lib_complex_udls                           201309L [C++14]
+    __cpp_lib_concepts                               202002L [C++20]
+    __cpp_lib_constexpr_algorithms                   201806L [C++20]
+    __cpp_lib_constexpr_bitset                       202207L [C++23]
+    __cpp_lib_constexpr_charconv                     202207L [C++23]
+    __cpp_lib_constexpr_cmath                        202202L [C++23]
+    __cpp_lib_constexpr_complex                      201711L [C++20]
+    __cpp_lib_constexpr_dynamic_alloc                201907L [C++20]
+    __cpp_lib_constexpr_functional                   201907L [C++20]
+    __cpp_lib_constexpr_iterator                     201811L [C++20]
+    __cpp_lib_constexpr_memory                       201811L [C++20]
+                                                     202202L [C++23]
+    __cpp_lib_constexpr_numeric                      201911L [C++20]
+    __cpp_lib_constexpr_string                       201907L [C++20]
+    __cpp_lib_constexpr_string_view                  201811L [C++20]
+    __cpp_lib_constexpr_tuple                        201811L [C++20]
+    __cpp_lib_constexpr_typeinfo                     202106L [C++23]
+    __cpp_lib_constexpr_utility                      201811L [C++20]
+    __cpp_lib_constexpr_vector                       201907L [C++20]
+    __cpp_lib_copyable_function                      202306L [C++26]
+    __cpp_lib_coroutine                              201902L [C++20]
+    __cpp_lib_destroying_delete                      201806L [C++20]
+    __cpp_lib_enable_shared_from_this                201603L [C++17]
+    __cpp_lib_endian                                 201907L [C++20]
+    __cpp_lib_erase_if                               202002L [C++20]
+    __cpp_lib_exchange_function                      201304L [C++14]
+    __cpp_lib_execution                              201603L [C++17]
+                                                     201902L [C++20]
+    __cpp_lib_expected                               202211L [C++23]
+    __cpp_lib_filesystem                             201703L [C++17]
+    __cpp_lib_format                                 202106L [C++20]
+    __cpp_lib_format_ranges                          202207L [C++23]
+    __cpp_lib_formatters                             202302L [C++23]
+    __cpp_lib_forward_like                           202207L [C++23]
+    __cpp_lib_fstream_native_handle                  202306L [C++26]
+    __cpp_lib_function_ref                           202306L [C++26]
+    __cpp_lib_gcd_lcm                                201606L [C++17]
+    __cpp_lib_generic_associative_lookup             201304L [C++14]
+    __cpp_lib_generic_unordered_lookup               201811L [C++20]
+    __cpp_lib_hardware_interference_size             201703L [C++17]
+    __cpp_lib_has_unique_object_representations      201606L [C++17]
+    __cpp_lib_hazard_pointer                         202306L [C++26]
+    __cpp_lib_hypot                                  201603L [C++17]
+    __cpp_lib_incomplete_container_elements          201505L [C++17]
+    __cpp_lib_int_pow2                               202002L [C++20]
+    __cpp_lib_integer_comparison_functions           202002L [C++20]
+    __cpp_lib_integer_sequence                       201304L [C++14]
+    __cpp_lib_integral_constant_callable             201304L [C++14]
+    __cpp_lib_interpolate                            201902L [C++20]
+    __cpp_lib_invoke                                 201411L [C++17]
+    __cpp_lib_invoke_r                               202106L [C++23]
+    __cpp_lib_is_aggregate                           201703L [C++17]
+    __cpp_lib_is_constant_evaluated                  201811L [C++20]
+    __cpp_lib_is_final                               201402L [C++14]
+    __cpp_lib_is_invocable                           201703L [C++17]
+    __cpp_lib_is_layout_compatible                   201907L [C++20]
+    __cpp_lib_is_nothrow_convertible                 201806L [C++20]
+    __cpp_lib_is_null_pointer                        201309L [C++14]
+    __cpp_lib_is_pointer_interconvertible            201907L [C++20]
+    __cpp_lib_is_scoped_enum                         202011L [C++23]
+    __cpp_lib_is_swappable                           201603L [C++17]
+    __cpp_lib_jthread                                201911L [C++20]
+    __cpp_lib_latch                                  201907L [C++20]
+    __cpp_lib_launder                                201606L [C++17]
+    __cpp_lib_list_remove_return_type                201806L [C++20]
+    __cpp_lib_logical_traits                         201510L [C++17]
+    __cpp_lib_make_from_tuple                        201606L [C++17]
+    __cpp_lib_make_reverse_iterator                  201402L [C++14]
+    __cpp_lib_make_unique                            201304L [C++14]
+    __cpp_lib_map_try_emplace                        201411L [C++17]
+    __cpp_lib_math_constants                         201907L [C++20]
+    __cpp_lib_math_special_functions                 201603L [C++17]
+    __cpp_lib_mdspan                                 202207L [C++23]
+    __cpp_lib_memory_resource                        201603L [C++17]
+    __cpp_lib_move_iterator_concept                  202207L [C++20]
+    __cpp_lib_move_only_function                     202110L [C++23]
+    __cpp_lib_node_extract                           201606L [C++17]
+    __cpp_lib_nonmember_container_access             201411L [C++17]
+    __cpp_lib_not_fn                                 201603L [C++17]
+    __cpp_lib_null_iterators                         201304L [C++14]
+    __cpp_lib_optional                               201606L [C++17]
+                                                     202110L [C++23]
+    __cpp_lib_out_ptr                                202106L [C++23]
+    __cpp_lib_parallel_algorithm                     201603L [C++17]
+    __cpp_lib_polymorphic_allocator                  201902L [C++20]
+    __cpp_lib_quoted_string_io                       201304L [C++14]
+    __cpp_lib_ranges                                 202106L [C++20]
+    __cpp_lib_ranges_as_rvalue                       202207L [C++23]
+    __cpp_lib_ranges_chunk                           202202L [C++23]
+    __cpp_lib_ranges_chunk_by                        202202L [C++23]
+    __cpp_lib_ranges_iota                            202202L [C++23]
+    __cpp_lib_ranges_join_with                       202202L [C++23]
+    __cpp_lib_ranges_slide                           202202L [C++23]
+    __cpp_lib_ranges_starts_ends_with                202106L [C++23]
+    __cpp_lib_ranges_to_container                    202202L [C++23]
+    __cpp_lib_ranges_zip                             202110L [C++23]
+    __cpp_lib_ratio                                  202306L [C++26]
+    __cpp_lib_raw_memory_algorithms                  201606L [C++17]
+    __cpp_lib_rcu                                    202306L [C++26]
+    __cpp_lib_reference_from_temporary               202202L [C++23]
+    __cpp_lib_remove_cvref                           201711L [C++20]
+    __cpp_lib_result_of_sfinae                       201210L [C++14]
+    __cpp_lib_robust_nonmodifying_seq_ops            201304L [C++14]
+    __cpp_lib_sample                                 201603L [C++17]
+    __cpp_lib_scoped_lock                            201703L [C++17]
+    __cpp_lib_semaphore                              201907L [C++20]
+    __cpp_lib_shared_mutex                           201505L [C++17]
+    __cpp_lib_shared_ptr_arrays                      201611L [C++17]
+                                                     201707L [C++20]
+    __cpp_lib_shared_ptr_weak_type                   201606L [C++17]
+    __cpp_lib_shared_timed_mutex                     201402L [C++14]
+    __cpp_lib_shift                                  201806L [C++20]
+    __cpp_lib_smart_ptr_for_overwrite                202002L [C++20]
+    __cpp_lib_smart_ptr_owner_equality               202306L [C++26]
+    __cpp_lib_source_location                        201907L [C++20]
+    __cpp_lib_span                                   202002L [C++20]
+    __cpp_lib_spanstream                             202106L [C++23]
+    __cpp_lib_ssize                                  201902L [C++20]
+    __cpp_lib_sstream_from_string_view               202306L [C++26]
+    __cpp_lib_stacktrace                             202011L [C++23]
+    __cpp_lib_starts_ends_with                       201711L [C++20]
+    __cpp_lib_stdatomic_h                            202011L [C++23]
+    __cpp_lib_string_contains                        202011L [C++23]
+    __cpp_lib_string_resize_and_overwrite            202110L [C++23]
+    __cpp_lib_string_udls                            201304L [C++14]
+    __cpp_lib_string_view                            201606L [C++17]
+                                                     201803L [C++20]
+    __cpp_lib_submdspan                              202306L [C++26]
+    __cpp_lib_syncbuf                                201803L [C++20]
+    __cpp_lib_text_encoding                          202306L [C++26]
+    __cpp_lib_three_way_comparison                   201907L [C++20]
+    __cpp_lib_to_address                             201711L [C++20]
+    __cpp_lib_to_array                               201907L [C++20]
+    __cpp_lib_to_chars                               201611L [C++17]
+    __cpp_lib_to_string                              202306L [C++23]
+    __cpp_lib_to_underlying                          202102L [C++23]
+    __cpp_lib_transformation_trait_aliases           201304L [C++14]
+    __cpp_lib_transparent_operators                  201210L [C++14]
+                                                     201510L [C++17]
+    __cpp_lib_tuple_element_t                        201402L [C++14]
+    __cpp_lib_tuples_by_type                         201304L [C++14]
+    __cpp_lib_type_identity                          201806L [C++20]
+    __cpp_lib_type_trait_variable_templates          201510L [C++17]
+    __cpp_lib_uncaught_exceptions                    201411L [C++17]
+    __cpp_lib_unordered_map_try_emplace              201411L [C++17]
+    __cpp_lib_unreachable                            202202L [C++23]
+    __cpp_lib_unwrap_ref                             201811L [C++20]
+    __cpp_lib_variant                                202102L [C++17]
+    __cpp_lib_void_t                                 201411L [C++17]
+    __cpp_lib_within_lifetime                        202306L [C++26]
 */
 
 #include <version>
@@ -230,6 +246,10 @@
 #   error "__cpp_lib_associative_heterogeneous_erasure should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_associative_heterogeneous_insertion
+#   error "__cpp_lib_associative_heterogeneous_insertion should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_assume_aligned
 #   error "__cpp_lib_assume_aligned should not be defined before c++20"
 # endif
@@ -286,6 +306,10 @@
 #   error "__cpp_lib_bitops should not be defined before c++20"
 # endif
 
+# ifdef __cpp_lib_bitset
+#   error "__cpp_lib_bitset should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_bool_constant
 #   error "__cpp_lib_bool_constant should not be defined before c++17"
 # endif
@@ -394,6 +418,10 @@
 #   error "__cpp_lib_constexpr_vector should not be defined before c++20"
 # endif
 
+# ifdef __cpp_lib_copyable_function
+#   error "__cpp_lib_copyable_function should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_coroutine
 #   error "__cpp_lib_coroutine should not be defined before c++20"
 # endif
@@ -446,6 +474,14 @@
 #   error "__cpp_lib_forward_like should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_fstream_native_handle
+#   error "__cpp_lib_fstream_native_handle should not be defined before c++26"
+# endif
+
+# ifdef __cpp_lib_function_ref
+#   error "__cpp_lib_function_ref should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_gcd_lcm
 #   error "__cpp_lib_gcd_lcm should not be defined before c++17"
 # endif
@@ -466,6 +502,10 @@
 #   error "__cpp_lib_has_unique_object_representations should not be defined before c++17"
 # endif
 
+# ifdef __cpp_lib_hazard_pointer
+#   error "__cpp_lib_hazard_pointer should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_hypot
 #   error "__cpp_lib_hypot should not be defined before c++17"
 # endif
@@ -678,10 +718,18 @@
 #   error "__cpp_lib_ranges_zip should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_ratio
+#   error "__cpp_lib_ratio should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_raw_memory_algorithms
 #   error "__cpp_lib_raw_memory_algorithms should not be defined before c++17"
 # endif
 
+# ifdef __cpp_lib_rcu
+#   error "__cpp_lib_rcu should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_reference_from_temporary
 #   error "__cpp_lib_reference_from_temporary should not be defined before c++23"
 # endif
@@ -734,6 +782,10 @@
 #   error "__cpp_lib_smart_ptr_for_overwrite should not be defined before c++20"
 # endif
 
+# ifdef __cpp_lib_smart_ptr_owner_equality
+#   error "__cpp_lib_smart_ptr_owner_equality should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_source_location
 #   error "__cpp_lib_source_location should not be defined before c++20"
 # endif
@@ -750,6 +802,10 @@
 #   error "__cpp_lib_ssize should not be defined before c++20"
 # endif
 
+# ifdef __cpp_lib_sstream_from_string_view
+#   error "__cpp_lib_sstream_from_string_view should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_stacktrace
 #   error "__cpp_lib_stacktrace should not be defined before c++23"
 # endif
@@ -778,10 +834,18 @@
 #   error "__cpp_lib_string_view should not be defined before c++17"
 # endif
 
+# ifdef __cpp_lib_submdspan
+#   error "__cpp_lib_submdspan should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_syncbuf
 #   error "__cpp_lib_syncbuf should not be defined before c++20"
 # endif
 
+# ifdef __cpp_lib_text_encoding
+#   error "__cpp_lib_text_encoding should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_three_way_comparison
 #   error "__cpp_lib_three_way_comparison should not be defined before c++20"
 # endif
@@ -798,6 +862,10 @@
 #   error "__cpp_lib_to_chars should not be defined before c++17"
 # endif
 
+# ifdef __cpp_lib_to_string
+#   error "__cpp_lib_to_string should not be defined before c++23"
+# endif
+
 # ifdef __cpp_lib_to_underlying
 #   error "__cpp_lib_to_underlying should not be defined before c++23"
 # endif
@@ -850,6 +918,10 @@
 #   error "__cpp_lib_void_t should not be defined before c++17"
 # endif
 
+# ifdef __cpp_lib_within_lifetime
+#   error "__cpp_lib_within_lifetime should not be defined before c++26"
+# endif
+
 #elif TEST_STD_VER == 14
 
 # ifdef __cpp_lib_adaptor_iterator_pair_constructor
@@ -888,6 +960,10 @@
 #   error "__cpp_lib_associative_heterogeneous_erasure should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_associative_heterogeneous_insertion
+#   error "__cpp_lib_associative_heterogeneous_insertion should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_assume_aligned
 #   error "__cpp_lib_assume_aligned should not be defined before c++20"
 # endif
@@ -944,6 +1020,10 @@
 #   error "__cpp_lib_bitops should not be defined before c++20"
 # endif
 
+# ifdef __cpp_lib_bitset
+#   error "__cpp_lib_bitset should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_bool_constant
 #   error "__cpp_lib_bool_constant should not be defined before c++17"
 # endif
@@ -1058,6 +1138,10 @@
 #   error "__cpp_lib_constexpr_vector should not be defined before c++20"
 # endif
 
+# ifdef __cpp_lib_copyable_function
+#   error "__cpp_lib_copyable_function should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_coroutine
 #   error "__cpp_lib_coroutine should not be defined before c++20"
 # endif
@@ -1113,6 +1197,14 @@
 #   error "__cpp_lib_forward_like should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_fstream_native_handle
+#   error "__cpp_lib_fstream_native_handle should not be defined before c++26"
+# endif
+
+# ifdef __cpp_lib_function_ref
+#   error "__cpp_lib_function_ref should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_gcd_lcm
 #   error "__cpp_lib_gcd_lcm should not be defined before c++17"
 # endif
@@ -1136,6 +1228,10 @@
 #   error "__cpp_lib_has_unique_object_representations should not be defined before c++17"
 # endif
 
+# ifdef __cpp_lib_hazard_pointer
+#   error "__cpp_lib_hazard_pointer should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_hypot
 #   error "__cpp_lib_hypot should not be defined before c++17"
 # endif
@@ -1372,10 +1468,18 @@
 #   error "__cpp_lib_ranges_zip should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_ratio
+#   error "__cpp_lib_ratio should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_raw_memory_algorithms
 #   error "__cpp_lib_raw_memory_algorithms should not be defined before c++17"
 # endif
 
+# ifdef __cpp_lib_rcu
+#   error "__cpp_lib_rcu should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_reference_from_temporary
 #   error "__cpp_lib_reference_from_temporary should not be defined before c++23"
 # endif
@@ -1443,6 +1547,10 @@
 #   error "__cpp_lib_smart_ptr_for_overwrite should not be defined before c++20"
 # endif
 
+# ifdef __cpp_lib_smart_ptr_owner_equality
+#   error "__cpp_lib_smart_ptr_owner_equality should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_source_location
 #   error "__cpp_lib_source_location should not be defined before c++20"
 # endif
@@ -1459,6 +1567,10 @@
 #   error "__cpp_lib_ssize should not be defined before c++20"
 # endif
 
+# ifdef __cpp_lib_sstream_from_string_view
+#   error "__cpp_lib_sstream_from_string_view should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_stacktrace
 #   error "__cpp_lib_stacktrace should not be defined before c++23"
 # endif
@@ -1490,10 +1602,18 @@
 #   error "__cpp_lib_string_view should not be defined before c++17"
 # endif
 
+# ifdef __cpp_lib_submdspan
+#   error "__cpp_lib_submdspan should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_syncbuf
 #   error "__cpp_lib_syncbuf should not be defined before c++20"
 # endif
 
+# ifdef __cpp_lib_text_encoding
+#   error "__cpp_lib_text_encoding should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_three_way_comparison
 #   error "__cpp_lib_three_way_comparison should not be defined before c++20"
 # endif
@@ -1510,6 +1630,10 @@
 #   error "__cpp_lib_to_chars should not be defined before c++17"
 # endif
 
+# ifdef __cpp_lib_to_string
+#   error "__cpp_lib_to_string should not be defined before c++23"
+# endif
+
 # ifdef __cpp_lib_to_underlying
 #   error "__cpp_lib_to_underlying should not be defined before c++23"
 # endif
@@ -1574,6 +1698,10 @@
 #   error "__cpp_lib_void_t should not be defined before c++17"
 # endif
 
+# ifdef __cpp_lib_within_lifetime
+#   error "__cpp_lib_within_lifetime should not be defined before c++26"
+# endif
+
 #elif TEST_STD_VER == 17
 
 # ifdef __cpp_lib_adaptor_iterator_pair_constructor
@@ -1630,6 +1758,10 @@
 #   error "__cpp_lib_associative_heterogeneous_erasure should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_associative_heterogeneous_insertion
+#   error "__cpp_lib_associative_heterogeneous_insertion should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_assume_aligned
 #   error "__cpp_lib_assume_aligned should not be defined before c++20"
 # endif
@@ -1689,6 +1821,10 @@
 #   error "__cpp_lib_bitops should not be defined before c++20"
 # endif
 
+# ifdef __cpp_lib_bitset
+#   error "__cpp_lib_bitset should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_bool_constant
 #   error "__cpp_lib_bool_constant should be defined in c++17"
 # endif
@@ -1818,6 +1954,10 @@
 #   error "__cpp_lib_constexpr_vector should not be defined before c++20"
 # endif
 
+# ifdef __cpp_lib_copyable_function
+#   error "__cpp_lib_copyable_function should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_coroutine
 #   error "__cpp_lib_coroutine should not be defined before c++20"
 # endif
@@ -1894,6 +2034,14 @@
 #   error "__cpp_lib_forward_like should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_fstream_native_handle
+#   error "__cpp_lib_fstream_native_handle should not be defined before c++26"
+# endif
+
+# ifdef __cpp_lib_function_ref
+#   error "__cpp_lib_function_ref should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_gcd_lcm
 #   error "__cpp_lib_gcd_lcm should be defined in c++17"
 # endif
@@ -1932,6 +2080,10 @@
 #   error "__cpp_lib_has_unique_object_representations should have the value 201606L in c++17"
 # endif
 
+# ifdef __cpp_lib_hazard_pointer
+#   error "__cpp_lib_hazard_pointer should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_hypot
 #   error "__cpp_lib_hypot should be defined in c++17"
 # endif
@@ -2231,6 +2383,10 @@
 #   error "__cpp_lib_ranges_zip should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_ratio
+#   error "__cpp_lib_ratio should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_raw_memory_algorithms
 #   error "__cpp_lib_raw_memory_algorithms should be defined in c++17"
 # endif
@@ -2238,6 +2394,10 @@
 #   error "__cpp_lib_raw_memory_algorithms should have the value 201606L in c++17"
 # endif
 
+# ifdef __cpp_lib_rcu
+#   error "__cpp_lib_rcu should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_reference_from_temporary
 #   error "__cpp_lib_reference_from_temporary should not be defined before c++23"
 # endif
@@ -2326,6 +2486,10 @@
 #   error "__cpp_lib_smart_ptr_for_overwrite should not be defined before c++20"
 # endif
 
+# ifdef __cpp_lib_smart_ptr_owner_equality
+#   error "__cpp_lib_smart_ptr_owner_equality should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_source_location
 #   error "__cpp_lib_source_location should not be defined before c++20"
 # endif
@@ -2342,6 +2506,10 @@
 #   error "__cpp_lib_ssize should not be defined before c++20"
 # endif
 
+# ifdef __cpp_lib_sstream_from_string_view
+#   error "__cpp_lib_sstream_from_string_view should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_stacktrace
 #   error "__cpp_lib_stacktrace should not be defined before c++23"
 # endif
@@ -2376,10 +2544,18 @@
 #   error "__cpp_lib_string_view should have the value 201606L in c++17"
 # endif
 
+# ifdef __cpp_lib_submdspan
+#   error "__cpp_lib_submdspan should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_syncbuf
 #   error "__cpp_lib_syncbuf should not be defined before c++20"
 # endif
 
+# ifdef __cpp_lib_text_encoding
+#   error "__cpp_lib_text_encoding should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_three_way_comparison
 #   error "__cpp_lib_three_way_comparison should not be defined before c++20"
 # endif
@@ -2405,6 +2581,10 @@
 #   endif
 # endif
 
+# ifdef __cpp_lib_to_string
+#   error "__cpp_lib_to_string should not be defined before c++23"
+# endif
+
 # ifdef __cpp_lib_to_underlying
 #   error "__cpp_lib_to_underlying should not be defined before c++23"
 # endif
@@ -2484,6 +2664,10 @@
 #   error "__cpp_lib_void_t should have the value 201411L in c++17"
 # endif
 
+# ifdef __cpp_lib_within_lifetime
+#   error "__cpp_lib_within_lifetime should not be defined before c++26"
+# endif
+
 #elif TEST_STD_VER == 20
 
 # ifdef __cpp_lib_adaptor_iterator_pair_constructor
@@ -2540,6 +2724,10 @@
 #   error "__cpp_lib_associative_heterogeneous_erasure should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_associative_heterogeneous_insertion
+#   error "__cpp_lib_associative_heterogeneous_insertion should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_assume_aligned
 #   error "__cpp_lib_assume_aligned should be defined in c++20"
 # endif
@@ -2671,6 +2859,10 @@
 #   endif
 # endif
 
+# ifdef __cpp_lib_bitset
+#   error "__cpp_lib_bitset should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_bool_constant
 #   error "__cpp_lib_bool_constant should be defined in c++20"
 # endif
@@ -2851,6 +3043,10 @@
 #   error "__cpp_lib_constexpr_vector should have the value 201907L in c++20"
 # endif
 
+# ifdef __cpp_lib_copyable_function
+#   error "__cpp_lib_copyable_function should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_coroutine
 #   error "__cpp_lib_coroutine should be defined in c++20"
 # endif
@@ -2954,6 +3150,14 @@
 #   error "__cpp_lib_forward_like should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_fstream_native_handle
+#   error "__cpp_lib_fstream_native_handle should not be defined before c++26"
+# endif
+
+# ifdef __cpp_lib_function_ref
+#   error "__cpp_lib_function_ref should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_gcd_lcm
 #   error "__cpp_lib_gcd_lcm should be defined in c++20"
 # endif
@@ -2995,6 +3199,10 @@
 #   error "__cpp_lib_has_unique_object_representations should have the value 201606L in c++20"
 # endif
 
+# ifdef __cpp_lib_hazard_pointer
+#   error "__cpp_lib_hazard_pointer should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_hypot
 #   error "__cpp_lib_hypot should be defined in c++20"
 # endif
@@ -3360,6 +3568,10 @@
 #   error "__cpp_lib_ranges_zip should not be defined before c++23"
 # endif
 
+# ifdef __cpp_lib_ratio
+#   error "__cpp_lib_ratio should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_raw_memory_algorithms
 #   error "__cpp_lib_raw_memory_algorithms should be defined in c++20"
 # endif
@@ -3367,6 +3579,10 @@
 #   error "__cpp_lib_raw_memory_algorithms should have the value 201606L in c++20"
 # endif
 
+# ifdef __cpp_lib_rcu
+#   error "__cpp_lib_rcu should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_reference_from_temporary
 #   error "__cpp_lib_reference_from_temporary should not be defined before c++23"
 # endif
@@ -3479,6 +3695,10 @@
 #   endif
 # endif
 
+# ifdef __cpp_lib_smart_ptr_owner_equality
+#   error "__cpp_lib_smart_ptr_owner_equality should not be defined before c++26"
+# endif
+
 # if __has_builtin(__builtin_source_location) && !(defined(TEST_APPLE_CLANG_VER) && TEST_APPLE_CLANG_VER <= 1403)
 #   ifndef __cpp_lib_source_location
 #     error "__cpp_lib_source_location should be defined in c++20"
@@ -3510,6 +3730,10 @@
 #   error "__cpp_lib_ssize should have the value 201902L in c++20"
 # endif
 
+# ifdef __cpp_lib_sstream_from_string_view
+#   error "__cpp_lib_sstream_from_string_view should not be defined before c++26"
+# endif
+
 # ifdef __cpp_lib_stacktrace
 #   error "__cpp_lib_stacktrace should not be defined before c++23"
 # endif
@@ -3547,6 +3771,10 @@
 #   error "__cpp_lib_string_view should have the value 201803L in c++20"
 # endif
 
+# ifdef __cpp_lib_submdspan
+#   error "__cpp_lib_submdspan should not be defined before c++26"
+# endif
+
 # if !defined(_LIBCPP_VERSION)
 #   ifndef __cpp_lib_syncbuf
 #     error "__cpp_lib_syncbuf should be defined in c++20"
@@ -3560,6 +3788,10 @@
 #   endif
 # endif
 
+# ifdef __cpp_lib_text_encoding
+#   error "__cpp_lib_text_encoding should not be defined before c++26"
+# endif
+
 # if !defined(_LIBCPP_VERSION)
 #   ifndef __cpp_lib_three_way_comparison
 #     error "__cpp_lib_three_way_comparison should be defined in c++20"
@@ -3600,6 +3832,10 @@
 #   endif
 # endif
 
+# ifdef __cpp_lib_to_string
+#   error "__cpp_lib_to_string should not be defined before c++23"
+# endif
+
 # ifdef __cpp_lib_to_underlying
 #   error "__cpp_lib_to_underlying should not be defined before c++23"
 # endif
@@ -3685,6 +3921,10 @@
 #   error "__cpp_lib_void_t should have the value 201411L in c++20"
 # endif
 
+# ifdef __cpp_lib_within_lifetime
+#   error "__cpp_lib_within_lifetime should not be defined before c++26"
+# endif
+
 #elif TEST_STD_VER == 23
 
 # ifndef __cpp_lib_adaptor_iterator_pair_constructor
@@ -3756,6 +3996,10 @@
 #   endif
 # endif
 
+# ifdef __cpp_lib_associative_heterogeneous_insertion
+#   error "__cpp_lib_associative_heterogeneous_insertion should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_assume_aligned
 #   error "__cpp_lib_assume_aligned should be defined in c++23"
 # endif
@@ -3896,6 +4140,10 @@
 #   endif
 # endif
 
+# ifdef __cpp_lib_bitset
+#   error "__cpp_lib_bitset should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_bool_constant
 #   error "__cpp_lib_bool_constant should be defined in c++23"
 # endif
@@ -4097,6 +4345,10 @@
 #   error "__cpp_lib_constexpr_vector should have the value 201907L in c++23"
 # endif
 
+# ifdef __cpp_lib_copyable_function
+#   error "__cpp_lib_copyable_function should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_coroutine
 #   error "__cpp_lib_coroutine should be defined in c++23"
 # endif
@@ -4218,6 +4470,14 @@
 #   error "__cpp_lib_forward_like should have the value 202207L in c++23"
 # endif
 
+# ifdef __cpp_lib_fstream_native_handle
+#   error "__cpp_lib_fstream_native_handle should not be defined before c++26"
+# endif
+
+# ifdef __cpp_lib_function_ref
+#   error "__cpp_lib_function_ref should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_gcd_lcm
 #   error "__cpp_lib_gcd_lcm should be defined in c++23"
 # endif
@@ -4259,6 +4519,10 @@
 #   error "__cpp_lib_has_unique_object_representations should have the value 201606L in c++23"
 # endif
 
+# ifdef __cpp_lib_hazard_pointer
+#   error "__cpp_lib_hazard_pointer should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_hypot
 #   error "__cpp_lib_hypot should be defined in c++23"
 # endif
@@ -4732,6 +4996,10 @@
 #   endif
 # endif
 
+# ifdef __cpp_lib_ratio
+#   error "__cpp_lib_ratio should not be defined before c++26"
+# endif
+
 # ifndef __cpp_lib_raw_memory_algorithms
 #   error "__cpp_lib_raw_memory_algorithms should be defined in c++23"
 # endif
@@ -4739,6 +5007,10 @@
 #   error "__cpp_lib_raw_memory_algorithms should have the value 201606L in c++23"
 # endif
 
+# ifdef __cpp_lib_rcu
+#   error "__cpp_lib_rcu should not be defined before c++26"
+# endif
+
 # if !defined(_LIBCPP_VERSION)
 #   ifndef __cpp_lib_reference_from_temporary
 #     error "__cpp_lib_reference_from_temporary should be defined in c++23"
@@ -4860,6 +5132,10 @@
 #   endif
 # endif
 
+# ifdef __cpp_lib_smart_ptr_owner_equality
+#   error "__cpp_lib_smart_ptr_owner_equality should not be defined before c++26"
+# endif
+
 # if __has_builtin(__builtin_source_location) && !(defined(TEST_APPLE_CLANG_VER) && TEST_APPLE_CLANG_VER <= 1403)
 #   ifndef __cpp_lib_source_location
 #     error "__cpp_lib_source_location should be defined in c++23"
@@ -4900,6 +5176,10 @@
 #   error "__cpp_lib_ssize should have the value 201902L in c++23"
 # endif
 
+# ifdef __cpp_lib_sstream_from_string_view
+#   error "__cpp_lib_sstream_from_string_view should not be defined before c++26"
+# endif
+
 # if !defined(_LIBCPP_VERSION)
 #   ifndef __cpp_lib_stacktrace
 #     error "__cpp_lib_stacktrace should be defined in c++23"
@@ -4955,6 +5235,10 @@
 #   error "__cpp_lib_string_view should have the value 201803L in c++23"
 # endif
 
+# ifdef __cpp_lib_submdspan
+#   error "__cpp_lib_submdspan should not be defined before c++26"
+# endif
+
 # if !defined(_LIBCPP_VERSION)
 #   ifndef __cpp_lib_syncbuf
 #     error "__cpp_lib_syncbuf should be defined in c++23"
@@ -4968,6 +5252,10 @@
 #   endif
 # endif
 
+# ifdef __cpp_lib_text_encoding
+#   error "__cpp_lib_text_encoding should not be defined before c++26"
+# endif
+
 # if !defined(_LIBCPP_VERSION)
 #   ifndef __cpp_lib_three_way_comparison
 #     error "__cpp_lib_three_way_comparison should be defined in c++23"
@@ -5008,6 +5296,19 @@
 #   endif
 # endif
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_to_string
+#     error "__cpp_lib_to_string should be defined in c++23"
+#   endif
+#   if __cpp_lib_to_string != 202306L
+#     error "__cpp_lib_to_string should have the value 202306L in c++23"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_to_string
+#     error "__cpp_lib_to_string should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 # ifndef __cpp_lib_to_underlying
 #   error "__cpp_lib_to_underlying should be defined in c++23"
 # endif
@@ -5099,6 +5400,10 @@
 #   error "__cpp_lib_void_t should have the value 201411L in c++23"
 # endif
 
+# ifdef __cpp_lib_within_lifetime
+#   error "__cpp_lib_within_lifetime should not be defined before c++26"
+# endif
+
 #elif TEST_STD_VER > 23
 
 # ifndef __cpp_lib_adaptor_iterator_pair_constructor
@@ -5170,6 +5475,19 @@
 #   endif
 # endif
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_associative_heterogeneous_insertion
+#     error "__cpp_lib_associative_heterogeneous_insertion should be defined in c++26"
+#   endif
+#   if __cpp_lib_associative_heterogeneous_insertion != 202306L
+#     error "__cpp_lib_associative_heterogeneous_insertion should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_associative_heterogeneous_insertion
+#     error "__cpp_lib_associative_heterogeneous_insertion should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 # ifndef __cpp_lib_assume_aligned
 #   error "__cpp_lib_assume_aligned should be defined in c++26"
 # endif
@@ -5274,8 +5592,8 @@
 #   ifndef __cpp_lib_bind_back
 #     error "__cpp_lib_bind_back should be defined in c++26"
 #   endif
-#   if __cpp_lib_bind_back != 202202L
-#     error "__cpp_lib_bind_back should have the value 202202L in c++26"
+#   if __cpp_lib_bind_back != 202306L
+#     error "__cpp_lib_bind_back should have the value 202306L in c++26"
 #   endif
 # else // _LIBCPP_VERSION
 #   ifdef __cpp_lib_bind_back
@@ -5286,8 +5604,8 @@
 # ifndef __cpp_lib_bind_front
 #   error "__cpp_lib_bind_front should be defined in c++26"
 # endif
-# if __cpp_lib_bind_front != 201907L
-#   error "__cpp_lib_bind_front should have the value 201907L in c++26"
+# if __cpp_lib_bind_front != 202306L
+#   error "__cpp_lib_bind_front should have the value 202306L in c++26"
 # endif
 
 # ifndef __cpp_lib_bit_cast
@@ -5310,6 +5628,19 @@
 #   endif
 # endif
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_bitset
+#     error "__cpp_lib_bitset should be defined in c++26"
+#   endif
+#   if __cpp_lib_bitset != 202306L
+#     error "__cpp_lib_bitset should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_bitset
+#     error "__cpp_lib_bitset should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 # ifndef __cpp_lib_bool_constant
 #   error "__cpp_lib_bool_constant should be defined in c++26"
 # endif
@@ -5511,6 +5842,19 @@
 #   error "__cpp_lib_constexpr_vector should have the value 201907L in c++26"
 # endif
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_copyable_function
+#     error "__cpp_lib_copyable_function should be defined in c++26"
+#   endif
+#   if __cpp_lib_copyable_function != 202306L
+#     error "__cpp_lib_copyable_function should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_copyable_function
+#     error "__cpp_lib_copyable_function should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 # ifndef __cpp_lib_coroutine
 #   error "__cpp_lib_coroutine should be defined in c++26"
 # endif
@@ -5632,6 +5976,32 @@
 #   error "__cpp_lib_forward_like should have the value 202207L in c++26"
 # endif
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_fstream_native_handle
+#     error "__cpp_lib_fstream_native_handle should be defined in c++26"
+#   endif
+#   if __cpp_lib_fstream_native_handle != 202306L
+#     error "__cpp_lib_fstream_native_handle should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_fstream_native_handle
+#     error "__cpp_lib_fstream_native_handle should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_function_ref
+#     error "__cpp_lib_function_ref should be defined in c++26"
+#   endif
+#   if __cpp_lib_function_ref != 202306L
+#     error "__cpp_lib_function_ref should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_function_ref
+#     error "__cpp_lib_function_ref should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 # ifndef __cpp_lib_gcd_lcm
 #   error "__cpp_lib_gcd_lcm should be defined in c++26"
 # endif
@@ -5673,6 +6043,19 @@
 #   error "__cpp_lib_has_unique_object_representations should have the value 201606L in c++26"
 # endif
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_hazard_pointer
+#     error "__cpp_lib_hazard_pointer should be defined in c++26"
+#   endif
+#   if __cpp_lib_hazard_pointer != 202306L
+#     error "__cpp_lib_hazard_pointer should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_hazard_pointer
+#     error "__cpp_lib_hazard_pointer should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 # ifndef __cpp_lib_hypot
 #   error "__cpp_lib_hypot should be defined in c++26"
 # endif
@@ -6146,6 +6529,19 @@
 #   endif
 # endif
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_ratio
+#     error "__cpp_lib_ratio should be defined in c++26"
+#   endif
+#   if __cpp_lib_ratio != 202306L
+#     error "__cpp_lib_ratio should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_ratio
+#     error "__cpp_lib_ratio should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 # ifndef __cpp_lib_raw_memory_algorithms
 #   error "__cpp_lib_raw_memory_algorithms should be defined in c++26"
 # endif
@@ -6153,6 +6549,19 @@
 #   error "__cpp_lib_raw_memory_algorithms should have the value 201606L in c++26"
 # endif
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_rcu
+#     error "__cpp_lib_rcu should be defined in c++26"
+#   endif
+#   if __cpp_lib_rcu != 202306L
+#     error "__cpp_lib_rcu should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_rcu
+#     error "__cpp_lib_rcu should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 # if !defined(_LIBCPP_VERSION)
 #   ifndef __cpp_lib_reference_from_temporary
 #     error "__cpp_lib_reference_from_temporary should be defined in c++26"
@@ -6274,6 +6683,19 @@
 #   endif
 # endif
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_smart_ptr_owner_equality
+#     error "__cpp_lib_smart_ptr_owner_equality should be defined in c++26"
+#   endif
+#   if __cpp_lib_smart_ptr_owner_equality != 202306L
+#     error "__cpp_lib_smart_ptr_owner_equality should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_smart_ptr_owner_equality
+#     error "__cpp_lib_smart_ptr_owner_equality should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 # if __has_builtin(__builtin_source_location) && !(defined(TEST_APPLE_CLANG_VER) && TEST_APPLE_CLANG_VER <= 1403)
 #   ifndef __cpp_lib_source_location
 #     error "__cpp_lib_source_location should be defined in c++26"
@@ -6314,6 +6736,19 @@
 #   error "__cpp_lib_ssize should have the value 201902L in c++26"
 # endif
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_sstream_from_string_view
+#     error "__cpp_lib_sstream_from_string_view should be defined in c++26"
+#   endif
+#   if __cpp_lib_sstream_from_string_view != 202306L
+#     error "__cpp_lib_sstream_from_string_view should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_sstream_from_string_view
+#     error "__cpp_lib_sstream_from_string_view should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 # if !defined(_LIBCPP_VERSION)
 #   ifndef __cpp_lib_stacktrace
 #     error "__cpp_lib_stacktrace should be defined in c++26"
@@ -6369,6 +6804,19 @@
 #   error "__cpp_lib_string_view should have the value 201803L in c++26"
 # endif
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_submdspan
+#     error "__cpp_lib_submdspan should be defined in c++26"
+#   endif
+#   if __cpp_lib_submdspan != 202306L
+#     error "__cpp_lib_submdspan should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_submdspan
+#     error "__cpp_lib_submdspan should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 # if !defined(_LIBCPP_VERSION)
 #   ifndef __cpp_lib_syncbuf
 #     error "__cpp_lib_syncbuf should be defined in c++26"
@@ -6382,6 +6830,19 @@
 #   endif
 # endif
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_text_encoding
+#     error "__cpp_lib_text_encoding should be defined in c++26"
+#   endif
+#   if __cpp_lib_text_encoding != 202306L
+#     error "__cpp_lib_text_encoding should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_text_encoding
+#     error "__cpp_lib_text_encoding should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 # if !defined(_LIBCPP_VERSION)
 #   ifndef __cpp_lib_three_way_comparison
 #     error "__cpp_lib_three_way_comparison should be defined in c++26"
@@ -6422,6 +6883,19 @@
 #   endif
 # endif
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_to_string
+#     error "__cpp_lib_to_string should be defined in c++26"
+#   endif
+#   if __cpp_lib_to_string != 202306L
+#     error "__cpp_lib_to_string should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_to_string
+#     error "__cpp_lib_to_string should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 # ifndef __cpp_lib_to_underlying
 #   error "__cpp_lib_to_underlying should be defined in c++26"
 # endif
@@ -6513,5 +6987,18 @@
 #   error "__cpp_lib_void_t should have the value 201411L in c++26"
 # endif
 
+# if !defined(_LIBCPP_VERSION)
+#   ifndef __cpp_lib_within_lifetime
+#     error "__cpp_lib_within_lifetime should be defined in c++26"
+#   endif
+#   if __cpp_lib_within_lifetime != 202306L
+#     error "__cpp_lib_within_lifetime should have the value 202306L in c++26"
+#   endif
+# else // _LIBCPP_VERSION
+#   ifdef __cpp_lib_within_lifetime
+#     error "__cpp_lib_within_lifetime should not be defined because it is unimplemented in libc++!"
+#   endif
+# endif
+
 #endif // TEST_STD_VER > 23
 
diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py
index 19738d80e6f18..72ae5bdd653a8 100755
--- a/libcxx/utils/generate_feature_test_macro_components.py
+++ b/libcxx/utils/generate_feature_test_macro_components.py
@@ -87,7 +87,11 @@ def add_version_header(tc):
         },
         {
             "name": "__cpp_lib_allocate_at_least",
-            "values": {"c++23": 202106},
+            "values": {
+                          "c++23": 202106,
+                          # Note LWG3887 Version macro for allocate_at_least
+                          #"c++26": 202302, # P2652R2 Disallow User Specialization of allocator_traits
+                      },
             "headers": ["memory"],
         },
         {
@@ -133,6 +137,12 @@ def add_version_header(tc):
             "headers": ["map", "set", "unordered_map", "unordered_set"],
             "unimplemented": True,
         },
+        {
+            "name": "__cpp_lib_associative_heterogeneous_insertion",
+            "values": {"c++26": 202306}, # P2363R5 Extending associative containers with the remaining heterogeneous overloads
+            "headers": ["map", "set", "unordered_map", "unordered_set"],
+            "unimplemented": True,
+        },
         {
             "name": "__cpp_lib_assume_aligned",
             "values": {"c++20": 201811},
@@ -192,13 +202,19 @@ def add_version_header(tc):
         },
         {
             "name": "__cpp_lib_bind_back",
-            "values": {"c++23": 202202},
+            "values": {
+                          "c++23": 202202,
+                          "c++26": 202306, # P2714R1 Bind front and back to NTTP callables
+                      },
             "headers": ["functional"],
             "unimplemented": True,
         },
         {
             "name": "__cpp_lib_bind_front",
-            "values": {"c++20": 201907},
+            "values": {
+                          "c++20": 201907,
+                          "c++26": 202306, # P2714R1 Bind front and back to NTTP callables
+                      },
             "headers": ["functional"],
         },
         {
@@ -212,6 +228,12 @@ def add_version_header(tc):
             "headers": ["bit"],
             "unimplemented": True,
         },
+        {
+            "name": "__cpp_lib_bitset",
+            "values": {"c++26": 202306}, # P2697R1 Interfacing bitset with string_view
+            "headers": ["bitset"],
+            "unimplemented": True,
+        },
         {
             "name": "__cpp_lib_bool_constant",
             "values": {"c++17": 201505},
@@ -255,7 +277,10 @@ def add_version_header(tc):
         },
         {
             "name": "__cpp_lib_chrono",
-            "values": {"c++17": 201611},
+            "values": {
+                          "c++17": 201611,
+                          #"c++26": 202306, # P2592R3 Hashing support for std::chrono value classes
+                      },
             "headers": ["chrono"],
         },
         {
@@ -280,7 +305,10 @@ def add_version_header(tc):
         },
         {
             "name": "__cpp_lib_constexpr_algorithms",
-            "values": {"c++20": 201806},
+            "values": {
+                          "c++20": 201806,
+                          #"c++26": 202306, # P2562R1 constexpr Stable Sorting
+                      },
             "headers": ["algorithm", "utility"],
         },
         {
@@ -359,6 +387,12 @@ def add_version_header(tc):
             "values": {"c++20": 201907},
             "headers": ["vector"],
         },
+        {
+            "name": "__cpp_lib_copyable_function",
+            "values": {"c++26": 202306}, # P2548R6 copyable_function
+            "headers": ["functional"],
+            "unimplemented": True,
+        },
         {
             "name": "__cpp_lib_coroutine",
             "values": {"c++20": 201902},
@@ -428,6 +462,10 @@ def add_version_header(tc):
                 "c++20": 202106,
                 # "c++23": 202207, Not implemented P2419R2 Clarify handling of encodings in localized formatting of chrono types
             },
+            # Note these three papers are adopted at the June 2023 meeting and have sequential numbering
+            # 202304 P2510R3 Formatting pointers
+            # 202305 P2757R3 Type-checking format args
+            # 202306 P2637R3 Member Visit
             "headers": ["format"],
             "unimplemented": True,
         },
@@ -447,6 +485,18 @@ def add_version_header(tc):
             "values": {"c++23": 202207},
             "headers": ["utility"],
         },
+        {
+            "name": "__cpp_lib_fstream_native_handle",
+            "values": {"c++26": 202306}, # P1759R6 Native handles and file streams
+            "headers": ["fstream"],
+            "unimplemented": True,
+        },
+        {
+            "name": "__cpp_lib_function_ref",
+            "values": {"c++26": 202306}, # P0792R14 function_ref: a type-erased callable reference
+            "headers": ["functional"],
+            "unimplemented": True,
+        },
         {
             "name": "__cpp_lib_gcd_lcm",
             "values": {"c++17": 201606},
@@ -474,6 +524,12 @@ def add_version_header(tc):
             "values": {"c++17": 201606},
             "headers": ["type_traits"],
         },
+        {
+            "name": "__cpp_lib_hazard_pointer",
+            "values": {"c++26": 202306},  # P2530R3 Hazard Pointers for C++26
+            "headers": ["hazard_pointer"],# TODO verify this entry since the paper was underspecified.
+            "unimplemented": True,
+        },
         {
             "name": "__cpp_lib_hypot",
             "values": {"c++17": 201603},
@@ -679,7 +735,10 @@ def add_version_header(tc):
         },
         {
             "name": "__cpp_lib_not_fn",
-            "values": {"c++17": 201603},
+            "values": {
+                          "c++17": 201603,
+                          #"c++26": 202306, # P2714R1 Bind front and back to NTTP callables
+                      },
             "headers": ["functional"],
         },
         {
@@ -785,11 +844,23 @@ def add_version_header(tc):
             "headers": ["ranges", "tuple", "utility"],
             "unimplemented": True,
         },
+        {
+            "name": "__cpp_lib_ratio",
+            "values": {"c++26": 202306}, # P2734R0 Adding the new SI prefixes
+            "headers": ["ratio"],
+            "unimplemented": True,
+        },
         {
             "name": "__cpp_lib_raw_memory_algorithms",
             "values": {"c++17": 201606},
             "headers": ["memory"],
         },
+        {
+            "name": "__cpp_lib_rcu",
+            "values": {"c++26": 202306}, # P2545R4 Read-Copy Update (RCU)
+            "headers": ["rcu"],          # TODO verify this entry since the paper was underspecified.
+            "unimplemented": True,
+        },
         {
             "name": "__cpp_lib_reference_from_temporary",
             "values": {"c++23": 202202},
@@ -863,6 +934,12 @@ def add_version_header(tc):
             "headers": ["memory"],
             "unimplemented": True,
         },
+        {
+            "name": "__cpp_lib_smart_ptr_owner_equality",
+            "values": {"c++26": 202306}, # P1901R2 Enabling the Use of weak_ptr as Keys in Unordered Associative Containers
+            "headers": ["memory"],
+            "unimplemented": True,
+        },
         {
             "name": "__cpp_lib_source_location",
             "values": {"c++20": 201907},
@@ -886,6 +963,12 @@ def add_version_header(tc):
             "values": {"c++20": 201902},
             "headers": ["iterator"],
         },
+        {
+            "name": "__cpp_lib_sstream_from_string_view",
+            "values": {"c++26": 202306},  # P2495R3 Interfacing stringstreams with string_view
+            "headers": ["sstream"],
+            "unimplemented": True,
+        },
         {
             "name": "__cpp_lib_stacktrace",
             "values": {"c++23": 202011},
@@ -922,12 +1005,24 @@ def add_version_header(tc):
             "values": {"c++17": 201606, "c++20": 201803},
             "headers": ["string", "string_view"],
         },
+        {
+            "name": "__cpp_lib_submdspan",
+            "values": {"c++26": 202306}, # P2630R4 submdspan
+            "headers": ["mdspan"],
+            "unimplemented": True,
+        },
         {
             "name": "__cpp_lib_syncbuf",
             "values": {"c++20": 201803},
             "headers": ["syncstream"],
             "unimplemented": True,
         },
+        {
+            "name": "__cpp_lib_text_encoding",
+            "values": {"c++26": 202306}, # P1885R12 Naming Text Encodings to Demystify Them
+            "headers": ["text_encoding"],
+            "unimplemented": True,
+        },
         {
             "name": "__cpp_lib_three_way_comparison",
             "values": {"c++20": 201907},
@@ -946,10 +1041,19 @@ def add_version_header(tc):
         },
         {
             "name": "__cpp_lib_to_chars",
-            "values": {"c++17": 201611},
+            "values": {
+                         "c++17": 201611,
+                         #"c++26: 202306, # P2497R0 Testing for success or failure of <charconv> functions
+                      },
             "headers": ["charconv"],
             "unimplemented": True,
         },
+        {
+            "name": "__cpp_lib_to_string",
+            "values": {"c++23": 202306}, # P2587R3 to_string or not to_string
+            "headers": ["string"],
+            "unimplemented": True,
+        },
         {
             "name": "__cpp_lib_to_underlying",
             "values": {"c++23": 202102},
@@ -1015,6 +1119,12 @@ def add_version_header(tc):
             "values": {"c++17": 201411},
             "headers": ["type_traits"],
         },
+        {
+            "name": "__cpp_lib_within_lifetime",
+            "values": {"c++26": 202306}, # P2641R4 Checking if a union alternative is active
+            "headers": ["type_traits"],
+            "unimplemented": True,
+        },
     ]
 ]
 
@@ -1048,6 +1158,7 @@ def add_version_header(tc):
 lit_markup = {
     "barrier": ["UNSUPPORTED: no-threads"],
     "filesystem": ["UNSUPPORTED: no-filesystem"],
+    "fstream": ["UNSUPPORTED: no-localization"],
     "iomanip": ["UNSUPPORTED: no-localization"],
     "ios": ["UNSUPPORTED: no-localization"],
     "iostream": ["UNSUPPORTED: no-localization"],
@@ -1059,6 +1170,7 @@ def add_version_header(tc):
     "regex": ["UNSUPPORTED: no-localization"],
     "semaphore": ["UNSUPPORTED: no-threads"],
     "shared_mutex": ["UNSUPPORTED: no-threads"],
+    "sstream": ["UNSUPPORTED: no-localization"],
     "stdatomic.h": ["UNSUPPORTED: no-threads"],
     "stop_token": ["UNSUPPORTED: no-threads"],
     "thread": ["UNSUPPORTED: no-threads"],

From e8b3ba235597de4587379f233d57826fda71855e Mon Sep 17 00:00:00 2001
From: Nico Weber <thakis@chromium.org>
Date: Mon, 19 Jun 2023 10:58:08 -0400
Subject: [PATCH 041/130] [gn] port 6f2e92c10cebca5 better (lld/unittests)

lld/test/Unit/lit.site.cfg.py.in got cleaned up in the reland.
---
 llvm/utils/gn/secondary/lld/test/BUILD.gn | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/llvm/utils/gn/secondary/lld/test/BUILD.gn b/llvm/utils/gn/secondary/lld/test/BUILD.gn
index 06256e1ba161b..d6c15608124e3 100644
--- a/llvm/utils/gn/secondary/lld/test/BUILD.gn
+++ b/llvm/utils/gn/secondary/lld/test/BUILD.gn
@@ -16,13 +16,7 @@ template("write_lit_cfg") {
           "## Autogenerated from $input, do not edit\n\n" + lit_path_function,
       "LLD_BINARY_DIR=" +
           rebase_path(get_label_info("//lld", "target_out_dir")),
-      "CURRENT_LIBS_DIR=",  # FIXME: for shared builds only (?)
-      "CURRENT_TOOLS_DIR=" + rebase_path("$root_out_dir/bin"),
       "LLD_SOURCE_DIR=" + rebase_path("//lld"),
-      "LLVM_BINARY_DIR=" +
-          rebase_path(get_label_info("//llvm", "target_out_dir")),
-      "LLVM_SOURCE_DIR=" + rebase_path("//llvm"),
-      "LLVM_TOOLS_DIR=" + rebase_path("$root_out_dir/bin"),
     ]
     values += invoker.extra_values
   }
@@ -35,10 +29,16 @@ write_lit_cfg("lit_site_cfg") {
   dir = get_path_info(output, "dir")
 
   extra_values = [
+    "CURRENT_LIBS_DIR=",  # FIXME: for shared builds only (?)
+    "CURRENT_TOOLS_DIR=" + rebase_path("$root_out_dir/bin"),
     "ENABLE_BACKTRACES=1",
+    "LLVM_BINARY_DIR=" +
+        rebase_path(get_label_info("//llvm", "target_out_dir")),
     "LLVM_HOST_TRIPLE=$llvm_current_triple",
     "LLVM_LIBS_DIR=",  # needed only for shared builds
     "LLVM_LIT_TOOLS_DIR=",  # Intentionally empty, matches cmake build.
+    "LLVM_SOURCE_DIR=" + rebase_path("//llvm"),
+    "LLVM_TOOLS_DIR=" + rebase_path("$root_out_dir/bin"),
     "Python3_EXECUTABLE=$python_path",
     "LLVM_TARGET_TRIPLE=$llvm_target_triple",
 
@@ -103,10 +103,7 @@ write_lit_cfg("lit_unit_site_cfg") {
   # Fully-qualified instead of relative for LIT_SITE_CFG_IN_HEADER.
   input = "//lld/test/Unit/lit.site.cfg.py.in"
   output = lld_lit_unit_site_cfg_file
-  extra_values = [
-    "ENABLE_SHARED=0",
-    "LLVM_BUILD_MODE=.",
-  ]
+  extra_values = [ "LLVM_BUILD_MODE=." ]
   dir = get_path_info(output, "dir")
   if (host_os == "win") {
     # See comment for Windows solink in llvm/utils/gn/build/toolchain/BUILD.gn

From a4f0764aefd6ea41e83a5974f582a1a7e985667d Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Sun, 18 Jun 2023 12:35:44 +0200
Subject: [PATCH 042/130] [libc++] Marks __cpp_lib_bitops as implemented.

This FTM was introduced in
  P0553R4 Bit operations

Which has been implemented since libc++ 9.

This was noticed while working on D153192.

Reviewed By: #libc, philnik

Differential Revision: https://reviews.llvm.org/D153225
---
 libcxx/docs/FeatureTestMacroTable.rst         |  2 +-
 libcxx/include/version                        |  2 +-
 .../bit.version.compile.pass.cpp              | 48 ++++++-------------
 .../version.version.compile.pass.cpp          | 48 ++++++-------------
 .../generate_feature_test_macro_components.py |  1 -
 5 files changed, 32 insertions(+), 69 deletions(-)

diff --git a/libcxx/docs/FeatureTestMacroTable.rst b/libcxx/docs/FeatureTestMacroTable.rst
index 45e580f397756..4c8e1ffc828bb 100644
--- a/libcxx/docs/FeatureTestMacroTable.rst
+++ b/libcxx/docs/FeatureTestMacroTable.rst
@@ -192,7 +192,7 @@ Status
     --------------------------------------------------- -----------------
     ``__cpp_lib_bit_cast``                              ``201806L``
     --------------------------------------------------- -----------------
-    ``__cpp_lib_bitops``                                *unimplemented*
+    ``__cpp_lib_bitops``                                ``201907L``
     --------------------------------------------------- -----------------
     ``__cpp_lib_bounded_array_traits``                  ``201902L``
     --------------------------------------------------- -----------------
diff --git a/libcxx/include/version b/libcxx/include/version
index 5f69eff42b0a1..10abf695dadac 100644
--- a/libcxx/include/version
+++ b/libcxx/include/version
@@ -332,7 +332,7 @@ __cpp_lib_within_lifetime                               202306L <type_traits>
 # endif
 # define __cpp_lib_bind_front                           201907L
 # define __cpp_lib_bit_cast                             201806L
-// # define __cpp_lib_bitops                               201907L
+# define __cpp_lib_bitops                               201907L
 # define __cpp_lib_bounded_array_traits                 201902L
 # if !defined(_LIBCPP_HAS_NO_CHAR8_T)
 #   define __cpp_lib_char8_t                            201907L
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/bit.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/bit.version.compile.pass.cpp
index b858ff65545d5..0e1d06b928019 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/bit.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/bit.version.compile.pass.cpp
@@ -101,17 +101,11 @@
 #   error "__cpp_lib_bit_cast should have the value 201806L in c++20"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_bitops
-#     error "__cpp_lib_bitops should be defined in c++20"
-#   endif
-#   if __cpp_lib_bitops != 201907L
-#     error "__cpp_lib_bitops should have the value 201907L in c++20"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_bitops
-#     error "__cpp_lib_bitops should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_bitops
+#   error "__cpp_lib_bitops should be defined in c++20"
+# endif
+# if __cpp_lib_bitops != 201907L
+#   error "__cpp_lib_bitops should have the value 201907L in c++20"
 # endif
 
 # ifdef __cpp_lib_byteswap
@@ -141,17 +135,11 @@
 #   error "__cpp_lib_bit_cast should have the value 201806L in c++23"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_bitops
-#     error "__cpp_lib_bitops should be defined in c++23"
-#   endif
-#   if __cpp_lib_bitops != 201907L
-#     error "__cpp_lib_bitops should have the value 201907L in c++23"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_bitops
-#     error "__cpp_lib_bitops should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_bitops
+#   error "__cpp_lib_bitops should be defined in c++23"
+# endif
+# if __cpp_lib_bitops != 201907L
+#   error "__cpp_lib_bitops should have the value 201907L in c++23"
 # endif
 
 # ifndef __cpp_lib_byteswap
@@ -184,17 +172,11 @@
 #   error "__cpp_lib_bit_cast should have the value 201806L in c++26"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_bitops
-#     error "__cpp_lib_bitops should be defined in c++26"
-#   endif
-#   if __cpp_lib_bitops != 201907L
-#     error "__cpp_lib_bitops should have the value 201907L in c++26"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_bitops
-#     error "__cpp_lib_bitops should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_bitops
+#   error "__cpp_lib_bitops should be defined in c++26"
+# endif
+# if __cpp_lib_bitops != 201907L
+#   error "__cpp_lib_bitops should have the value 201907L in c++26"
 # endif
 
 # ifndef __cpp_lib_byteswap
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
index dbad0e6b3cf0c..fb7081f2ebbf7 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
@@ -2846,17 +2846,11 @@
 #   error "__cpp_lib_bit_cast should have the value 201806L in c++20"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_bitops
-#     error "__cpp_lib_bitops should be defined in c++20"
-#   endif
-#   if __cpp_lib_bitops != 201907L
-#     error "__cpp_lib_bitops should have the value 201907L in c++20"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_bitops
-#     error "__cpp_lib_bitops should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_bitops
+#   error "__cpp_lib_bitops should be defined in c++20"
+# endif
+# if __cpp_lib_bitops != 201907L
+#   error "__cpp_lib_bitops should have the value 201907L in c++20"
 # endif
 
 # ifdef __cpp_lib_bitset
@@ -4127,17 +4121,11 @@
 #   error "__cpp_lib_bit_cast should have the value 201806L in c++23"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_bitops
-#     error "__cpp_lib_bitops should be defined in c++23"
-#   endif
-#   if __cpp_lib_bitops != 201907L
-#     error "__cpp_lib_bitops should have the value 201907L in c++23"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_bitops
-#     error "__cpp_lib_bitops should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_bitops
+#   error "__cpp_lib_bitops should be defined in c++23"
+# endif
+# if __cpp_lib_bitops != 201907L
+#   error "__cpp_lib_bitops should have the value 201907L in c++23"
 # endif
 
 # ifdef __cpp_lib_bitset
@@ -5615,17 +5603,11 @@
 #   error "__cpp_lib_bit_cast should have the value 201806L in c++26"
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_bitops
-#     error "__cpp_lib_bitops should be defined in c++26"
-#   endif
-#   if __cpp_lib_bitops != 201907L
-#     error "__cpp_lib_bitops should have the value 201907L in c++26"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_bitops
-#     error "__cpp_lib_bitops should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_bitops
+#   error "__cpp_lib_bitops should be defined in c++26"
+# endif
+# if __cpp_lib_bitops != 201907L
+#   error "__cpp_lib_bitops should have the value 201907L in c++26"
 # endif
 
 # if !defined(_LIBCPP_VERSION)
diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py
index 72ae5bdd653a8..4d454f7e42986 100755
--- a/libcxx/utils/generate_feature_test_macro_components.py
+++ b/libcxx/utils/generate_feature_test_macro_components.py
@@ -226,7 +226,6 @@ def add_version_header(tc):
             "name": "__cpp_lib_bitops",
             "values": {"c++20": 201907},
             "headers": ["bit"],
-            "unimplemented": True,
         },
         {
             "name": "__cpp_lib_bitset",

From cdcfc5ec9b33da441218041b6cd4f4ad19620eca Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Tue, 16 Aug 2022 08:15:51 +0200
Subject: [PATCH 043/130] [libc++][regex] Removes operator!=.

Implements part of:
- P1614R2 The Mothership has Landed

Reviewed By: #libc, H-G-Hristov, philnik

Differential Revision: https://reviews.llvm.org/D153222
---
 libcxx/docs/Status/SpaceshipProjects.csv             |  8 ++++----
 libcxx/include/regex                                 | 12 +++++++++---
 libcxx/modules/std/regex.cppm                        |  3 ---
 .../re.regiter.comp/tested_elsewhere.pass.cpp        |  2 +-
 .../re.tokiter/re.tokiter.comp/equal.pass.cpp        |  2 +-
 .../re.results/re.results.nonmember/equal.pass.cpp   |  2 +-
 6 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/libcxx/docs/Status/SpaceshipProjects.csv b/libcxx/docs/Status/SpaceshipProjects.csv
index 86ce5d9387a3f..17f9f92d67440 100644
--- a/libcxx/docs/Status/SpaceshipProjects.csv
+++ b/libcxx/docs/Status/SpaceshipProjects.csv
@@ -188,13 +188,13 @@ Section,Description,Dependencies,Assignee,Complete
 "| `[fs.class.directory.entry] <https://wg21.link/fs.class.directory.entry>`_
 | `[fs.dir.entry.obs] <https://wg21.link/fs.dir.entry.obs>`_",| `filesystem::directory_entry <https://reviews.llvm.org/D130860>`_,None,Adrian Vogelsgesang,|Complete|
 - `5.15 Clause 30: Regular expressions library <https://wg21.link/p1614r2#clause-30-regular-expressions-library>`_,,,,
-| `[re.syn] <https://wg21.link/re.syn>`_,|,None,Mark de Wever,|In Progress|
+| `[re.syn] <https://wg21.link/re.syn>`_,|,None,Mark de Wever,|Complete|
 | `[re.submatch.op] <https://wg21.link/re.submatch.op>`_,| `sub_match <https://reviews.llvm.org/D132310>`_,None,Mark de Wever,|Complete|
-| `[re.results.nonmember] <https://wg21.link/re.results.nonmember>`_,| remove ops `match_results`,None,Mark de Wever,|In Progress|
+| `[re.results.nonmember] <https://wg21.link/re.results.nonmember>`_,| remove ops `match_results`,None,Mark de Wever,|Complete|
 "| `[re.regiter] <https://wg21.link/re.regiter>`_,
-| `[re.regiter.comp] <https://wg21.link/re.regiter.comp>`_",| remove ops `regex_iterator`,None,Mark de Wever,|In Progress|
+| `[re.regiter.comp] <https://wg21.link/re.regiter.comp>`_",| remove ops `regex_iterator`,None,Mark de Wever,|Complete|
 "| `[re.tokiter] <https://wg21.link/re.tokiter>`_
-| `[re.tokiter.comp] <https://wg21.link/re.tokiter.comp>`_",| remove ops `regex_token_iterator`,None,Mark de Wever,|In Progress|
+| `[re.tokiter.comp] <https://wg21.link/re.tokiter.comp>`_",| remove ops `regex_token_iterator`,None,Mark de Wever,|Complete|
 - `5.16 Clause 31: Atomic operations library <https://wg21.link/p1614r2#clause-31-atomic-operations-library>`_,,,,
 - `5.17 Clause 32: Thread support library <https://wg21.link/p1614r2#clause-32-thread-support-library>`_,,,,
 | `[thread.thread.id] <https://wg21.link/thread.thread.id>`_,| `thread::id <https://reviews.llvm.org/D131362>`_,None,Adrian Vogelsgesang,|Complete|
diff --git a/libcxx/include/regex b/libcxx/include/regex
index b4545580163ba..ea2711fddafca 100644
--- a/libcxx/include/regex
+++ b/libcxx/include/regex
@@ -543,7 +543,7 @@ template <class BidirectionalIterator, class Allocator>
     operator==(const match_results<BidirectionalIterator, Allocator>& m1,
                const match_results<BidirectionalIterator, Allocator>& m2);
 
-template <class BidirectionalIterator, class Allocator>
+template <class BidirectionalIterator, class Allocator>                    // Removed in C++20
     bool
     operator!=(const match_results<BidirectionalIterator, Allocator>& m1,
                const match_results<BidirectionalIterator, Allocator>& m2);
@@ -710,7 +710,7 @@ public:
     regex_iterator& operator=(const regex_iterator&);
 
     bool operator==(const regex_iterator&) const;
-    bool operator!=(const regex_iterator&) const;
+    bool operator!=(const regex_iterator&) const;                                // Removed in C++20
 
     const value_type& operator*() const;
     const value_type* operator->() const;
@@ -768,7 +768,7 @@ public:
     regex_token_iterator& operator=(const regex_token_iterator&);
 
     bool operator==(const regex_token_iterator&) const;
-    bool operator!=(const regex_token_iterator&) const;
+    bool operator!=(const regex_token_iterator&) const;                  // Removed in C++20
 
     const value_type& operator*() const;
     const value_type* operator->() const;
@@ -5816,6 +5816,7 @@ operator==(const match_results<_BidirectionalIterator, _Allocator>& __x,
            __x.__suffix_ == __y.__suffix_;
 }
 
+#if _LIBCPP_STD_VER < 20
 template <class _BidirectionalIterator, class _Allocator>
 inline _LIBCPP_INLINE_VISIBILITY
 bool
@@ -5824,6 +5825,7 @@ operator!=(const match_results<_BidirectionalIterator, _Allocator>& __x,
 {
     return !(__x == __y);
 }
+#endif
 
 template <class _BidirectionalIterator, class _Allocator>
 inline _LIBCPP_INLINE_VISIBILITY
@@ -6427,8 +6429,10 @@ public:
 #endif
 
     bool operator==(const regex_iterator& __x) const;
+#if _LIBCPP_STD_VER < 20
     _LIBCPP_INLINE_VISIBILITY
     bool operator!=(const regex_iterator& __x) const {return !(*this == __x);}
+#endif
 
     _LIBCPP_INLINE_VISIBILITY
     reference operator*() const {return  __match_;}
@@ -6606,7 +6610,9 @@ public:
 
     bool operator==(const regex_token_iterator& __x) const;
     _LIBCPP_INLINE_VISIBILITY
+#if _LIBCPP_STD_VER < 20
     bool operator!=(const regex_token_iterator& __x) const {return !(*this == __x);}
+#endif
 
     _LIBCPP_INLINE_VISIBILITY
     const value_type& operator*() const {return *__result_;}
diff --git a/libcxx/modules/std/regex.cppm b/libcxx/modules/std/regex.cppm
index 222ebf94a78a8..7c23e6309c501 100644
--- a/libcxx/modules/std/regex.cppm
+++ b/libcxx/modules/std/regex.cppm
@@ -64,9 +64,6 @@ export namespace std {
   // [re.submatch.op], sub_match non-member operators
   using std::operator==;
   using std::operator<=>;
-#  if 1 // P1614
-  using std::operator!=;
-#  endif
 
   using std::operator<<;
 
diff --git a/libcxx/test/std/re/re.iter/re.regiter/re.regiter.comp/tested_elsewhere.pass.cpp b/libcxx/test/std/re/re.iter/re.regiter/re.regiter.comp/tested_elsewhere.pass.cpp
index f7f71bc8b7c5a..80f6aa3c75051 100644
--- a/libcxx/test/std/re/re.iter/re.regiter/re.regiter.comp/tested_elsewhere.pass.cpp
+++ b/libcxx/test/std/re/re.iter/re.regiter/re.regiter.comp/tested_elsewhere.pass.cpp
@@ -11,7 +11,7 @@
 // class regex_iterator<BidirectionalIterator, charT, traits>
 
 // bool operator==(const regex_iterator& right) const;
-// bool operator!=(const regex_iterator& right) const;
+// bool operator!=(const regex_iterator& right) const; // generated by the compiler in C++20
 
 int main(int, char**)
 {
diff --git a/libcxx/test/std/re/re.iter/re.tokiter/re.tokiter.comp/equal.pass.cpp b/libcxx/test/std/re/re.iter/re.tokiter/re.tokiter.comp/equal.pass.cpp
index 1c602010319e8..7ccb700c1d182 100644
--- a/libcxx/test/std/re/re.iter/re.tokiter/re.tokiter.comp/equal.pass.cpp
+++ b/libcxx/test/std/re/re.iter/re.tokiter/re.tokiter.comp/equal.pass.cpp
@@ -11,7 +11,7 @@
 // class regex_token_iterator<BidirectionalIterator, charT, traits>
 
 // bool operator==(const regex_token_iterator& right) const;
-// bool operator!=(const regex_token_iterator& right) const;
+// bool operator!=(const regex_token_iterator& right) const; // generated by the compiler in C++20
 
 #include <regex>
 #include <cassert>
diff --git a/libcxx/test/std/re/re.results/re.results.nonmember/equal.pass.cpp b/libcxx/test/std/re/re.results/re.results.nonmember/equal.pass.cpp
index 3723ad4740f47..1f01e95861fbc 100644
--- a/libcxx/test/std/re/re.results/re.results.nonmember/equal.pass.cpp
+++ b/libcxx/test/std/re/re.results/re.results.nonmember/equal.pass.cpp
@@ -15,7 +15,7 @@
 //    operator==(const match_results<BidirectionalIterator, Allocator>& m1,
 //               const match_results<BidirectionalIterator, Allocator>& m2);
 
-// template <class BidirectionalIterator, class Allocator>
+// template <class BidirectionalIterator, class Allocator> // generated by the compiler in C++20
 //    bool
 //    operator!=(const match_results<BidirectionalIterator, Allocator>& m1,
 //               const match_results<BidirectionalIterator, Allocator>& m2);

From f805c799bfceae68251ff19a21f60f7335bfaac5 Mon Sep 17 00:00:00 2001
From: Mark de Wever <koraq@xs4all.nl>
Date: Sat, 17 Jun 2023 15:50:50 +0200
Subject: [PATCH 044/130] [libc++] "Implements" new SI prefixis.

Like yocto, zepto, zetta, and yotta. The new prefixes quecto, ronto,
ronna, and quetta can't be implemented in a intmax_t. So their
implementation does nothing.

Implements
- P2734R0 Adding the new SI prefixes

Depends on D153192

Reviewed By: #libc, philnik

Differential Revision: https://reviews.llvm.org/D153200
---
 libcxx/docs/FeatureTestMacroTable.rst            |  2 +-
 libcxx/docs/Status/Cxx2cPapers.csv               |  2 +-
 libcxx/include/ratio                             |  4 ++++
 libcxx/include/version                           |  2 +-
 .../ratio.version.compile.pass.cpp               | 16 +++++-----------
 .../version.version.compile.pass.cpp             | 16 +++++-----------
 .../generate_feature_test_macro_components.py    |  1 -
 7 files changed, 17 insertions(+), 26 deletions(-)

diff --git a/libcxx/docs/FeatureTestMacroTable.rst b/libcxx/docs/FeatureTestMacroTable.rst
index 4c8e1ffc828bb..9fa449557ec06 100644
--- a/libcxx/docs/FeatureTestMacroTable.rst
+++ b/libcxx/docs/FeatureTestMacroTable.rst
@@ -394,7 +394,7 @@ Status
     --------------------------------------------------- -----------------
     ``__cpp_lib_hazard_pointer``                        *unimplemented*
     --------------------------------------------------- -----------------
-    ``__cpp_lib_ratio``                                 *unimplemented*
+    ``__cpp_lib_ratio``                                 ``202306L``
     --------------------------------------------------- -----------------
     ``__cpp_lib_rcu``                                   *unimplemented*
     --------------------------------------------------- -----------------
diff --git a/libcxx/docs/Status/Cxx2cPapers.csv b/libcxx/docs/Status/Cxx2cPapers.csv
index 494e70f22a356..cd2e93377b20e 100644
--- a/libcxx/docs/Status/Cxx2cPapers.csv
+++ b/libcxx/docs/Status/Cxx2cPapers.csv
@@ -22,7 +22,7 @@
 "`P1759R6 <https://wg21.link/P1759R6>`__","LWG","Native handles and file streams","Varna June 2023","","",""
 "`P2697R1 <https://wg21.link/P2697R1>`__","LWG","Interfacing ``bitset`` with ``string_view``","Varna June 2023","","",""
 "`P1383R2 <https://wg21.link/P1383R2>`__","LWG","More ``constexpr`` for ``<cmath>`` and ``<complex>``","Varna June 2023","","",""
-"`P2734R0 <https://wg21.link/P2734R0>`__","LWG","Adding the new SI prefixes","Varna June 2023","","",""
+"`P2734R0 <https://wg21.link/P2734R0>`__","LWG","Adding the new SI prefixes","Varna June 2023","|Complete|","Clang 17",""
 "`P2548R6 <https://wg21.link/P2548R6>`__","LWG","``copyable_function``","Varna June 2023","","",""
 "`P2714R1 <https://wg21.link/P2714R1>`__","LWG","Bind front and back to NTTP callables","Varna June 2023","","",""
 "`P2630R4 <https://wg21.link/P2630R4>`__","LWG","``submdspan``","Varna June 2023","","",""
diff --git a/libcxx/include/ratio b/libcxx/include/ratio
index 327e08cf1b4f1..c9637ab818cda 100644
--- a/libcxx/include/ratio
+++ b/libcxx/include/ratio
@@ -40,6 +40,8 @@ template <class R1, class R2> struct ratio_greater;
 template <class R1, class R2> struct ratio_greater_equal;
 
 // convenience SI typedefs
+using quecto = ratio <1, 1'000'000'000'000'000'000'000'000'000'000>; // Since C++26; not supported
+using ronto = ratio <1, 1'000'000'000'000'000'000'000'000'000>;      // Since C++26; not supported
 typedef ratio<1, 1000000000000000000000000> yocto;  // not supported
 typedef ratio<1,    1000000000000000000000> zepto;  // not supported
 typedef ratio<1,       1000000000000000000> atto;
@@ -60,6 +62,8 @@ typedef ratio<         1000000000000000, 1> peta;
 typedef ratio<      1000000000000000000, 1> exa;
 typedef ratio<   1000000000000000000000, 1> zetta;  // not supported
 typedef ratio<1000000000000000000000000, 1> yotta;  // not supported
+using ronna = ratio <1'000'000'000'000'000'000'000'000'000, 1>;      // Since C++26; not supported
+using quetta = ratio <1'000'000'000'000'000'000'000'000'000'000, 1>; // Since C++26; not supported
 
   // 20.11.5, ratio comparison
   template <class R1, class R2> inline constexpr bool ratio_equal_v
diff --git a/libcxx/include/version b/libcxx/include/version
index 10abf695dadac..0cdf06d6d370c 100644
--- a/libcxx/include/version
+++ b/libcxx/include/version
@@ -456,7 +456,7 @@ __cpp_lib_within_lifetime                               202306L <type_traits>
 // # define __cpp_lib_fstream_native_handle                202306L
 // # define __cpp_lib_function_ref                         202306L
 // # define __cpp_lib_hazard_pointer                       202306L
-// # define __cpp_lib_ratio                                202306L
+# define __cpp_lib_ratio                                202306L
 // # define __cpp_lib_rcu                                  202306L
 // # define __cpp_lib_smart_ptr_owner_equality             202306L
 // # define __cpp_lib_sstream_from_string_view             202306L
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/ratio.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/ratio.version.compile.pass.cpp
index f782c24898b43..38adba62079d7 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/ratio.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/ratio.version.compile.pass.cpp
@@ -54,17 +54,11 @@
 
 #elif TEST_STD_VER > 23
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_ratio
-#     error "__cpp_lib_ratio should be defined in c++26"
-#   endif
-#   if __cpp_lib_ratio != 202306L
-#     error "__cpp_lib_ratio should have the value 202306L in c++26"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_ratio
-#     error "__cpp_lib_ratio should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_ratio
+#   error "__cpp_lib_ratio should be defined in c++26"
+# endif
+# if __cpp_lib_ratio != 202306L
+#   error "__cpp_lib_ratio should have the value 202306L in c++26"
 # endif
 
 #endif // TEST_STD_VER > 23
diff --git a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
index fb7081f2ebbf7..040dc9c6cd9e7 100644
--- a/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
+++ b/libcxx/test/std/language.support/support.limits/support.limits.general/version.version.compile.pass.cpp
@@ -6511,17 +6511,11 @@
 #   endif
 # endif
 
-# if !defined(_LIBCPP_VERSION)
-#   ifndef __cpp_lib_ratio
-#     error "__cpp_lib_ratio should be defined in c++26"
-#   endif
-#   if __cpp_lib_ratio != 202306L
-#     error "__cpp_lib_ratio should have the value 202306L in c++26"
-#   endif
-# else // _LIBCPP_VERSION
-#   ifdef __cpp_lib_ratio
-#     error "__cpp_lib_ratio should not be defined because it is unimplemented in libc++!"
-#   endif
+# ifndef __cpp_lib_ratio
+#   error "__cpp_lib_ratio should be defined in c++26"
+# endif
+# if __cpp_lib_ratio != 202306L
+#   error "__cpp_lib_ratio should have the value 202306L in c++26"
 # endif
 
 # ifndef __cpp_lib_raw_memory_algorithms
diff --git a/libcxx/utils/generate_feature_test_macro_components.py b/libcxx/utils/generate_feature_test_macro_components.py
index 4d454f7e42986..4b835791aee53 100755
--- a/libcxx/utils/generate_feature_test_macro_components.py
+++ b/libcxx/utils/generate_feature_test_macro_components.py
@@ -847,7 +847,6 @@ def add_version_header(tc):
             "name": "__cpp_lib_ratio",
             "values": {"c++26": 202306}, # P2734R0 Adding the new SI prefixes
             "headers": ["ratio"],
-            "unimplemented": True,
         },
         {
             "name": "__cpp_lib_raw_memory_algorithms",

From b4bb6211a5e7679092086324a3bd398bb8fee1f4 Mon Sep 17 00:00:00 2001
From: Job Noorman <jnoorman@igalia.com>
Date: Mon, 19 Jun 2023 16:51:43 +0200
Subject: [PATCH 045/130] [BOLT] Implement composed relocations

BOLT currently assumes (and asserts) that no two relocations can share
the same offset. Although this is true in most cases, ELF has a feature
called (not sure if this is an official term) composed relocations [1]
where multiple relocations at the same offset are combined to produce a
single value.

For example, to support label subtraction (a - b) on RISC-V, two
relocations are emitted at the same offset:
- R_RISCV_ADD32 a + 0
- R_RISCV_SUB32 b + 0
which, when combined, will produce the value of (a - b).

To support this in BOLT, first, RelocationSetType in BinarySection is
changed to be a multiset in order to allow it to store multiple
relocations at the same offset.

Next, Relocation::emit() is changed to receive an iterator pair of
relocations. In most cases, these will point to a single relocation in
which case its behavior is unaltered by this patch. For composed
relocations, they should point to all relocations at the same offset and
the following happens:
- A new method Relocation::createExpr() is called for every relocation.
  This method is essentially the same as the original emit() except that
  it returns the MCExpr without emitting it.
- The MCExprs of relocations i and i+1 are combined using the opcode
  returned by the new method Relocation::getComposeOpcodeFor().
- After combining all MCExprs, the last one is emitted.

Note that in the current patch, getComposeOpcodeFor() simply calls
llvm_unreachable() since none of the current targets use composed
relocations. This will change once the RISC-V target lands.

Finally, BinarySection::emitAsData() is updated to group relocations by
offset and emit them all at once.

Note that this means composed relocations are only supported in data
sections. Since this is the only place they seem to be used in RISC-V, I
believe it's reasonable to only support them there for now to avoid
further code complexity.

[1]: https://www.sco.com/developers/gabi/latest/ch4.reloc.html

Reviewed By: rafauler

Differential Revision: https://reviews.llvm.org/D146546
---
 bolt/include/bolt/Core/BinarySection.h |  5 ++-
 bolt/include/bolt/Core/Relocation.h    | 31 ++++++++++++++-
 bolt/lib/Core/BinarySection.cpp        | 53 +++++++++++++++++--------
 bolt/lib/Core/Relocation.cpp           | 55 +++++++++++++++-----------
 4 files changed, 101 insertions(+), 43 deletions(-)

diff --git a/bolt/include/bolt/Core/BinarySection.h b/bolt/include/bolt/Core/BinarySection.h
index 6890ced83b13b..078b33bc9c5ab 100644
--- a/bolt/include/bolt/Core/BinarySection.h
+++ b/bolt/include/bolt/Core/BinarySection.h
@@ -59,7 +59,7 @@ class BinarySection {
 
   // Relocations associated with this section. Relocation offsets are
   // wrt. to the original section address and size.
-  using RelocationSetType = std::set<Relocation, std::less<>>;
+  using RelocationSetType = std::multiset<Relocation, std::less<>>;
   RelocationSetType Relocations;
 
   // Dynamic relocations associated with this section. Relocation offsets are
@@ -345,7 +345,8 @@ class BinarySection {
   bool removeRelocationAt(uint64_t Offset) {
     auto Itr = Relocations.find(Offset);
     if (Itr != Relocations.end()) {
-      Relocations.erase(Itr);
+      auto End = Relocations.upper_bound(Offset);
+      Relocations.erase(Itr, End);
       return true;
     }
     return false;
diff --git a/bolt/include/bolt/Core/Relocation.h b/bolt/include/bolt/Core/Relocation.h
index 1296c001db57c..5ae288a91986e 100644
--- a/bolt/include/bolt/Core/Relocation.h
+++ b/bolt/include/bolt/Core/Relocation.h
@@ -14,10 +14,11 @@
 #ifndef BOLT_CORE_RELOCATION_H
 #define BOLT_CORE_RELOCATION_H
 
+#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCStreamer.h"
 #include "llvm/TargetParser/Triple.h"
 
 namespace llvm {
-class MCStreamer;
 class MCSymbol;
 class raw_ostream;
 
@@ -122,8 +123,36 @@ struct Relocation {
   /// responsible for setting the position correctly.
   size_t emit(MCStreamer *Streamer) const;
 
+  /// Emit a group of composed relocations. All relocations must have the same
+  /// offset. If std::distance(Begin, End) == 1, this is equivalent to
+  /// Begin->emit(Streamer).
+  template <typename RelocIt>
+  static size_t emit(RelocIt Begin, RelocIt End, MCStreamer *Streamer) {
+    if (Begin == End)
+      return 0;
+
+    const MCExpr *Value = nullptr;
+
+    for (auto RI = Begin; RI != End; ++RI) {
+      assert(RI->Offset == Begin->Offset &&
+             "emitting composed relocations with different offsets");
+      Value = RI->createExpr(Streamer, Value);
+    }
+
+    assert(Value && "failed to create relocation value");
+    auto Size = std::prev(End)->getSize();
+    Streamer->emitValue(Value, Size);
+    return Size;
+  }
+
   /// Print a relocation to \p OS.
   void print(raw_ostream &OS) const;
+
+private:
+  const MCExpr *createExpr(MCStreamer *Streamer) const;
+  const MCExpr *createExpr(MCStreamer *Streamer,
+                           const MCExpr *RetainedValue) const;
+  static MCBinaryExpr::Opcode getComposeOpcodeFor(uint64_t Type);
 };
 
 /// Relocation ordering by offset.
diff --git a/bolt/lib/Core/BinarySection.cpp b/bolt/lib/Core/BinarySection.cpp
index e0e7b15d04a57..e3a1e79b98b15 100644
--- a/bolt/lib/Core/BinarySection.cpp
+++ b/bolt/lib/Core/BinarySection.cpp
@@ -90,23 +90,44 @@ void BinarySection::emitAsData(MCStreamer &Streamer,
     Streamer.emitBytes(SectionContents);
   } else {
     uint64_t SectionOffset = 0;
-    for (const Relocation &Relocation : relocations()) {
-      assert(Relocation.Offset < SectionContents.size() && "overflow detected");
+    for (auto RI = Relocations.begin(), RE = Relocations.end(); RI != RE;) {
+      auto RelocationOffset = RI->Offset;
+      assert(RelocationOffset < SectionContents.size() && "overflow detected");
+
+      if (SectionOffset < RelocationOffset) {
+        Streamer.emitBytes(SectionContents.substr(
+            SectionOffset, RelocationOffset - SectionOffset));
+        SectionOffset = RelocationOffset;
+      }
+
+      // Get iterators to all relocations with the same offset. Usually, there
+      // is only one such relocation but there can be more for composed
+      // relocations.
+      auto ROI = RI;
+      auto ROE = Relocations.upper_bound(RelocationOffset);
+
+      // Start from the next offset on the next iteration.
+      RI = ROE;
+
       // Skip undefined symbols.
-      if (BC.UndefinedSymbols.count(Relocation.Symbol))
+      auto HasUndefSym = [this](const auto &Relocation) {
+        return BC.UndefinedSymbols.count(Relocation.Symbol);
+      };
+
+      if (std::any_of(ROI, ROE, HasUndefSym))
         continue;
-      if (SectionOffset < Relocation.Offset) {
-        Streamer.emitBytes(SectionContents.substr(
-            SectionOffset, Relocation.Offset - SectionOffset));
-        SectionOffset = Relocation.Offset;
+
+      for (const auto &Relocation : make_range(ROI, ROE)) {
+        LLVM_DEBUG(
+            dbgs() << "BOLT-DEBUG: emitting relocation for symbol "
+                   << (Relocation.Symbol ? Relocation.Symbol->getName()
+                                         : StringRef("<none>"))
+                   << " at offset 0x" << Twine::utohexstr(Relocation.Offset)
+                   << " with size "
+                   << Relocation::getSizeForType(Relocation.Type) << '\n');
       }
-      LLVM_DEBUG(dbgs() << "BOLT-DEBUG: emitting relocation for symbol "
-                        << (Relocation.Symbol ? Relocation.Symbol->getName()
-                                              : StringRef("<none>"))
-                        << " at offset 0x"
-                        << Twine::utohexstr(Relocation.Offset) << " with size "
-                        << Relocation::getSizeForType(Relocation.Type) << '\n');
-      size_t RelocationSize = Relocation.emit(&Streamer);
+
+      size_t RelocationSize = Relocation::emit(ROI, ROE, &Streamer);
       SectionOffset += RelocationSize;
     }
     assert(SectionOffset <= SectionContents.size() && "overflow error");
@@ -221,9 +242,7 @@ BinarySection::reorderRelocations(bool Inplace) const {
     assert(NewRel.Offset < getSize());
     LLVM_DEBUG(dbgs() << "BOLT-DEBUG: moving " << Rel << " -> " << NewRel
                       << "\n");
-    auto Res = NewRelocations.emplace(std::move(NewRel));
-    (void)Res;
-    assert(Res.second && "Can't overwrite existing relocation");
+    NewRelocations.emplace(std::move(NewRel));
   }
   return NewRelocations;
 }
diff --git a/bolt/lib/Core/Relocation.cpp b/bolt/lib/Core/Relocation.cpp
index 0901a1465193c..b02ca61964a74 100644
--- a/bolt/lib/Core/Relocation.cpp
+++ b/bolt/lib/Core/Relocation.cpp
@@ -817,39 +817,48 @@ uint64_t Relocation::getRelative() {
 
 size_t Relocation::emit(MCStreamer *Streamer) const {
   const size_t Size = getSizeForType(Type);
+  const auto *Value = createExpr(Streamer);
+  Streamer->emitValue(Value, Size);
+  return Size;
+}
+
+const MCExpr *Relocation::createExpr(MCStreamer *Streamer) const {
   MCContext &Ctx = Streamer->getContext();
+  const MCExpr *Value = nullptr;
+
+  if (Symbol && Addend) {
+    Value = MCBinaryExpr::createAdd(MCSymbolRefExpr::create(Symbol, Ctx),
+                                    MCConstantExpr::create(Addend, Ctx), Ctx);
+  } else if (Symbol) {
+    Value = MCSymbolRefExpr::create(Symbol, Ctx);
+  } else {
+    Value = MCConstantExpr::create(Addend, Ctx);
+  }
+
   if (isPCRelative(Type)) {
     MCSymbol *TempLabel = Ctx.createNamedTempSymbol();
     Streamer->emitLabel(TempLabel);
-    const MCExpr *Value = nullptr;
-    if (Symbol) {
-      Value = MCSymbolRefExpr::create(Symbol, Ctx);
-      if (Addend) {
-        Value = MCBinaryExpr::createAdd(
-            Value, MCConstantExpr::create(Addend, Ctx), Ctx);
-      }
-    } else {
-      Value = MCConstantExpr::create(Addend, Ctx);
-    }
     Value = MCBinaryExpr::createSub(
         Value, MCSymbolRefExpr::create(TempLabel, Ctx), Ctx);
-    Streamer->emitValue(Value, Size);
-
-    return Size;
   }
 
-  if (Symbol && Addend) {
-    auto Value =
-        MCBinaryExpr::createAdd(MCSymbolRefExpr::create(Symbol, Ctx),
-                                MCConstantExpr::create(Addend, Ctx), Ctx);
-    Streamer->emitValue(Value, Size);
-  } else if (Symbol) {
-    Streamer->emitSymbolValue(Symbol, Size);
-  } else {
-    Streamer->emitIntValue(Addend, Size);
+  return Value;
+}
+
+const MCExpr *Relocation::createExpr(MCStreamer *Streamer,
+                                     const MCExpr *RetainedValue) const {
+  const auto *Value = createExpr(Streamer);
+
+  if (RetainedValue) {
+    Value = MCBinaryExpr::create(getComposeOpcodeFor(Type), RetainedValue,
+                                 Value, Streamer->getContext());
   }
 
-  return Size;
+  return Value;
+}
+
+MCBinaryExpr::Opcode Relocation::getComposeOpcodeFor(uint64_t Type) {
+  llvm_unreachable("not implemented");
 }
 
 #define ELF_RELOC(name, value) #name,

From bb4f4a3efeae2bc0785edc19d2347f21729c816c Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Mon, 19 Jun 2023 17:04:49 +0200
Subject: [PATCH 046/130] [CVP] Add additional tests for PR63330 (NFC)

---
 .../udiv-expansion.ll                         | 18 ++++++++++++++++++
 .../urem-expansion.ll                         | 19 +++++++++++++++++++
 2 files changed, 37 insertions(+)

diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/udiv-expansion.ll b/llvm/test/Transforms/CorrelatedValuePropagation/udiv-expansion.ll
index 7284f280fa764..8b796f79e60d0 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/udiv-expansion.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/udiv-expansion.ll
@@ -298,3 +298,21 @@ define i8 @large.divisor.with.overflow.v2.unbound.x(i8 %x) {
   %div = udiv i8 %x, 128
   ret i8 %div
 }
+
+define i8 @known_uge(i8 noundef %x) {
+; CHECK-LABEL: @known_uge(
+; CHECK-NEXT:    [[CMP_X_UPPER:%.*]] = icmp ult i8 [[X:%.*]], 6
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP_X_UPPER]])
+; CHECK-NEXT:    [[CMP_X_LOWER:%.*]] = icmp uge i8 [[X]], 3
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP_X_LOWER]])
+; CHECK-NEXT:    [[DIV_CMP:%.*]] = icmp uge i8 [[X]], 3
+; CHECK-NEXT:    [[DIV:%.*]] = zext i1 [[DIV_CMP]] to i8
+; CHECK-NEXT:    ret i8 [[DIV]]
+;
+  %cmp.x.upper = icmp ult i8 %x, 6
+  call void @llvm.assume(i1 %cmp.x.upper)
+  %cmp.x.lower = icmp uge i8 %x, 3
+  call void @llvm.assume(i1 %cmp.x.lower)
+  %div = udiv i8 %x, 3
+  ret i8 %div
+}
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll b/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll
index 2f1ca2483f67d..da38e5a269e6d 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll
@@ -342,3 +342,22 @@ define i1 @icmp_after_expansion(i8 noundef %x) {
   %cmp = icmp eq i8 %rem, 3
   ret i1 %cmp
 }
+
+define i8 @known_uge(i8 noundef %x) {
+; CHECK-LABEL: @known_uge(
+; CHECK-NEXT:    [[CMP_X_UPPER:%.*]] = icmp ult i8 [[X:%.*]], 6
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP_X_UPPER]])
+; CHECK-NEXT:    [[CMP_X_LOWER:%.*]] = icmp uge i8 [[X]], 3
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP_X_LOWER]])
+; CHECK-NEXT:    [[REM_UREM:%.*]] = sub nuw i8 [[X]], 3
+; CHECK-NEXT:    [[REM_CMP:%.*]] = icmp ult i8 [[X]], 3
+; CHECK-NEXT:    [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X]], i8 [[REM_UREM]]
+; CHECK-NEXT:    ret i8 [[REM]]
+;
+  %cmp.x.upper = icmp ult i8 %x, 6
+  call void @llvm.assume(i1 %cmp.x.upper)
+  %cmp.x.lower = icmp uge i8 %x, 3
+  call void @llvm.assume(i1 %cmp.x.lower)
+  %rem = urem i8 %x, 3
+  ret i8 %rem
+}

From 7cfc82f331182855e2634c21d242c3b418f84bd1 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Mon, 19 Jun 2023 17:14:37 +0200
Subject: [PATCH 047/130] [CVP] Use simpler urem expansion when LHS >= RHS
 (PR63330)

In this case we don't need to emit the comparison and select.

This is papering over a weakness in CVP in that newly added
instructions don't get revisited. If they were revisited, the
icmp would be folded at that point.

However, even without that it makes sense to handle this explicitly,
because it avoids the need to insert freeze, which may prevent
further analysis of the operation by LVI.

Proofs: https://alive2.llvm.org/ce/z/quyBxp

Fixes https://github.com/llvm/llvm-project/issues/63330.
---
 llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp | 8 +++++++-
 .../CorrelatedValuePropagation/udiv-expansion.ll          | 4 +---
 .../CorrelatedValuePropagation/urem-expansion.ll          | 4 +---
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
index 488297110d6fa..49d11b790c7ff 100644
--- a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
+++ b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
@@ -763,7 +763,13 @@ static bool expandUDivOrURem(BinaryOperator *Instr, const ConstantRange &XCR,
 
   IRBuilder<> B(Instr);
   Value *ExpandedOp;
-  if (IsRem) {
+  if (XCR.icmp(ICmpInst::ICMP_UGE, YCR)) {
+    // If X is between Y and 2*Y the result is known.
+    if (IsRem)
+      ExpandedOp = B.CreateNUWSub(X, Y);
+    else
+      ExpandedOp = ConstantInt::get(Instr->getType(), 1);
+  } else if (IsRem) {
     // NOTE: this transformation introduces two uses of X,
     //       but it may be undef so we must freeze it first.
     Value *FrozenX = X;
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/udiv-expansion.ll b/llvm/test/Transforms/CorrelatedValuePropagation/udiv-expansion.ll
index 8b796f79e60d0..a2a767084fbff 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/udiv-expansion.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/udiv-expansion.ll
@@ -305,9 +305,7 @@ define i8 @known_uge(i8 noundef %x) {
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP_X_UPPER]])
 ; CHECK-NEXT:    [[CMP_X_LOWER:%.*]] = icmp uge i8 [[X]], 3
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP_X_LOWER]])
-; CHECK-NEXT:    [[DIV_CMP:%.*]] = icmp uge i8 [[X]], 3
-; CHECK-NEXT:    [[DIV:%.*]] = zext i1 [[DIV_CMP]] to i8
-; CHECK-NEXT:    ret i8 [[DIV]]
+; CHECK-NEXT:    ret i8 1
 ;
   %cmp.x.upper = icmp ult i8 %x, 6
   call void @llvm.assume(i1 %cmp.x.upper)
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll b/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll
index da38e5a269e6d..cd0ba2f189dc8 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll
@@ -349,9 +349,7 @@ define i8 @known_uge(i8 noundef %x) {
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP_X_UPPER]])
 ; CHECK-NEXT:    [[CMP_X_LOWER:%.*]] = icmp uge i8 [[X]], 3
 ; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP_X_LOWER]])
-; CHECK-NEXT:    [[REM_UREM:%.*]] = sub nuw i8 [[X]], 3
-; CHECK-NEXT:    [[REM_CMP:%.*]] = icmp ult i8 [[X]], 3
-; CHECK-NEXT:    [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X]], i8 [[REM_UREM]]
+; CHECK-NEXT:    [[REM:%.*]] = sub nuw i8 [[X]], 3
 ; CHECK-NEXT:    ret i8 [[REM]]
 ;
   %cmp.x.upper = icmp ult i8 %x, 6

From 923dbb01ea6be1ec919d0b71b34551ae91169bc7 Mon Sep 17 00:00:00 2001
From: Andrea Di Biagio <andrea.dibiagio@sony.com>
Date: Mon, 19 Jun 2023 14:37:22 +0100
Subject: [PATCH 048/130] [llvm-mca][TimelineView] Skip invalid entries when
 printing the json output.

---
 llvm/tools/llvm-mca/Views/TimelineView.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/llvm/tools/llvm-mca/Views/TimelineView.cpp b/llvm/tools/llvm-mca/Views/TimelineView.cpp
index 5c05edbdea686..2eca48aadfd70 100644
--- a/llvm/tools/llvm-mca/Views/TimelineView.cpp
+++ b/llvm/tools/llvm-mca/Views/TimelineView.cpp
@@ -315,6 +315,10 @@ json::Value TimelineView::toJSON() const {
   json::Array TimelineInfo;
 
   for (const TimelineViewEntry &TLE : Timeline) {
+    // Check if the timeline-max-cycles has been reached.
+    if (!TLE.CycleRetired && TLE.CycleExecuted)
+      break;
+
     TimelineInfo.push_back(
         json::Object({{"CycleDispatched", TLE.CycleDispatched},
                       {"CycleReady", TLE.CycleReady},

From 5aa03b648b827128d439f705cd7d57d59673741d Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Fri, 16 Jun 2023 09:49:04 -0400
Subject: [PATCH 049/130] [libc++][NFC] Apply clang-format on large parts of
 the code base

This commit does a pass of clang-format over files in libc++ that
don't require major changes to conform to our style guide, or for
which we're not overly concerned about conflicting with in-flight
patches or hindering the git blame.

This roughly covers:
- benchmarks
- range algorithms
- concepts
- type traits

I did a manual verification of all the changes, and in particular I
applied clang-format on/off annotations in a few places where the
result was less readable after than before. This was not necessary
in a lot of places, however I did find that clang-format had pretty
bad taste when it comes to formatting concepts.

Differential Revision: https://reviews.llvm.org/D153140
---
 libcxx/benchmarks/CartesianBenchmarks.h       |  26 +-
 libcxx/benchmarks/ContainerBenchmarks.h       | 150 +++--
 libcxx/benchmarks/GenerateInput.h             | 143 ++---
 libcxx/benchmarks/Utilities.h                 |  22 +-
 libcxx/benchmarks/VariantBenchmarks.h         |  11 +-
 .../algorithms.partition_point.bench.cpp      |  13 +-
 libcxx/benchmarks/algorithms/common.h         |  53 +-
 .../algorithms/lower_bound.bench.cpp          |   6 +-
 .../benchmarks/algorithms/make_heap.bench.cpp |   9 +-
 .../make_heap_then_sort_heap.bench.cpp        |  12 +-
 .../benchmarks/algorithms/pop_heap.bench.cpp  |  15 +-
 .../benchmarks/algorithms/push_heap.bench.cpp |  14 +-
 .../algorithms/ranges_make_heap.bench.cpp     |   9 +-
 .../ranges_make_heap_then_sort_heap.bench.cpp |  12 +-
 .../algorithms/ranges_pop_heap.bench.cpp      |  15 +-
 .../algorithms/ranges_push_heap.bench.cpp     |  14 +-
 .../algorithms/ranges_sort.bench.cpp          |   9 +-
 .../algorithms/ranges_sort_heap.bench.cpp     |  10 +-
 .../algorithms/ranges_stable_sort.bench.cpp   |   9 +-
 libcxx/benchmarks/algorithms/sort.bench.cpp   |  11 +-
 .../benchmarks/algorithms/sort_heap.bench.cpp |  10 +-
 .../algorithms/stable_sort.bench.cpp          |   9 +-
 libcxx/benchmarks/allocation.bench.cpp        |  60 +-
 libcxx/benchmarks/deque.bench.cpp             |  29 +-
 libcxx/benchmarks/filesystem.bench.cpp        |  72 ++-
 libcxx/benchmarks/format_to.bench.cpp         |  12 +-
 libcxx/benchmarks/format_to_n.bench.cpp       |  12 +-
 libcxx/benchmarks/formatter_float.bench.cpp   |  15 +-
 libcxx/benchmarks/formatter_int.bench.cpp     |   7 +-
 libcxx/benchmarks/function.bench.cpp          |  78 ++-
 libcxx/benchmarks/map.bench.cpp               | 216 ++-----
 libcxx/benchmarks/ordered_set.bench.cpp       |  49 +-
 libcxx/benchmarks/random.bench.cpp            |   2 +-
 libcxx/benchmarks/string.bench.cpp            | 224 +++----
 libcxx/benchmarks/stringstream.bench.cpp      |  24 +-
 .../unordered_set_operations.bench.cpp        | 375 +++++------
 libcxx/benchmarks/vector_operations.bench.cpp |  39 +-
 .../__algorithm/ranges_adjacent_find.h        |  24 +-
 libcxx/include/__algorithm/ranges_all_of.h    |  21 +-
 libcxx/include/__algorithm/ranges_any_of.h    |  21 +-
 .../__algorithm/ranges_binary_search.h        |  23 +-
 libcxx/include/__algorithm/ranges_clamp.h     |  14 +-
 libcxx/include/__algorithm/ranges_copy.h      |  11 +-
 .../__algorithm/ranges_copy_backward.h        |  13 +-
 libcxx/include/__algorithm/ranges_copy_if.h   |  23 +-
 libcxx/include/__algorithm/ranges_copy_n.h    |  15 +-
 libcxx/include/__algorithm/ranges_count.h     |  10 +-
 libcxx/include/__algorithm/ranges_count_if.h  |  22 +-
 libcxx/include/__algorithm/ranges_equal.h     |  60 +-
 .../include/__algorithm/ranges_equal_range.h  |  29 +-
 libcxx/include/__algorithm/ranges_fill.h      |  10 +-
 libcxx/include/__algorithm/ranges_fill_n.h    |   6 +-
 libcxx/include/__algorithm/ranges_find.h      |  10 +-
 libcxx/include/__algorithm/ranges_find_end.h  |  34 +-
 .../__algorithm/ranges_find_first_of.h        |  70 +--
 libcxx/include/__algorithm/ranges_find_if.h   |  21 +-
 .../include/__algorithm/ranges_find_if_not.h  |  17 +-
 libcxx/include/__algorithm/ranges_for_each.h  |  20 +-
 .../include/__algorithm/ranges_for_each_n.h   |  15 +-
 libcxx/include/__algorithm/ranges_generate.h  |  17 +-
 .../include/__algorithm/ranges_generate_n.h   |  10 +-
 libcxx/include/__algorithm/ranges_includes.h  |  30 +-
 .../__algorithm/ranges_inplace_merge.h        |  56 +-
 libcxx/include/__algorithm/ranges_is_heap.h   |  24 +-
 .../__algorithm/ranges_is_heap_until.h        |  25 +-
 .../__algorithm/ranges_is_partitioned.h       |  18 +-
 .../__algorithm/ranges_is_permutation.h       |  69 +-
 libcxx/include/__algorithm/ranges_is_sorted.h |  17 +-
 .../__algorithm/ranges_is_sorted_until.h      |  21 +-
 .../ranges_lexicographical_compare.h          |  73 +--
 .../include/__algorithm/ranges_lower_bound.h  |  22 +-
 libcxx/include/__algorithm/ranges_make_heap.h |  14 +-
 libcxx/include/__algorithm/ranges_max.h       |  33 +-
 .../include/__algorithm/ranges_max_element.h  |  17 +-
 libcxx/include/__algorithm/ranges_merge.h     |  91 ++-
 libcxx/include/__algorithm/ranges_min.h       |  33 +-
 .../include/__algorithm/ranges_min_element.h  |  20 +-
 libcxx/include/__algorithm/ranges_minmax.h    |  32 +-
 .../__algorithm/ranges_minmax_element.h       |  16 +-
 libcxx/include/__algorithm/ranges_mismatch.h  |  42 +-
 libcxx/include/__algorithm/ranges_move.h      |  16 +-
 .../__algorithm/ranges_move_backward.h        |  16 +-
 libcxx/include/__algorithm/ranges_none_of.h   |  22 +-
 .../include/__algorithm/ranges_nth_element.h  |  15 +-
 .../include/__algorithm/ranges_partial_sort.h |  15 +-
 .../__algorithm/ranges_partial_sort_copy.h    |  73 ++-
 libcxx/include/__algorithm/ranges_partition.h |  27 +-
 .../__algorithm/ranges_partition_copy.h       |  49 +-
 .../__algorithm/ranges_partition_point.h      |  24 +-
 libcxx/include/__algorithm/ranges_pop_heap.h  |  16 +-
 .../__algorithm/ranges_prev_permutation.h     |   8 +-
 libcxx/include/__algorithm/ranges_push_heap.h |  14 +-
 libcxx/include/__algorithm/ranges_remove.h    |  15 +-
 .../include/__algorithm/ranges_remove_copy.h  |  48 +-
 .../__algorithm/ranges_remove_copy_if.h       |  46 +-
 libcxx/include/__algorithm/ranges_remove_if.h |  19 +-
 libcxx/include/__algorithm/ranges_replace.h   |  29 +-
 .../include/__algorithm/ranges_replace_copy.h |  75 ++-
 .../__algorithm/ranges_replace_copy_if.h      |  56 +-
 .../include/__algorithm/ranges_replace_if.h   |  15 +-
 libcxx/include/__algorithm/ranges_reverse.h   |  10 +-
 .../include/__algorithm/ranges_reverse_copy.h |  12 +-
 libcxx/include/__algorithm/ranges_rotate.h    |  19 +-
 .../include/__algorithm/ranges_rotate_copy.h  |  10 +-
 libcxx/include/__algorithm/ranges_sample.h    |  27 +-
 libcxx/include/__algorithm/ranges_search.h    |  35 +-
 libcxx/include/__algorithm/ranges_search_n.h  |  34 +-
 .../__algorithm/ranges_set_difference.h       |  45 +-
 .../__algorithm/ranges_set_intersection.h     |  59 +-
 .../ranges_set_symmetric_difference.h         |  53 +-
 libcxx/include/__algorithm/ranges_set_union.h |  57 +-
 libcxx/include/__algorithm/ranges_shuffle.h   |  16 +-
 libcxx/include/__algorithm/ranges_sort.h      |  14 +-
 libcxx/include/__algorithm/ranges_sort_heap.h |  14 +-
 .../__algorithm/ranges_stable_partition.h     |  29 +-
 .../include/__algorithm/ranges_stable_sort.h  |  12 +-
 .../include/__algorithm/ranges_swap_ranges.h  |  11 +-
 libcxx/include/__algorithm/ranges_transform.h | 120 ++--
 libcxx/include/__algorithm/ranges_unique.h    |  48 +-
 .../include/__algorithm/ranges_unique_copy.h  |   6 +-
 .../include/__algorithm/ranges_upper_bound.h  |  29 +-
 libcxx/include/__concepts/arithmetic.h        |   8 +-
 libcxx/include/__concepts/assignable.h        |  12 +-
 libcxx/include/__concepts/boolean_testable.h  |   4 +-
 libcxx/include/__concepts/class_or_enum.h     |   4 +-
 .../__concepts/common_reference_with.h        |   7 +-
 libcxx/include/__concepts/common_with.h       |  30 +-
 libcxx/include/__concepts/constructible.h     |  29 +-
 libcxx/include/__concepts/convertible_to.h    |   8 +-
 libcxx/include/__concepts/copyable.h          |  14 +-
 libcxx/include/__concepts/derived_from.h      |   6 +-
 libcxx/include/__concepts/destructible.h      |   2 +-
 libcxx/include/__concepts/different_from.h    |   2 +-
 .../include/__concepts/equality_comparable.h  |  36 +-
 libcxx/include/__concepts/invocable.h         |   4 +-
 libcxx/include/__concepts/movable.h           |   8 +-
 libcxx/include/__concepts/predicate.h         |   5 +-
 libcxx/include/__concepts/regular.h           |   2 +-
 libcxx/include/__concepts/relation.h          |   9 +-
 libcxx/include/__concepts/same_as.h           |   4 +-
 libcxx/include/__concepts/semiregular.h       |   2 +-
 libcxx/include/__concepts/swappable.h         | 115 ++--
 libcxx/include/__concepts/totally_ordered.h   |  45 +-
 libcxx/include/__type_traits/add_const.h      |   6 +-
 libcxx/include/__type_traits/add_cv.h         |   6 +-
 .../__type_traits/add_lvalue_reference.h      |   3 +-
 libcxx/include/__type_traits/add_pointer.h    |  12 +-
 libcxx/include/__type_traits/add_volatile.h   |   6 +-
 .../include/__type_traits/aligned_storage.h   |  90 +--
 libcxx/include/__type_traits/aligned_union.h  |  29 +-
 libcxx/include/__type_traits/alignment_of.h   |   4 +-
 .../include/__type_traits/can_extract_key.h   |   9 +-
 .../include/__type_traits/common_reference.h  | 125 ++--
 libcxx/include/__type_traits/common_type.h    |  45 +-
 libcxx/include/__type_traits/copy_cv.h        |  20 +-
 libcxx/include/__type_traits/copy_cvref.h     |  15 +-
 libcxx/include/__type_traits/decay.h          |  27 +-
 libcxx/include/__type_traits/enable_if.h      |  14 +-
 libcxx/include/__type_traits/extent.h         |  35 +-
 .../has_unique_object_representation.h        |   6 +-
 .../__type_traits/has_virtual_destructor.h    |   4 +-
 .../include/__type_traits/integral_constant.h |  15 +-
 libcxx/include/__type_traits/invoke.h         | 595 ++++++++----------
 libcxx/include/__type_traits/is_abstract.h    |   4 +-
 libcxx/include/__type_traits/is_aggregate.h   |   4 +-
 libcxx/include/__type_traits/is_allocator.h   |  10 +-
 .../__type_traits/is_always_bitcastable.h     |  83 +--
 libcxx/include/__type_traits/is_arithmetic.h  |   6 +-
 libcxx/include/__type_traits/is_array.h       |  22 +-
 libcxx/include/__type_traits/is_assignable.h  |   4 +-
 libcxx/include/__type_traits/is_base_of.h     |   3 +-
 .../include/__type_traits/is_bounded_array.h  |  15 +-
 libcxx/include/__type_traits/is_callable.h    |   6 +-
 libcxx/include/__type_traits/is_class.h       |   4 +-
 libcxx/include/__type_traits/is_compound.h    |  16 +-
 libcxx/include/__type_traits/is_const.h       |  16 +-
 .../__type_traits/is_constant_evaluated.h     |   8 +-
 .../include/__type_traits/is_constructible.h  |   8 +-
 libcxx/include/__type_traits/is_convertible.h | 105 ++--
 .../__type_traits/is_copy_assignable.h        |   3 +-
 .../__type_traits/is_copy_constructible.h     |   5 +-
 .../__type_traits/is_core_convertible.h       |   5 +-
 .../__type_traits/is_default_constructible.h  |   4 +-
 .../include/__type_traits/is_destructible.h   |  43 +-
 libcxx/include/__type_traits/is_empty.h       |   3 +-
 libcxx/include/__type_traits/is_enum.h        |   4 +-
 libcxx/include/__type_traits/is_final.h       |   8 +-
 .../include/__type_traits/is_floating_point.h |   6 +-
 libcxx/include/__type_traits/is_fundamental.h |  19 +-
 .../is_implicitly_default_constructible.h     |  16 +-
 libcxx/include/__type_traits/is_integral.h    |  16 +-
 .../include/__type_traits/is_literal_type.h   |  12 +-
 .../is_member_function_pointer.h              |  34 +-
 .../__type_traits/is_member_object_pointer.h  |  18 +-
 .../include/__type_traits/is_member_pointer.h |  17 +-
 .../__type_traits/is_move_assignable.h        |   4 +-
 .../__type_traits/is_move_constructible.h     |   3 +-
 .../__type_traits/is_nothrow_assignable.h     |   4 +-
 .../__type_traits/is_nothrow_constructible.h  |  38 +-
 .../__type_traits/is_nothrow_convertible.h    |  15 +-
 .../is_nothrow_copy_assignable.h              |   8 +-
 .../is_nothrow_copy_constructible.h           |   6 +-
 .../is_nothrow_default_constructible.h        |   6 +-
 .../__type_traits/is_nothrow_destructible.h   |  45 +-
 .../is_nothrow_move_assignable.h              |   5 +-
 .../is_nothrow_move_constructible.h           |  12 +-
 .../include/__type_traits/is_null_pointer.h   |  18 +-
 libcxx/include/__type_traits/is_object.h      |  22 +-
 libcxx/include/__type_traits/is_pod.h         |   4 +-
 libcxx/include/__type_traits/is_pointer.h     |  32 +-
 libcxx/include/__type_traits/is_polymorphic.h |   3 +-
 .../__type_traits/is_primary_template.h       |   9 +-
 libcxx/include/__type_traits/is_reference.h   |  45 +-
 .../__type_traits/is_reference_wrapper.h      |  15 +-
 libcxx/include/__type_traits/is_same.h        |   2 +-
 libcxx/include/__type_traits/is_scalar.h      |  36 +-
 libcxx/include/__type_traits/is_scoped_enum.h |   6 +-
 libcxx/include/__type_traits/is_signed.h      |  20 +-
 .../include/__type_traits/is_signed_integer.h |  16 +-
 .../__type_traits/is_standard_layout.h        |   5 +-
 libcxx/include/__type_traits/is_swappable.h   | 100 ++-
 libcxx/include/__type_traits/is_trivial.h     |   5 +-
 .../__type_traits/is_trivially_assignable.h   |   4 +-
 .../is_trivially_constructible.h              |   4 +-
 .../is_trivially_copy_assignable.h            |   8 +-
 .../is_trivially_copy_constructible.h         |   6 +-
 .../__type_traits/is_trivially_copyable.h     |   5 +-
 .../is_trivially_default_constructible.h      |   6 +-
 .../__type_traits/is_trivially_destructible.h |  10 +-
 .../__type_traits/is_unbounded_array.h        |  15 +-
 libcxx/include/__type_traits/is_union.h       |   4 +-
 libcxx/include/__type_traits/is_unsigned.h    |  20 +-
 .../__type_traits/is_unsigned_integer.h       |  16 +-
 .../__type_traits/is_valid_expansion.h        |   6 +-
 libcxx/include/__type_traits/is_void.h        |  14 +-
 libcxx/include/__type_traits/is_volatile.h    |  16 +-
 libcxx/include/__type_traits/lazy.h           |   2 +-
 .../__type_traits/make_32_64_or_128_bit.h     |  19 +-
 .../__type_traits/make_const_lvalue_ref.h     |   2 +-
 libcxx/include/__type_traits/make_signed.h    |  33 +-
 libcxx/include/__type_traits/make_unsigned.h  |  38 +-
 libcxx/include/__type_traits/maybe_const.h    |   2 +-
 libcxx/include/__type_traits/nat.h            |  11 +-
 libcxx/include/__type_traits/negation.h       |   2 +-
 .../noexcept_move_assign_container.h          |  14 +-
 libcxx/include/__type_traits/promote.h        |  84 ++-
 libcxx/include/__type_traits/rank.h           |  12 +-
 .../__type_traits/remove_all_extents.h        |  21 +-
 libcxx/include/__type_traits/remove_const.h   |  13 +-
 libcxx/include/__type_traits/remove_cv.h      |   9 +-
 libcxx/include/__type_traits/remove_cvref.h   |   5 +-
 libcxx/include/__type_traits/remove_extent.h  |  21 +-
 libcxx/include/__type_traits/remove_pointer.h |   5 +-
 .../include/__type_traits/remove_reference.h  |   5 +-
 .../include/__type_traits/remove_volatile.h   |  13 +-
 libcxx/include/__type_traits/result_of.h      |  23 +-
 .../include/__type_traits/strip_signature.h   |   8 +-
 libcxx/include/__type_traits/type_identity.h  |  12 +-
 libcxx/include/__type_traits/type_list.h      |  20 +-
 .../include/__type_traits/underlying_type.h   |  11 +-
 libcxx/include/__type_traits/unwrap_ref.h     |  15 +-
 libcxx/include/__type_traits/void_t.h         |   3 +-
 libcxx/utils/data/ignore_format.txt           | 262 --------
 263 files changed, 3365 insertions(+), 4090 deletions(-)

diff --git a/libcxx/benchmarks/CartesianBenchmarks.h b/libcxx/benchmarks/CartesianBenchmarks.h
index 2eea156819330..eca4e15cd009b 100644
--- a/libcxx/benchmarks/CartesianBenchmarks.h
+++ b/libcxx/benchmarks/CartesianBenchmarks.h
@@ -6,7 +6,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-
 #include <string>
 #include <tuple>
 #include <type_traits>
@@ -22,7 +21,7 @@ struct EnumValue : std::integral_constant<E, static_cast<E>(I)> {
   static std::string name() { return std::string("_") + D::Names[I]; }
 };
 
-template <class D, class E, size_t ...Idxs>
+template <class D, class E, size_t... Idxs>
 constexpr auto makeEnumValueTuple(std::index_sequence<Idxs...>) {
   return std::make_tuple(EnumValue<D, E, Idxs>{}...);
 }
@@ -41,8 +40,7 @@ void makeBenchmarkFromValuesImpl(const Args& A, std::index_sequence<Is...>) {
   for (auto& V : A) {
     B Bench{std::get<Is>(V)...};
     if (!internal::skip(Bench, 0)) {
-      benchmark::RegisterBenchmark(Bench.name().c_str(),
-                                   [=](benchmark::State& S) { Bench.run(S); });
+      benchmark::RegisterBenchmark(Bench.name().c_str(), [=](benchmark::State& S) { Bench.run(S); });
     }
   }
 }
@@ -57,10 +55,8 @@ void makeBenchmarkImpl(const Args& A, std::tuple<U...> t) {
   makeBenchmarkFromValues<B<U...> >(A);
 }
 
-template <template <class...> class B, class Args, class... U,
-          class... T, class... Tuples>
-void makeBenchmarkImpl(const Args& A, std::tuple<U...>, std::tuple<T...>,
-                       Tuples... rest) {
+template <template <class...> class B, class Args, class... U, class... T, class... Tuples>
+void makeBenchmarkImpl(const Args& A, std::tuple<U...>, std::tuple<T...>, Tuples... rest) {
   (internal::makeBenchmarkImpl<B>(A, std::tuple<U..., T>(), rest...), ...);
 }
 
@@ -70,15 +66,13 @@ void allValueCombinations(R& Result, const T& Final) {
 }
 
 template <class R, class T, class V, class... Vs>
-void allValueCombinations(R& Result, const T& Prev, const V& Value,
-                          const Vs&... Values) {
+void allValueCombinations(R& Result, const T& Prev, const V& Value, const Vs&... Values) {
   for (const auto& E : Value) {
-    allValueCombinations(Result, std::tuple_cat(Prev, std::make_tuple(E)),
-                         Values...);
+    allValueCombinations(Result, std::tuple_cat(Prev, std::make_tuple(E)), Values...);
   }
 }
 
-}  // namespace internal
+} // namespace internal
 
 // CRTP class that enables using enum types as a dimension for
 // makeCartesianProductBenchmark below.
@@ -93,8 +87,7 @@ void allValueCombinations(R& Result, const T& Prev, const V& Value,
 // };
 template <class Derived, class EnumType, size_t NumLabels>
 using EnumValuesAsTuple =
-    decltype(internal::makeEnumValueTuple<Derived, EnumType>(
-        std::make_index_sequence<NumLabels>{}));
+    decltype(internal::makeEnumValueTuple<Derived, EnumType>(std::make_index_sequence<NumLabels>{}));
 
 // Instantiates B<T0, T1, ..., TN> where <Ti...> are the combinations in the
 // cartesian product of `Tuples...`, and pass (arg0, ..., argN) as constructor
@@ -128,6 +121,7 @@ int makeCartesianProductBenchmark(const Args&... A) {
 // It returns `value`.
 template <class T>
 TEST_ALWAYS_INLINE inline T maybeOpaque(T value, bool opaque) {
-  if (opaque) benchmark::DoNotOptimize(value);
+  if (opaque)
+    benchmark::DoNotOptimize(value);
   return value;
 }
diff --git a/libcxx/benchmarks/ContainerBenchmarks.h b/libcxx/benchmarks/ContainerBenchmarks.h
index 4dae95748077f..bf027f5f41469 100644
--- a/libcxx/benchmarks/ContainerBenchmarks.h
+++ b/libcxx/benchmarks/ContainerBenchmarks.h
@@ -59,116 +59,114 @@ void BM_ConstructSizeValue(benchmark::State& st, Container, typename Container::
 
 template <class Container, class GenInputs>
 void BM_ConstructIterIter(benchmark::State& st, Container, GenInputs gen) {
-    auto in = gen(st.range(0));
-    const auto begin = in.begin();
-    const auto end = in.end();
-    benchmark::DoNotOptimize(&in);
-    while (st.KeepRunning()) {
-        Container c(begin, end);
-        DoNotOptimizeData(c);
-    }
+  auto in          = gen(st.range(0));
+  const auto begin = in.begin();
+  const auto end   = in.end();
+  benchmark::DoNotOptimize(&in);
+  while (st.KeepRunning()) {
+    Container c(begin, end);
+    DoNotOptimizeData(c);
+  }
 }
 
 template <class Container, class GenInputs>
 void BM_InsertValue(benchmark::State& st, Container c, GenInputs gen) {
-    auto in = gen(st.range(0));
-    const auto end = in.end();
-    while (st.KeepRunning()) {
-        c.clear();
-        for (auto it = in.begin(); it != end; ++it) {
-            benchmark::DoNotOptimize(&(*c.insert(*it).first));
-        }
-        benchmark::ClobberMemory();
+  auto in        = gen(st.range(0));
+  const auto end = in.end();
+  while (st.KeepRunning()) {
+    c.clear();
+    for (auto it = in.begin(); it != end; ++it) {
+      benchmark::DoNotOptimize(&(*c.insert(*it).first));
     }
+    benchmark::ClobberMemory();
+  }
 }
 
 template <class Container, class GenInputs>
 void BM_InsertValueRehash(benchmark::State& st, Container c, GenInputs gen) {
-    auto in = gen(st.range(0));
-    const auto end = in.end();
-    while (st.KeepRunning()) {
-        c.clear();
-        c.rehash(16);
-        for (auto it = in.begin(); it != end; ++it) {
-            benchmark::DoNotOptimize(&(*c.insert(*it).first));
-        }
-        benchmark::ClobberMemory();
+  auto in        = gen(st.range(0));
+  const auto end = in.end();
+  while (st.KeepRunning()) {
+    c.clear();
+    c.rehash(16);
+    for (auto it = in.begin(); it != end; ++it) {
+      benchmark::DoNotOptimize(&(*c.insert(*it).first));
     }
+    benchmark::ClobberMemory();
+  }
 }
 
-
 template <class Container, class GenInputs>
 void BM_InsertDuplicate(benchmark::State& st, Container c, GenInputs gen) {
-    auto in = gen(st.range(0));
-    const auto end = in.end();
-    c.insert(in.begin(), in.end());
-    benchmark::DoNotOptimize(&c);
-    benchmark::DoNotOptimize(&in);
-    while (st.KeepRunning()) {
-        for (auto it = in.begin(); it != end; ++it) {
-            benchmark::DoNotOptimize(&(*c.insert(*it).first));
-        }
-        benchmark::ClobberMemory();
+  auto in        = gen(st.range(0));
+  const auto end = in.end();
+  c.insert(in.begin(), in.end());
+  benchmark::DoNotOptimize(&c);
+  benchmark::DoNotOptimize(&in);
+  while (st.KeepRunning()) {
+    for (auto it = in.begin(); it != end; ++it) {
+      benchmark::DoNotOptimize(&(*c.insert(*it).first));
     }
+    benchmark::ClobberMemory();
+  }
 }
 
-
 template <class Container, class GenInputs>
 void BM_EmplaceDuplicate(benchmark::State& st, Container c, GenInputs gen) {
-    auto in = gen(st.range(0));
-    const auto end = in.end();
-    c.insert(in.begin(), in.end());
-    benchmark::DoNotOptimize(&c);
-    benchmark::DoNotOptimize(&in);
-    while (st.KeepRunning()) {
-        for (auto it = in.begin(); it != end; ++it) {
-            benchmark::DoNotOptimize(&(*c.emplace(*it).first));
-        }
-        benchmark::ClobberMemory();
+  auto in        = gen(st.range(0));
+  const auto end = in.end();
+  c.insert(in.begin(), in.end());
+  benchmark::DoNotOptimize(&c);
+  benchmark::DoNotOptimize(&in);
+  while (st.KeepRunning()) {
+    for (auto it = in.begin(); it != end; ++it) {
+      benchmark::DoNotOptimize(&(*c.emplace(*it).first));
     }
+    benchmark::ClobberMemory();
+  }
 }
 
 template <class Container, class GenInputs>
 static void BM_Find(benchmark::State& st, Container c, GenInputs gen) {
-    auto in = gen(st.range(0));
-    c.insert(in.begin(), in.end());
-    benchmark::DoNotOptimize(&(*c.begin()));
-    const auto end = in.data() + in.size();
-    while (st.KeepRunning()) {
-        for (auto it = in.data(); it != end; ++it) {
-            benchmark::DoNotOptimize(&(*c.find(*it)));
-        }
-        benchmark::ClobberMemory();
+  auto in = gen(st.range(0));
+  c.insert(in.begin(), in.end());
+  benchmark::DoNotOptimize(&(*c.begin()));
+  const auto end = in.data() + in.size();
+  while (st.KeepRunning()) {
+    for (auto it = in.data(); it != end; ++it) {
+      benchmark::DoNotOptimize(&(*c.find(*it)));
     }
+    benchmark::ClobberMemory();
+  }
 }
 
 template <class Container, class GenInputs>
 static void BM_FindRehash(benchmark::State& st, Container c, GenInputs gen) {
-    c.rehash(8);
-    auto in = gen(st.range(0));
-    c.insert(in.begin(), in.end());
-    benchmark::DoNotOptimize(&(*c.begin()));
-    const auto end = in.data() + in.size();
-    while (st.KeepRunning()) {
-        for (auto it = in.data(); it != end; ++it) {
-            benchmark::DoNotOptimize(&(*c.find(*it)));
-        }
-        benchmark::ClobberMemory();
+  c.rehash(8);
+  auto in = gen(st.range(0));
+  c.insert(in.begin(), in.end());
+  benchmark::DoNotOptimize(&(*c.begin()));
+  const auto end = in.data() + in.size();
+  while (st.KeepRunning()) {
+    for (auto it = in.data(); it != end; ++it) {
+      benchmark::DoNotOptimize(&(*c.find(*it)));
     }
+    benchmark::ClobberMemory();
+  }
 }
 
 template <class Container, class GenInputs>
 static void BM_Rehash(benchmark::State& st, Container c, GenInputs gen) {
-    auto in = gen(st.range(0));
-    c.max_load_factor(3.0);
-    c.insert(in.begin(), in.end());
-    benchmark::DoNotOptimize(c);
-    const auto bucket_count = c.bucket_count();
-    while (st.KeepRunning()) {
-        c.rehash(bucket_count + 1);
-        c.rehash(bucket_count);
-        benchmark::ClobberMemory();
-    }
+  auto in = gen(st.range(0));
+  c.max_load_factor(3.0);
+  c.insert(in.begin(), in.end());
+  benchmark::DoNotOptimize(c);
+  const auto bucket_count = c.bucket_count();
+  while (st.KeepRunning()) {
+    c.rehash(bucket_count + 1);
+    c.rehash(bucket_count);
+    benchmark::ClobberMemory();
+  }
 }
 
 } // end namespace ContainerBenchmarks
diff --git a/libcxx/benchmarks/GenerateInput.h b/libcxx/benchmarks/GenerateInput.h
index 3e63f8413303f..3980b1e15aee2 100644
--- a/libcxx/benchmarks/GenerateInput.h
+++ b/libcxx/benchmarks/GenerateInput.h
@@ -2,142 +2,133 @@
 #define BENCHMARK_GENERATE_INPUT_H
 
 #include <algorithm>
-#include <random>
-#include <vector>
-#include <string>
 #include <climits>
 #include <cstddef>
+#include <random>
+#include <string>
+#include <vector>
 
 static const char Letters[] = {
-    '0','1','2','3','4',
-    '5','6','7','8','9',
-    'A','B','C','D','E','F',
-    'G','H','I','J','K',
-    'L','M','N','O','P',
-    'Q','R','S','T','U',
-    'V','W','X','Y','Z',
-    'a','b','c','d','e','f',
-    'g','h','i','j','k',
-    'l','m','n','o','p',
-    'q','r','s','t','u',
-    'v','w','x','y','z'
-};
+    '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K',
+    'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f',
+    'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'};
 static const std::size_t LettersSize = sizeof(Letters);
 
 inline std::default_random_engine& getRandomEngine() {
-    static std::default_random_engine RandEngine(std::random_device{}());
-    return RandEngine;
+  static std::default_random_engine RandEngine(std::random_device{}());
+  return RandEngine;
 }
 
-
 inline char getRandomChar() {
-    std::uniform_int_distribution<> LettersDist(0, LettersSize-1);
-    return Letters[LettersDist(getRandomEngine())];
+  std::uniform_int_distribution<> LettersDist(0, LettersSize - 1);
+  return Letters[LettersDist(getRandomEngine())];
 }
 
 template <class IntT>
 inline IntT getRandomInteger(IntT Min, IntT Max) {
-    std::uniform_int_distribution<unsigned long long> dist(Min, Max);
-    return static_cast<IntT>(dist(getRandomEngine()));
+  std::uniform_int_distribution<unsigned long long> dist(Min, Max);
+  return static_cast<IntT>(dist(getRandomEngine()));
 }
 
 inline std::string getRandomString(std::size_t Len) {
-    std::string str(Len, 0);
-    std::generate_n(str.begin(), Len, &getRandomChar);
-    return str;
+  std::string str(Len, 0);
+  std::generate_n(str.begin(), Len, &getRandomChar);
+  return str;
 }
 
 template <class IntT>
 inline std::vector<IntT> getDuplicateIntegerInputs(size_t N) {
-    std::vector<IntT> inputs(N, static_cast<IntT>(-1));
-    return inputs;
+  std::vector<IntT> inputs(N, static_cast<IntT>(-1));
+  return inputs;
 }
 
 template <class IntT>
 inline std::vector<IntT> getSortedIntegerInputs(size_t N) {
-    std::vector<IntT> inputs;
-    for (size_t i=0; i < N; i += 1)
-        inputs.push_back(i);
-    return inputs;
+  std::vector<IntT> inputs;
+  for (size_t i = 0; i < N; i += 1)
+    inputs.push_back(i);
+  return inputs;
 }
 
 template <class IntT>
 std::vector<IntT> getSortedLargeIntegerInputs(size_t N) {
-    std::vector<IntT> inputs;
-    for (size_t i=0; i < N; ++i) {
-        inputs.push_back(i + N);
-    }
-    return inputs;
+  std::vector<IntT> inputs;
+  for (size_t i = 0; i < N; ++i) {
+    inputs.push_back(i + N);
+  }
+  return inputs;
 }
 
 template <class IntT>
 std::vector<IntT> getSortedTopBitsIntegerInputs(size_t N) {
-    std::vector<IntT> inputs = getSortedIntegerInputs<IntT>(N);
-    for (auto& E : inputs) E <<= ((sizeof(IntT) / 2) * CHAR_BIT);
-    return inputs;
+  std::vector<IntT> inputs = getSortedIntegerInputs<IntT>(N);
+  for (auto& E : inputs)
+    E <<= ((sizeof(IntT) / 2) * CHAR_BIT);
+  return inputs;
 }
 
 template <class IntT>
 inline std::vector<IntT> getReverseSortedIntegerInputs(size_t N) {
-    std::vector<IntT> inputs;
-    std::size_t i = N;
-    while (i > 0) {
-        --i;
-        inputs.push_back(i);
-    }
-    return inputs;
+  std::vector<IntT> inputs;
+  std::size_t i = N;
+  while (i > 0) {
+    --i;
+    inputs.push_back(i);
+  }
+  return inputs;
 }
 
 template <class IntT>
 std::vector<IntT> getPipeOrganIntegerInputs(size_t N) {
-    std::vector<IntT> v; v.reserve(N);
-    for (size_t i = 0; i < N/2; ++i) v.push_back(i);
-    for (size_t i = N/2; i < N; ++i) v.push_back(N - i);
-    return v;
+  std::vector<IntT> v;
+  v.reserve(N);
+  for (size_t i = 0; i < N / 2; ++i)
+    v.push_back(i);
+  for (size_t i = N / 2; i < N; ++i)
+    v.push_back(N - i);
+  return v;
 }
 
-
 template <class IntT>
 std::vector<IntT> getRandomIntegerInputs(size_t N) {
-    std::vector<IntT> inputs;
-    for (size_t i=0; i < N; ++i) {
-        inputs.push_back(getRandomInteger<IntT>(0, std::numeric_limits<IntT>::max()));
-    }
-    return inputs;
+  std::vector<IntT> inputs;
+  for (size_t i = 0; i < N; ++i) {
+    inputs.push_back(getRandomInteger<IntT>(0, std::numeric_limits<IntT>::max()));
+  }
+  return inputs;
 }
 
 inline std::vector<std::string> getDuplicateStringInputs(size_t N) {
-    std::vector<std::string> inputs(N, getRandomString(1024));
-    return inputs;
+  std::vector<std::string> inputs(N, getRandomString(1024));
+  return inputs;
 }
 
 inline std::vector<std::string> getRandomStringInputs(size_t N) {
-    std::vector<std::string> inputs;
-    for (size_t i=0; i < N; ++i) {
-        inputs.push_back(getRandomString(1024));
-    }
-    return inputs;
+  std::vector<std::string> inputs;
+  for (size_t i = 0; i < N; ++i) {
+    inputs.push_back(getRandomString(1024));
+  }
+  return inputs;
 }
 
 inline std::vector<std::string> getSortedStringInputs(size_t N) {
-    std::vector<std::string> inputs = getRandomStringInputs(N);
-    std::sort(inputs.begin(), inputs.end());
-    return inputs;
+  std::vector<std::string> inputs = getRandomStringInputs(N);
+  std::sort(inputs.begin(), inputs.end());
+  return inputs;
 }
 
 inline std::vector<std::string> getReverseSortedStringInputs(size_t N) {
-    std::vector<std::string> inputs = getSortedStringInputs(N);
-    std::reverse(inputs.begin(), inputs.end());
-    return inputs;
+  std::vector<std::string> inputs = getSortedStringInputs(N);
+  std::reverse(inputs.begin(), inputs.end());
+  return inputs;
 }
 
 inline std::vector<const char*> getRandomCStringInputs(size_t N) {
-    static std::vector<std::string> inputs = getRandomStringInputs(N);
-    std::vector<const char*> cinputs;
-    for (auto const& str : inputs)
-        cinputs.push_back(str.c_str());
-    return cinputs;
+  static std::vector<std::string> inputs = getRandomStringInputs(N);
+  std::vector<const char*> cinputs;
+  for (auto const& str : inputs)
+    cinputs.push_back(str.c_str());
+  return cinputs;
 }
 
-
 #endif // BENCHMARK_GENERATE_INPUT_H
diff --git a/libcxx/benchmarks/Utilities.h b/libcxx/benchmarks/Utilities.h
index 9ad2a58796ea3..fed16ba51f995 100644
--- a/libcxx/benchmarks/Utilities.h
+++ b/libcxx/benchmarks/Utilities.h
@@ -16,18 +16,22 @@
 #include "benchmark/benchmark.h"
 
 namespace UtilitiesInternal {
-  template <class Container>
-  auto HaveDataImpl(int) -> decltype((std::declval<Container&>().data(), std::true_type{}));
-  template <class Container>
-  auto HaveDataImpl(long) -> std::false_type;
-  template <class T>
-  using HasData = decltype(HaveDataImpl<T>(0));
+template <class Container>
+auto HaveDataImpl(int) -> decltype((std::declval<Container&>().data(), std::true_type{}));
+template <class Container>
+auto HaveDataImpl(long) -> std::false_type;
+template <class T>
+using HasData = decltype(HaveDataImpl<T>(0));
 } // namespace UtilitiesInternal
 
 template <class Container, std::enable_if_t<UtilitiesInternal::HasData<Container>::value>* = nullptr>
-void DoNotOptimizeData(Container &c) { benchmark::DoNotOptimize(c.data()); }
-template <class Container, std::enable_if_t<!UtilitiesInternal::HasData<Container>::value>* = nullptr>
-void DoNotOptimizeData(Container &c) { benchmark::DoNotOptimize(&c); }
+void DoNotOptimizeData(Container& c) {
+  benchmark::DoNotOptimize(c.data());
+}
 
+template <class Container, std::enable_if_t<!UtilitiesInternal::HasData<Container>::value>* = nullptr>
+void DoNotOptimizeData(Container& c) {
+  benchmark::DoNotOptimize(&c);
+}
 
 #endif // BENCHMARK_UTILITIES_H
diff --git a/libcxx/benchmarks/VariantBenchmarks.h b/libcxx/benchmarks/VariantBenchmarks.h
index 2feaeab51b97e..a8e9c9febd728 100644
--- a/libcxx/benchmarks/VariantBenchmarks.h
+++ b/libcxx/benchmarks/VariantBenchmarks.h
@@ -29,8 +29,8 @@ struct S {
 
 template <std::size_t N, std::size_t... Is>
 static auto genVariants(std::index_sequence<Is...>) {
-  using V = std::variant<S<Is>...>;
-  using F = V (*)();
+  using V                 = std::variant<S<Is>...>;
+  using F                 = V (*)();
   static constexpr F fs[] = {[] { return V(std::in_place_index<Is>); }...};
 
   std::array<V, N> result = {};
@@ -45,11 +45,8 @@ template <std::size_t N, std::size_t Alts>
 static void BM_Visit(benchmark::State& state) {
   auto args = genVariants<N>(std::make_index_sequence<Alts>{});
   for (auto _ : state) {
-    benchmark::DoNotOptimize(std::apply(
-        [](auto... vs) {
-          return std::visit([](auto... is) { return (is.v + ... + 0); }, vs...);
-        },
-        args));
+    benchmark::DoNotOptimize(
+        std::apply([](auto... vs) { return std::visit([](auto... is) { return (is.v + ... + 0); }, vs...); }, args));
   }
 }
 
diff --git a/libcxx/benchmarks/algorithms.partition_point.bench.cpp b/libcxx/benchmarks/algorithms.partition_point.bench.cpp
index 8192f97206d9b..77115175fa540 100644
--- a/libcxx/benchmarks/algorithms.partition_point.bench.cpp
+++ b/libcxx/benchmarks/algorithms.partition_point.bench.cpp
@@ -1,5 +1,5 @@
-#include <array>
 #include <algorithm>
+#include <array>
 #include <cassert>
 #include <cstdint>
 #include <tuple>
@@ -29,8 +29,7 @@ template <class IntT>
 struct TestIntBase {
   static std::vector<IntT> generateInput(size_t size) {
     std::vector<IntT> Res(size);
-    std::generate(Res.begin(), Res.end(),
-                  [] { return getRandomInteger<IntT>(0, std::numeric_limits<IntT>::max()); });
+    std::generate(Res.begin(), Res.end(), [] { return getRandomInteger<IntT>(0, std::numeric_limits<IntT>::max()); });
     return Res;
   }
 };
@@ -48,7 +47,7 @@ struct TestUint32 : TestIntBase<std::uint32_t> {
 };
 
 struct TestMediumString {
-  static constexpr const char* Name = "TestMediumString";
+  static constexpr const char* Name  = "TestMediumString";
   static constexpr size_t StringSize = 32;
 
   static std::vector<std::string> generateInput(size_t size) {
@@ -94,8 +93,7 @@ struct PartitionPointBench {
   size_t Quantity;
 
   std::string name() const {
-    return std::string("PartitionPointBench_") + Alg::Name + "_" +
-           TestType::Name + '/' + std::to_string(Quantity);
+    return std::string("PartitionPointBench_") + Alg::Name + "_" + TestType::Name + '/' + std::to_string(Quantity);
   }
 
   void run(benchmark::State& state) const {
@@ -118,7 +116,6 @@ int main(int argc, char** argv) {
     return 1;
 
   const std::vector<size_t> Quantities = {1 << 8, 1 << 10, 1 << 20};
-  makeCartesianProductBenchmark<PartitionPointBench, AllAlgs, AllTestTypes>(
-      Quantities);
+  makeCartesianProductBenchmark<PartitionPointBench, AllAlgs, AllTestTypes>(Quantities);
   benchmark::RunSpecifiedBenchmarks();
 }
diff --git a/libcxx/benchmarks/algorithms/common.h b/libcxx/benchmarks/algorithms/common.h
index ffa39e26be54b..43131a4c9c922 100644
--- a/libcxx/benchmarks/algorithms/common.h
+++ b/libcxx/benchmarks/algorithms/common.h
@@ -19,12 +19,17 @@
 
 enum class ValueType { Uint32, Uint64, Pair, Tuple, String, Float };
 struct AllValueTypes : EnumValuesAsTuple<AllValueTypes, ValueType, 6> {
-  static constexpr const char* Names[] = {"uint32", "uint64", "pair<uint32, uint32>", "tuple<uint32, uint64, uint32>",
-                                          "string", "float"};
+  static constexpr const char* Names[] = {
+      "uint32", "uint64", "pair<uint32, uint32>", "tuple<uint32, uint64, uint32>", "string", "float"};
 };
 
-using Types = std::tuple< uint32_t, uint64_t, std::pair<uint32_t, uint32_t>, std::tuple<uint32_t, uint64_t, uint32_t>,
-                          std::string, float >;
+using Types =
+    std::tuple<uint32_t,
+               uint64_t,
+               std::pair<uint32_t, uint32_t>,
+               std::tuple<uint32_t, uint64_t, uint32_t>,
+               std::string,
+               float>;
 
 template <class V>
 using Value = std::tuple_element_t<(int)V::value, Types>;
@@ -39,10 +44,8 @@ enum class Order {
   QuickSortAdversary,
 };
 struct AllOrders : EnumValuesAsTuple<AllOrders, Order, 7> {
-  static constexpr const char* Names[] = {"Random",     "Ascending",
-                                          "Descending", "SingleElement",
-                                          "PipeOrgan",  "Heap",
-                                          "QuickSortAdversary"};
+  static constexpr const char* Names[] = {
+      "Random", "Ascending", "Descending", "SingleElement", "PipeOrgan", "Heap", "QuickSortAdversary"};
 };
 
 // fillAdversarialQuickSortInput fills the input vector with N int-like values.
@@ -64,7 +67,7 @@ void fillAdversarialQuickSortInput(T& V, size_t N) {
   }
   // Candidate for the pivot position.
   int candidate = 0;
-  int nsolid = 0;
+  int nsolid    = 0;
   // Populate all positions in the generated input to gas.
   std::vector<int> ascVals(V.size());
   // Fill up with ascending values from 0 to V.size()-1.  These will act as
@@ -185,8 +188,7 @@ constexpr size_t TestSetElements =
 #endif
 
 template <class ValueType>
-std::vector<std::vector<Value<ValueType> > > makeOrderedValues(size_t N,
-                                                               Order O) {
+std::vector<std::vector<Value<ValueType> > > makeOrderedValues(size_t N, Order O) {
   std::vector<std::vector<Value<ValueType> > > Ret;
   const size_t NumCopies = std::max(size_t{1}, TestSetElements / N);
   Ret.resize(NumCopies);
@@ -198,8 +200,7 @@ std::vector<std::vector<Value<ValueType> > > makeOrderedValues(size_t N,
 }
 
 template <class T, class U>
-TEST_ALWAYS_INLINE void resetCopies(benchmark::State& state, T& Copies,
-                                    U& Orig) {
+TEST_ALWAYS_INLINE void resetCopies(benchmark::State& state, T& Copies, U& Orig) {
   state.PauseTiming();
   for (auto& Copy : Copies)
     Copy = Orig;
@@ -212,14 +213,11 @@ enum class BatchSize {
 };
 
 template <class ValueType, class F>
-void runOpOnCopies(benchmark::State& state, size_t Quantity, Order O,
-                   BatchSize Count, F Body) {
+void runOpOnCopies(benchmark::State& state, size_t Quantity, Order O, BatchSize Count, F Body) {
   auto Copies = makeOrderedValues<ValueType>(Quantity, O);
-  auto Orig = Copies;
+  auto Orig   = Copies;
 
-  const size_t Batch = Count == BatchSize::CountElements
-                           ? Copies.size() * Quantity
-                           : Copies.size();
+  const size_t Batch = Count == BatchSize::CountElements ? Copies.size() * Quantity : Copies.size();
   while (state.KeepRunningBatch(Batch)) {
     for (auto& Copy : Copies) {
       Body(Copy);
@@ -231,13 +229,18 @@ void runOpOnCopies(benchmark::State& state, size_t Quantity, Order O,
   }
 }
 
-
-const std::vector<size_t> Quantities = {1 << 0, 1 << 2,  1 << 4,  1 << 6,
-                                        1 << 8, 1 << 10, 1 << 14,
-    // Running each benchmark in parallel consumes too much memory with MSAN
-    // and can lead to the test process being killed.
+const std::vector<size_t> Quantities = {
+    1 << 0,
+    1 << 2,
+    1 << 4,
+    1 << 6,
+    1 << 8,
+    1 << 10,
+    1 << 14,
+// Running each benchmark in parallel consumes too much memory with MSAN
+// and can lead to the test process being killed.
 #if !TEST_HAS_FEATURE(memory_sanitizer)
-                                        1 << 18
+    1 << 18
 #endif
 };
 
diff --git a/libcxx/benchmarks/algorithms/lower_bound.bench.cpp b/libcxx/benchmarks/algorithms/lower_bound.bench.cpp
index ce366d5e9d4a0..3be5010f5bdf2 100644
--- a/libcxx/benchmarks/algorithms/lower_bound.bench.cpp
+++ b/libcxx/benchmarks/algorithms/lower_bound.bench.cpp
@@ -18,7 +18,7 @@ template <class ValueType>
 struct LowerBound {
   size_t Quantity;
 
-  mutable std::mt19937_64 rng { std::random_device{}() };
+  mutable std::mt19937_64 rng{std::random_device{}()};
 
   void run(benchmark::State& state) const {
     runOpOnCopies<ValueType>(state, Quantity, Order::Ascending, BatchSize::CountBatch, [&](auto& Copy) {
@@ -27,9 +27,7 @@ struct LowerBound {
     });
   }
 
-  std::string name() const {
-    return "BM_LowerBound" + ValueType::name() + "_" + std::to_string(Quantity);
-  }
+  std::string name() const { return "BM_LowerBound" + ValueType::name() + "_" + std::to_string(Quantity); }
 };
 } // namespace
 
diff --git a/libcxx/benchmarks/algorithms/make_heap.bench.cpp b/libcxx/benchmarks/algorithms/make_heap.bench.cpp
index e2caf84d137ac..dade7b8273107 100644
--- a/libcxx/benchmarks/algorithms/make_heap.bench.cpp
+++ b/libcxx/benchmarks/algorithms/make_heap.bench.cpp
@@ -16,14 +16,13 @@ struct MakeHeap {
   size_t Quantity;
 
   void run(benchmark::State& state) const {
-    runOpOnCopies<ValueType>(
-        state, Quantity, Order(), BatchSize::CountElements,
-        [](auto& Copy) { std::make_heap(Copy.begin(), Copy.end()); });
+    runOpOnCopies<ValueType>(state, Quantity, Order(), BatchSize::CountElements, [](auto& Copy) {
+      std::make_heap(Copy.begin(), Copy.end());
+    });
   }
 
   std::string name() const {
-    return "BM_MakeHeap" + ValueType::name() + Order::name() + "_" +
-           std::to_string(Quantity);
+    return "BM_MakeHeap" + ValueType::name() + Order::name() + "_" + std::to_string(Quantity);
   };
 };
 } // namespace
diff --git a/libcxx/benchmarks/algorithms/make_heap_then_sort_heap.bench.cpp b/libcxx/benchmarks/algorithms/make_heap_then_sort_heap.bench.cpp
index 5254c261e3ec1..48f34f8651b97 100644
--- a/libcxx/benchmarks/algorithms/make_heap_then_sort_heap.bench.cpp
+++ b/libcxx/benchmarks/algorithms/make_heap_then_sort_heap.bench.cpp
@@ -16,16 +16,14 @@ struct MakeThenSortHeap {
   size_t Quantity;
 
   void run(benchmark::State& state) const {
-    runOpOnCopies<ValueType>(state, Quantity, Order(), BatchSize::CountElements,
-                             [](auto& Copy) {
-                               std::make_heap(Copy.begin(), Copy.end());
-                               std::sort_heap(Copy.begin(), Copy.end());
-                             });
+    runOpOnCopies<ValueType>(state, Quantity, Order(), BatchSize::CountElements, [](auto& Copy) {
+      std::make_heap(Copy.begin(), Copy.end());
+      std::sort_heap(Copy.begin(), Copy.end());
+    });
   }
 
   std::string name() const {
-    return "BM_MakeThenSortHeap" + ValueType::name() + Order::name() + "_" +
-           std::to_string(Quantity);
+    return "BM_MakeThenSortHeap" + ValueType::name() + Order::name() + "_" + std::to_string(Quantity);
   };
 };
 } // namespace
diff --git a/libcxx/benchmarks/algorithms/pop_heap.bench.cpp b/libcxx/benchmarks/algorithms/pop_heap.bench.cpp
index ef6e5a669740e..26cdd25cce90d 100644
--- a/libcxx/benchmarks/algorithms/pop_heap.bench.cpp
+++ b/libcxx/benchmarks/algorithms/pop_heap.bench.cpp
@@ -16,17 +16,14 @@ struct PopHeap {
   size_t Quantity;
 
   void run(benchmark::State& state) const {
-    runOpOnCopies<ValueType>(
-        state, Quantity, Order(), BatchSize::CountElements, [](auto& Copy) {
-          for (auto B = Copy.begin(), I = Copy.end(); I != B; --I) {
-            std::pop_heap(B, I);
-          }
-        });
+    runOpOnCopies<ValueType>(state, Quantity, Order(), BatchSize::CountElements, [](auto& Copy) {
+      for (auto B = Copy.begin(), I = Copy.end(); I != B; --I) {
+        std::pop_heap(B, I);
+      }
+    });
   }
 
-  std::string name() const {
-    return "BM_PopHeap" + ValueType::name() + "_" + std::to_string(Quantity);
-  };
+  std::string name() const { return "BM_PopHeap" + ValueType::name() + "_" + std::to_string(Quantity); };
 };
 } // namespace
 
diff --git a/libcxx/benchmarks/algorithms/push_heap.bench.cpp b/libcxx/benchmarks/algorithms/push_heap.bench.cpp
index 8c333240f7caf..ba96fa1469e60 100644
--- a/libcxx/benchmarks/algorithms/push_heap.bench.cpp
+++ b/libcxx/benchmarks/algorithms/push_heap.bench.cpp
@@ -16,19 +16,17 @@ struct PushHeap {
   size_t Quantity;
 
   void run(benchmark::State& state) const {
-    runOpOnCopies<ValueType>(
-        state, Quantity, Order(), BatchSize::CountElements, [](auto& Copy) {
-          for (auto I = Copy.begin(), E = Copy.end(); I != E; ++I) {
-            std::push_heap(Copy.begin(), I + 1);
-          }
-        });
+    runOpOnCopies<ValueType>(state, Quantity, Order(), BatchSize::CountElements, [](auto& Copy) {
+      for (auto I = Copy.begin(), E = Copy.end(); I != E; ++I) {
+        std::push_heap(Copy.begin(), I + 1);
+      }
+    });
   }
 
   bool skip() const { return Order() == ::Order::Heap; }
 
   std::string name() const {
-    return "BM_PushHeap" + ValueType::name() + Order::name() + "_" +
-           std::to_string(Quantity);
+    return "BM_PushHeap" + ValueType::name() + Order::name() + "_" + std::to_string(Quantity);
   };
 };
 } // namespace
diff --git a/libcxx/benchmarks/algorithms/ranges_make_heap.bench.cpp b/libcxx/benchmarks/algorithms/ranges_make_heap.bench.cpp
index b5b6132382d69..66a8335a8710c 100644
--- a/libcxx/benchmarks/algorithms/ranges_make_heap.bench.cpp
+++ b/libcxx/benchmarks/algorithms/ranges_make_heap.bench.cpp
@@ -16,14 +16,13 @@ struct RangesMakeHeap {
   size_t Quantity;
 
   void run(benchmark::State& state) const {
-    runOpOnCopies<ValueType>(
-        state, Quantity, Order(), BatchSize::CountElements,
-        [](auto& Copy) { std::ranges::make_heap(Copy); });
+    runOpOnCopies<ValueType>(state, Quantity, Order(), BatchSize::CountElements, [](auto& Copy) {
+      std::ranges::make_heap(Copy);
+    });
   }
 
   std::string name() const {
-    return "BM_RangesMakeHeap" + ValueType::name() + Order::name() + "_" +
-           std::to_string(Quantity);
+    return "BM_RangesMakeHeap" + ValueType::name() + Order::name() + "_" + std::to_string(Quantity);
   };
 };
 } // namespace
diff --git a/libcxx/benchmarks/algorithms/ranges_make_heap_then_sort_heap.bench.cpp b/libcxx/benchmarks/algorithms/ranges_make_heap_then_sort_heap.bench.cpp
index f28695135304a..01632c807c178 100644
--- a/libcxx/benchmarks/algorithms/ranges_make_heap_then_sort_heap.bench.cpp
+++ b/libcxx/benchmarks/algorithms/ranges_make_heap_then_sort_heap.bench.cpp
@@ -16,16 +16,14 @@ struct RangesMakeThenSortHeap {
   size_t Quantity;
 
   void run(benchmark::State& state) const {
-    runOpOnCopies<ValueType>(state, Quantity, Order(), BatchSize::CountElements,
-                             [](auto& Copy) {
-                               std::ranges::make_heap(Copy);
-                               std::ranges::sort_heap(Copy);
-                             });
+    runOpOnCopies<ValueType>(state, Quantity, Order(), BatchSize::CountElements, [](auto& Copy) {
+      std::ranges::make_heap(Copy);
+      std::ranges::sort_heap(Copy);
+    });
   }
 
   std::string name() const {
-    return "BM_RangesMakeThenSortHeap" + ValueType::name() + Order::name() + "_" +
-           std::to_string(Quantity);
+    return "BM_RangesMakeThenSortHeap" + ValueType::name() + Order::name() + "_" + std::to_string(Quantity);
   };
 };
 } // namespace
diff --git a/libcxx/benchmarks/algorithms/ranges_pop_heap.bench.cpp b/libcxx/benchmarks/algorithms/ranges_pop_heap.bench.cpp
index b26ccdd97a7d7..bcc7a834a655a 100644
--- a/libcxx/benchmarks/algorithms/ranges_pop_heap.bench.cpp
+++ b/libcxx/benchmarks/algorithms/ranges_pop_heap.bench.cpp
@@ -16,17 +16,14 @@ struct RangesPopHeap {
   size_t Quantity;
 
   void run(benchmark::State& state) const {
-    runOpOnCopies<ValueType>(
-        state, Quantity, Order(), BatchSize::CountElements, [](auto& Copy) {
-          for (auto B = Copy.begin(), I = Copy.end(); I != B; --I) {
-            std::ranges::pop_heap(B, I);
-          }
-        });
+    runOpOnCopies<ValueType>(state, Quantity, Order(), BatchSize::CountElements, [](auto& Copy) {
+      for (auto B = Copy.begin(), I = Copy.end(); I != B; --I) {
+        std::ranges::pop_heap(B, I);
+      }
+    });
   }
 
-  std::string name() const {
-    return "BM_RangesPopHeap" + ValueType::name() + "_" + std::to_string(Quantity);
-  };
+  std::string name() const { return "BM_RangesPopHeap" + ValueType::name() + "_" + std::to_string(Quantity); };
 };
 } // namespace
 
diff --git a/libcxx/benchmarks/algorithms/ranges_push_heap.bench.cpp b/libcxx/benchmarks/algorithms/ranges_push_heap.bench.cpp
index 28ccf41b96488..902f481228e46 100644
--- a/libcxx/benchmarks/algorithms/ranges_push_heap.bench.cpp
+++ b/libcxx/benchmarks/algorithms/ranges_push_heap.bench.cpp
@@ -16,19 +16,17 @@ struct RangesPushHeap {
   size_t Quantity;
 
   void run(benchmark::State& state) const {
-    runOpOnCopies<ValueType>(
-        state, Quantity, Order(), BatchSize::CountElements, [](auto& Copy) {
-          for (auto I = Copy.begin(), E = Copy.end(); I != E; ++I) {
-            std::ranges::push_heap(Copy.begin(), I + 1);
-          }
-        });
+    runOpOnCopies<ValueType>(state, Quantity, Order(), BatchSize::CountElements, [](auto& Copy) {
+      for (auto I = Copy.begin(), E = Copy.end(); I != E; ++I) {
+        std::ranges::push_heap(Copy.begin(), I + 1);
+      }
+    });
   }
 
   bool skip() const { return Order() == ::Order::Heap; }
 
   std::string name() const {
-    return "BM_RangesPushHeap" + ValueType::name() + Order::name() + "_" +
-           std::to_string(Quantity);
+    return "BM_RangesPushHeap" + ValueType::name() + Order::name() + "_" + std::to_string(Quantity);
   };
 };
 } // namespace
diff --git a/libcxx/benchmarks/algorithms/ranges_sort.bench.cpp b/libcxx/benchmarks/algorithms/ranges_sort.bench.cpp
index b9299d3fe1449..aeb2aedd34af3 100644
--- a/libcxx/benchmarks/algorithms/ranges_sort.bench.cpp
+++ b/libcxx/benchmarks/algorithms/ranges_sort.bench.cpp
@@ -16,16 +16,15 @@ struct Sort {
   size_t Quantity;
 
   void run(benchmark::State& state) const {
-    runOpOnCopies<ValueType>(
-        state, Quantity, Order(), BatchSize::CountElements,
-        [](auto& Copy) { std::ranges::sort(Copy); });
+    runOpOnCopies<ValueType>(state, Quantity, Order(), BatchSize::CountElements, [](auto& Copy) {
+      std::ranges::sort(Copy);
+    });
   }
 
   bool skip() const { return Order() == ::Order::Heap; }
 
   std::string name() const {
-    return "BM_RangesSort" + ValueType::name() + Order::name() + "_" +
-           std::to_string(Quantity);
+    return "BM_RangesSort" + ValueType::name() + Order::name() + "_" + std::to_string(Quantity);
   }
 };
 } // namespace
diff --git a/libcxx/benchmarks/algorithms/ranges_sort_heap.bench.cpp b/libcxx/benchmarks/algorithms/ranges_sort_heap.bench.cpp
index 12828a029582f..62c607cf7a5ed 100644
--- a/libcxx/benchmarks/algorithms/ranges_sort_heap.bench.cpp
+++ b/libcxx/benchmarks/algorithms/ranges_sort_heap.bench.cpp
@@ -16,14 +16,12 @@ struct RangesSortHeap {
   size_t Quantity;
 
   void run(benchmark::State& state) const {
-    runOpOnCopies<ValueType>(
-        state, Quantity, Order::Heap, BatchSize::CountElements,
-        [](auto& Copy) { std::ranges::sort_heap(Copy); });
+    runOpOnCopies<ValueType>(state, Quantity, Order::Heap, BatchSize::CountElements, [](auto& Copy) {
+      std::ranges::sort_heap(Copy);
+    });
   }
 
-  std::string name() const {
-    return "BM_RangesSortHeap" + ValueType::name() + "_" + std::to_string(Quantity);
-  };
+  std::string name() const { return "BM_RangesSortHeap" + ValueType::name() + "_" + std::to_string(Quantity); };
 };
 } // namespace
 
diff --git a/libcxx/benchmarks/algorithms/ranges_stable_sort.bench.cpp b/libcxx/benchmarks/algorithms/ranges_stable_sort.bench.cpp
index 94f80032afcb1..88327483cf43c 100644
--- a/libcxx/benchmarks/algorithms/ranges_stable_sort.bench.cpp
+++ b/libcxx/benchmarks/algorithms/ranges_stable_sort.bench.cpp
@@ -16,16 +16,15 @@ struct StableSort {
   size_t Quantity;
 
   void run(benchmark::State& state) const {
-    runOpOnCopies<ValueType>(
-        state, Quantity, Order(), BatchSize::CountElements,
-        [](auto& Copy) { std::ranges::stable_sort(Copy); });
+    runOpOnCopies<ValueType>(state, Quantity, Order(), BatchSize::CountElements, [](auto& Copy) {
+      std::ranges::stable_sort(Copy);
+    });
   }
 
   bool skip() const { return Order() == ::Order::Heap; }
 
   std::string name() const {
-    return "BM_RangesStableSort" + ValueType::name() + Order::name() + "_" +
-           std::to_string(Quantity);
+    return "BM_RangesStableSort" + ValueType::name() + Order::name() + "_" + std::to_string(Quantity);
   }
 };
 } // namespace
diff --git a/libcxx/benchmarks/algorithms/sort.bench.cpp b/libcxx/benchmarks/algorithms/sort.bench.cpp
index 6f8b12efe4531..f87434b779712 100644
--- a/libcxx/benchmarks/algorithms/sort.bench.cpp
+++ b/libcxx/benchmarks/algorithms/sort.bench.cpp
@@ -16,17 +16,14 @@ struct Sort {
   size_t Quantity;
 
   void run(benchmark::State& state) const {
-    runOpOnCopies<ValueType>(
-        state, Quantity, Order(), BatchSize::CountElements,
-        [](auto& Copy) { std::sort(Copy.begin(), Copy.end()); });
+    runOpOnCopies<ValueType>(state, Quantity, Order(), BatchSize::CountElements, [](auto& Copy) {
+      std::sort(Copy.begin(), Copy.end());
+    });
   }
 
   bool skip() const { return Order() == ::Order::Heap; }
 
-  std::string name() const {
-    return "BM_Sort" + ValueType::name() + Order::name() + "_" +
-           std::to_string(Quantity);
-  };
+  std::string name() const { return "BM_Sort" + ValueType::name() + Order::name() + "_" + std::to_string(Quantity); };
 };
 } // namespace
 
diff --git a/libcxx/benchmarks/algorithms/sort_heap.bench.cpp b/libcxx/benchmarks/algorithms/sort_heap.bench.cpp
index a217211a2f8ee..1372b4d5c9ab7 100644
--- a/libcxx/benchmarks/algorithms/sort_heap.bench.cpp
+++ b/libcxx/benchmarks/algorithms/sort_heap.bench.cpp
@@ -16,14 +16,12 @@ struct SortHeap {
   size_t Quantity;
 
   void run(benchmark::State& state) const {
-    runOpOnCopies<ValueType>(
-        state, Quantity, Order::Heap, BatchSize::CountElements,
-        [](auto& Copy) { std::sort_heap(Copy.begin(), Copy.end()); });
+    runOpOnCopies<ValueType>(state, Quantity, Order::Heap, BatchSize::CountElements, [](auto& Copy) {
+      std::sort_heap(Copy.begin(), Copy.end());
+    });
   }
 
-  std::string name() const {
-    return "BM_SortHeap" + ValueType::name() + "_" + std::to_string(Quantity);
-  };
+  std::string name() const { return "BM_SortHeap" + ValueType::name() + "_" + std::to_string(Quantity); };
 };
 } // namespace
 
diff --git a/libcxx/benchmarks/algorithms/stable_sort.bench.cpp b/libcxx/benchmarks/algorithms/stable_sort.bench.cpp
index 1ef55ce9fe956..fe88d01dd66f8 100644
--- a/libcxx/benchmarks/algorithms/stable_sort.bench.cpp
+++ b/libcxx/benchmarks/algorithms/stable_sort.bench.cpp
@@ -16,16 +16,15 @@ struct StableSort {
   size_t Quantity;
 
   void run(benchmark::State& state) const {
-    runOpOnCopies<ValueType>(
-        state, Quantity, Order(), BatchSize::CountElements,
-        [](auto& Copy) { std::stable_sort(Copy.begin(), Copy.end()); });
+    runOpOnCopies<ValueType>(state, Quantity, Order(), BatchSize::CountElements, [](auto& Copy) {
+      std::stable_sort(Copy.begin(), Copy.end());
+    });
   }
 
   bool skip() const { return Order() == ::Order::Heap; }
 
   std::string name() const {
-    return "BM_StableSort" + ValueType::name() + Order::name() + "_" +
-           std::to_string(Quantity);
+    return "BM_StableSort" + ValueType::name() + Order::name() + "_" + std::to_string(Quantity);
   };
 };
 } // namespace
diff --git a/libcxx/benchmarks/allocation.bench.cpp b/libcxx/benchmarks/allocation.bench.cpp
index ad962de5bf1fb..16be2b17fb70e 100644
--- a/libcxx/benchmarks/allocation.bench.cpp
+++ b/libcxx/benchmarks/allocation.bench.cpp
@@ -8,59 +8,34 @@
 
 #include "benchmark/benchmark.h"
 
+#include <cassert>
 #include <new>
 #include <vector>
-#include <cassert>
 
 struct PointerList {
   PointerList* Next = nullptr;
 };
 
 struct MallocWrapper {
-  __attribute__((always_inline))
-  static void* Allocate(size_t N) {
-    return std::malloc(N);
-  }
-  __attribute__((always_inline))
-  static void Deallocate(void* P, size_t) {
-    std::free(P);
-  }
+  __attribute__((always_inline)) static void* Allocate(size_t N) { return std::malloc(N); }
+  __attribute__((always_inline)) static void Deallocate(void* P, size_t) { std::free(P); }
 };
 
 struct NewWrapper {
-  __attribute__((always_inline))
-  static void* Allocate(size_t N) {
-    return ::operator new(N);
-  }
-  __attribute__((always_inline))
-  static void Deallocate(void* P, size_t) {
-    ::operator delete(P);
-  }
+  __attribute__((always_inline)) static void* Allocate(size_t N) { return ::operator new(N); }
+  __attribute__((always_inline)) static void Deallocate(void* P, size_t) { ::operator delete(P); }
 };
 
 struct BuiltinNewWrapper {
-  __attribute__((always_inline))
-  static void* Allocate(size_t N) {
-    return __builtin_operator_new(N);
-  }
-  __attribute__((always_inline))
-  static void Deallocate(void* P, size_t) {
-    __builtin_operator_delete(P);
-  }
+  __attribute__((always_inline)) static void* Allocate(size_t N) { return __builtin_operator_new(N); }
+  __attribute__((always_inline)) static void Deallocate(void* P, size_t) { __builtin_operator_delete(P); }
 };
 
 struct BuiltinSizedNewWrapper {
-  __attribute__((always_inline))
-  static void* Allocate(size_t N) {
-    return __builtin_operator_new(N);
-  }
-  __attribute__((always_inline))
-  static void Deallocate(void* P, size_t N) {
-    __builtin_operator_delete(P, N);
-  }
+  __attribute__((always_inline)) static void* Allocate(size_t N) { return __builtin_operator_new(N); }
+  __attribute__((always_inline)) static void Deallocate(void* P, size_t N) { __builtin_operator_delete(P, N); }
 };
 
-
 template <class AllocWrapper>
 static void BM_AllocateAndDeallocate(benchmark::State& st) {
   const size_t alloc_size = st.range(0);
@@ -71,23 +46,22 @@ static void BM_AllocateAndDeallocate(benchmark::State& st) {
   }
 }
 
-
 template <class AllocWrapper>
 static void BM_AllocateOnly(benchmark::State& st) {
   const size_t alloc_size = st.range(0);
-  PointerList *Start = nullptr;
+  PointerList* Start      = nullptr;
 
   while (st.KeepRunning()) {
     PointerList* p = (PointerList*)AllocWrapper::Allocate(alloc_size);
     benchmark::DoNotOptimize(p);
     p->Next = Start;
-    Start = p;
+    Start   = p;
   }
 
-  PointerList *Next = Start;
+  PointerList* Next = Start;
   while (Next) {
-    PointerList *Tmp = Next;
-    Next = Tmp->Next;
+    PointerList* Tmp = Next;
+    Next             = Tmp->Next;
     AllocWrapper::Deallocate(Tmp, alloc_size);
   }
 }
@@ -95,14 +69,14 @@ static void BM_AllocateOnly(benchmark::State& st) {
 template <class AllocWrapper>
 static void BM_DeallocateOnly(benchmark::State& st) {
   const size_t alloc_size = st.range(0);
-  const auto NumAllocs = st.max_iterations;
+  const auto NumAllocs    = st.max_iterations;
 
   std::vector<void*> Pointers(NumAllocs);
   for (auto& p : Pointers) {
     p = AllocWrapper::Allocate(alloc_size);
   }
 
-  void** Data = Pointers.data();
+  void** Data      = Pointers.data();
   void** const End = Pointers.data() + Pointers.size();
   while (st.KeepRunning()) {
     AllocWrapper::Deallocate(*Data, alloc_size);
@@ -112,7 +86,7 @@ static void BM_DeallocateOnly(benchmark::State& st) {
 }
 
 static int RegisterAllocBenchmarks() {
-  using FnType = void(*)(benchmark::State&);
+  using FnType = void (*)(benchmark::State&);
   struct {
     const char* name;
     FnType func;
diff --git a/libcxx/benchmarks/deque.bench.cpp b/libcxx/benchmarks/deque.bench.cpp
index 61b45cbf4ecd7..ced2ba230f6c8 100644
--- a/libcxx/benchmarks/deque.bench.cpp
+++ b/libcxx/benchmarks/deque.bench.cpp
@@ -17,30 +17,17 @@ using namespace ContainerBenchmarks;
 
 constexpr std::size_t TestNumInputs = 1024;
 
-BENCHMARK_CAPTURE(BM_ConstructSize,
-    deque_byte,
-    std::deque<unsigned char>{})->Arg(5140480);
+BENCHMARK_CAPTURE(BM_ConstructSize, deque_byte, std::deque<unsigned char>{})->Arg(5140480);
 
-BENCHMARK_CAPTURE(BM_ConstructSizeValue,
-    deque_byte,
-    std::deque<unsigned char>{}, 0)->Arg(5140480);
-
-BENCHMARK_CAPTURE(BM_ConstructIterIter,
-  deque_char,
-  std::deque<char>{},
-  getRandomIntegerInputs<char>)->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_ConstructIterIter,
-  deque_size_t,
-  std::deque<size_t>{},
-  getRandomIntegerInputs<size_t>)->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_ConstructIterIter,
-  deque_string,
-  std::deque<std::string>{},
-  getRandomStringInputs)->Arg(TestNumInputs);
+BENCHMARK_CAPTURE(BM_ConstructSizeValue, deque_byte, std::deque<unsigned char>{}, 0)->Arg(5140480);
 
+BENCHMARK_CAPTURE(BM_ConstructIterIter, deque_char, std::deque<char>{}, getRandomIntegerInputs<char>)
+    ->Arg(TestNumInputs);
 
+BENCHMARK_CAPTURE(BM_ConstructIterIter, deque_size_t, std::deque<size_t>{}, getRandomIntegerInputs<size_t>)
+    ->Arg(TestNumInputs);
 
+BENCHMARK_CAPTURE(BM_ConstructIterIter, deque_string, std::deque<std::string>{}, getRandomStringInputs)
+    ->Arg(TestNumInputs);
 
 BENCHMARK_MAIN();
diff --git a/libcxx/benchmarks/filesystem.bench.cpp b/libcxx/benchmarks/filesystem.bench.cpp
index 4fe4fc503ca13..44200fb3cab08 100644
--- a/libcxx/benchmarks/filesystem.bench.cpp
+++ b/libcxx/benchmarks/filesystem.bench.cpp
@@ -1,13 +1,12 @@
-#include "benchmark/benchmark.h"
 #include "GenerateInput.h"
-#include "test_iterators.h"
+#include "benchmark/benchmark.h"
 #include "filesystem_include.h"
+#include "test_iterators.h"
 
 static const size_t TestNumInputs = 1024;
 
-
 template <class GenInputs>
-void BM_PathConstructString(benchmark::State &st, GenInputs gen) {
+void BM_PathConstructString(benchmark::State& st, GenInputs gen) {
   using fs::path;
   const auto in = gen(st.range(0));
   path PP;
@@ -20,12 +19,10 @@ void BM_PathConstructString(benchmark::State &st, GenInputs gen) {
   }
   st.SetComplexityN(st.range(0));
 }
-BENCHMARK_CAPTURE(BM_PathConstructString, large_string,
-  getRandomStringInputs)->Range(8, TestNumInputs)->Complexity();
-
+BENCHMARK_CAPTURE(BM_PathConstructString, large_string, getRandomStringInputs)->Range(8, TestNumInputs)->Complexity();
 
 template <class GenInputs>
-void BM_PathConstructCStr(benchmark::State &st, GenInputs gen) {
+void BM_PathConstructCStr(benchmark::State& st, GenInputs gen) {
   using fs::path;
   const auto in = gen(st.range(0));
   path PP;
@@ -37,20 +34,18 @@ void BM_PathConstructCStr(benchmark::State &st, GenInputs gen) {
     benchmark::DoNotOptimize(P.native().data());
   }
 }
-BENCHMARK_CAPTURE(BM_PathConstructCStr, large_string,
-  getRandomStringInputs)->Arg(TestNumInputs);
-
+BENCHMARK_CAPTURE(BM_PathConstructCStr, large_string, getRandomStringInputs)->Arg(TestNumInputs);
 
 template <template <class...> class ItType, class GenInputs>
-void BM_PathConstructIter(benchmark::State &st, GenInputs gen) {
+void BM_PathConstructIter(benchmark::State& st, GenInputs gen) {
   using fs::path;
-  using Iter = ItType<std::string::const_iterator>;
+  using Iter    = ItType<std::string::const_iterator>;
   const auto in = gen(st.range(0));
   path PP;
   for (auto& Part : in)
     PP /= Part;
   auto Start = Iter(PP.native().begin());
-  auto End = Iter(PP.native().end());
+  auto End   = Iter(PP.native().end());
   benchmark::DoNotOptimize(PP.native().data());
   benchmark::DoNotOptimize(Start);
   benchmark::DoNotOptimize(End);
@@ -61,21 +56,22 @@ void BM_PathConstructIter(benchmark::State &st, GenInputs gen) {
   st.SetComplexityN(st.range(0));
 }
 template <class GenInputs>
-void BM_PathConstructInputIter(benchmark::State &st, GenInputs gen) {
+void BM_PathConstructInputIter(benchmark::State& st, GenInputs gen) {
   BM_PathConstructIter<cpp17_input_iterator>(st, gen);
 }
 template <class GenInputs>
-void BM_PathConstructForwardIter(benchmark::State &st, GenInputs gen) {
+void BM_PathConstructForwardIter(benchmark::State& st, GenInputs gen) {
   BM_PathConstructIter<forward_iterator>(st, gen);
 }
-BENCHMARK_CAPTURE(BM_PathConstructInputIter, large_string,
-  getRandomStringInputs)->Range(8, TestNumInputs)->Complexity();
-BENCHMARK_CAPTURE(BM_PathConstructForwardIter, large_string,
-  getRandomStringInputs)->Range(8, TestNumInputs)->Complexity();
-
+BENCHMARK_CAPTURE(BM_PathConstructInputIter, large_string, getRandomStringInputs)
+    ->Range(8, TestNumInputs)
+    ->Complexity();
+BENCHMARK_CAPTURE(BM_PathConstructForwardIter, large_string, getRandomStringInputs)
+    ->Range(8, TestNumInputs)
+    ->Complexity();
 
 template <class GenInputs>
-void BM_PathIterateMultipleTimes(benchmark::State &st, GenInputs gen) {
+void BM_PathIterateMultipleTimes(benchmark::State& st, GenInputs gen) {
   using fs::path;
   const auto in = gen(st.range(0));
   path PP;
@@ -90,12 +86,12 @@ void BM_PathIterateMultipleTimes(benchmark::State &st, GenInputs gen) {
   }
   st.SetComplexityN(st.range(0));
 }
-BENCHMARK_CAPTURE(BM_PathIterateMultipleTimes, iterate_elements,
-  getRandomStringInputs)->Range(8, TestNumInputs)->Complexity();
-
+BENCHMARK_CAPTURE(BM_PathIterateMultipleTimes, iterate_elements, getRandomStringInputs)
+    ->Range(8, TestNumInputs)
+    ->Complexity();
 
 template <class GenInputs>
-void BM_PathIterateOnce(benchmark::State &st, GenInputs gen) {
+void BM_PathIterateOnce(benchmark::State& st, GenInputs gen) {
   using fs::path;
   const auto in = gen(st.range(0));
   path PP;
@@ -111,11 +107,10 @@ void BM_PathIterateOnce(benchmark::State &st, GenInputs gen) {
   }
   st.SetComplexityN(st.range(0));
 }
-BENCHMARK_CAPTURE(BM_PathIterateOnce, iterate_elements,
-  getRandomStringInputs)->Range(8, TestNumInputs)->Complexity();
+BENCHMARK_CAPTURE(BM_PathIterateOnce, iterate_elements, getRandomStringInputs)->Range(8, TestNumInputs)->Complexity();
 
 template <class GenInputs>
-void BM_PathIterateOnceBackwards(benchmark::State &st, GenInputs gen) {
+void BM_PathIterateOnceBackwards(benchmark::State& st, GenInputs gen) {
   using fs::path;
   const auto in = gen(st.range(0));
   path PP;
@@ -125,7 +120,7 @@ void BM_PathIterateOnceBackwards(benchmark::State &st, GenInputs gen) {
   while (st.KeepRunning()) {
     const path P = PP.native();
     const auto B = P.begin();
-    auto I = P.end();
+    auto I       = P.end();
     while (I != B) {
       --I;
       benchmark::DoNotOptimize(*I);
@@ -133,8 +128,7 @@ void BM_PathIterateOnceBackwards(benchmark::State &st, GenInputs gen) {
     benchmark::DoNotOptimize(*I);
   }
 }
-BENCHMARK_CAPTURE(BM_PathIterateOnceBackwards, iterate_elements,
-  getRandomStringInputs)->Arg(TestNumInputs);
+BENCHMARK_CAPTURE(BM_PathIterateOnceBackwards, iterate_elements, getRandomStringInputs)->Arg(TestNumInputs);
 
 static fs::path getRandomPaths(int NumParts, int PathLen) {
   fs::path Result;
@@ -146,7 +140,7 @@ static fs::path getRandomPaths(int NumParts, int PathLen) {
 }
 
 template <class GenInput>
-void BM_LexicallyNormal(benchmark::State &st, GenInput gen, size_t PathLen) {
+void BM_LexicallyNormal(benchmark::State& st, GenInput gen, size_t PathLen) {
   using fs::path;
   auto In = gen(st.range(0), PathLen);
   benchmark::DoNotOptimize(&In);
@@ -155,9 +149,13 @@ void BM_LexicallyNormal(benchmark::State &st, GenInput gen, size_t PathLen) {
   }
   st.SetComplexityN(st.range(0));
 }
-BENCHMARK_CAPTURE(BM_LexicallyNormal, small_path,
-  getRandomPaths, /*PathLen*/5)->RangeMultiplier(2)->Range(2, 256)->Complexity();
-BENCHMARK_CAPTURE(BM_LexicallyNormal, large_path,
-  getRandomPaths, /*PathLen*/32)->RangeMultiplier(2)->Range(2, 256)->Complexity();
+BENCHMARK_CAPTURE(BM_LexicallyNormal, small_path, getRandomPaths, /*PathLen*/ 5)
+    ->RangeMultiplier(2)
+    ->Range(2, 256)
+    ->Complexity();
+BENCHMARK_CAPTURE(BM_LexicallyNormal, large_path, getRandomPaths, /*PathLen*/ 32)
+    ->RangeMultiplier(2)
+    ->Range(2, 256)
+    ->Complexity();
 
 BENCHMARK_MAIN();
diff --git a/libcxx/benchmarks/format_to.bench.cpp b/libcxx/benchmarks/format_to.bench.cpp
index 4d6489786b928..4e6041d448f50 100644
--- a/libcxx/benchmarks/format_to.bench.cpp
+++ b/libcxx/benchmarks/format_to.bench.cpp
@@ -7,9 +7,9 @@
 
 #include <format>
 
-#include <iterator>
 #include <algorithm>
 #include <array>
+#include <iterator>
 #include <list>
 #include <span>
 #include <string>
@@ -26,7 +26,7 @@ template <class Container>
 static void BM_format_to_string_back_inserter(benchmark::State& state) {
   using CharT = typename Container::value_type;
   size_t size = state.range(0);
-  auto str = std::basic_string<CharT>(size, CharT('*'));
+  auto str    = std::basic_string<CharT>(size, CharT('*'));
 
   for (auto _ : state) {
     Container output;
@@ -41,7 +41,7 @@ template <class Container>
 static void BM_format_to_string_begin(benchmark::State& state) {
   using CharT = typename Container::value_type;
   size_t size = state.range(0);
-  auto str = std::basic_string<CharT>(size, CharT('*'));
+  auto str    = std::basic_string<CharT>(size, CharT('*'));
 
   Container output(size, CharT('-'));
   for (auto _ : state)
@@ -55,7 +55,7 @@ static void BM_format_to_string_begin(benchmark::State& state) {
 template <class CharT>
 static void BM_format_to_string_span(benchmark::State& state) {
   size_t size = state.range(0);
-  auto str = std::basic_string<CharT>(size, CharT('*'));
+  auto str    = std::basic_string<CharT>(size, CharT('*'));
 
   auto buffer = std::basic_string<CharT>(size, CharT('-'));
   std::span<CharT> output{buffer};
@@ -68,9 +68,9 @@ static void BM_format_to_string_span(benchmark::State& state) {
 template <class CharT>
 static void BM_format_to_string_pointer(benchmark::State& state) {
   size_t size = state.range(0);
-  auto str = std::basic_string<CharT>(size, CharT('*'));
+  auto str    = std::basic_string<CharT>(size, CharT('*'));
 
-  auto buffer = std::basic_string<CharT>(size, CharT('-'));
+  auto buffer   = std::basic_string<CharT>(size, CharT('-'));
   CharT* output = buffer.data();
   for (auto _ : state)
     benchmark::DoNotOptimize(std::format_to(output, CSTR("{}"), str));
diff --git a/libcxx/benchmarks/format_to_n.bench.cpp b/libcxx/benchmarks/format_to_n.bench.cpp
index c1e52b0104b32..f5816d49f6dd5 100644
--- a/libcxx/benchmarks/format_to_n.bench.cpp
+++ b/libcxx/benchmarks/format_to_n.bench.cpp
@@ -7,9 +7,9 @@
 
 #include <format>
 
-#include <iterator>
 #include <algorithm>
 #include <array>
+#include <iterator>
 #include <list>
 #include <span>
 #include <string>
@@ -26,7 +26,7 @@ template <class Container>
 static void BM_format_to_n_string_back_inserter(benchmark::State& state) {
   using CharT = typename Container::value_type;
   size_t size = state.range(0);
-  auto str = std::basic_string<CharT>(2 * size, CharT('*'));
+  auto str    = std::basic_string<CharT>(2 * size, CharT('*'));
 
   for (auto _ : state) {
     Container output;
@@ -41,7 +41,7 @@ template <class Container>
 static void BM_format_to_n_string_begin(benchmark::State& state) {
   using CharT = typename Container::value_type;
   size_t size = state.range(0);
-  auto str = std::basic_string<CharT>(2 * size, CharT('*'));
+  auto str    = std::basic_string<CharT>(2 * size, CharT('*'));
 
   Container output(size, CharT('-'));
   for (auto _ : state)
@@ -55,7 +55,7 @@ static void BM_format_to_n_string_begin(benchmark::State& state) {
 template <class CharT>
 static void BM_format_to_n_string_span(benchmark::State& state) {
   size_t size = state.range(0);
-  auto str = std::basic_string<CharT>(2 * size, CharT('*'));
+  auto str    = std::basic_string<CharT>(2 * size, CharT('*'));
 
   auto buffer = std::basic_string<CharT>(size, CharT('-'));
   std::span<CharT> output{buffer};
@@ -68,9 +68,9 @@ static void BM_format_to_n_string_span(benchmark::State& state) {
 template <class CharT>
 static void BM_format_to_n_string_pointer(benchmark::State& state) {
   size_t size = state.range(0);
-  auto str = std::basic_string<CharT>(2 * size, CharT('*'));
+  auto str    = std::basic_string<CharT>(2 * size, CharT('*'));
 
-  auto buffer = std::basic_string<CharT>(size, CharT('-'));
+  auto buffer   = std::basic_string<CharT>(size, CharT('-'));
   CharT* output = buffer.data();
   for (auto _ : state)
     benchmark::DoNotOptimize(std::format_to_n(output, size, CSTR("{}"), str));
diff --git a/libcxx/benchmarks/formatter_float.bench.cpp b/libcxx/benchmarks/formatter_float.bench.cpp
index 09963b6108feb..e7a673bb5d655 100644
--- a/libcxx/benchmarks/formatter_float.bench.cpp
+++ b/libcxx/benchmarks/formatter_float.bench.cpp
@@ -107,8 +107,8 @@ enum class DisplayTypeE {
   General,
 };
 struct AllDisplayTypes : EnumValuesAsTuple<AllDisplayTypes, DisplayTypeE, 5> {
-  static constexpr const char* Names[] = {"DisplayDefault", "DisplayHex", "DisplayScientific", "DisplayFixed",
-                                          "DisplayGeneral"};
+  static constexpr const char* Names[] = {
+      "DisplayDefault", "DisplayHex", "DisplayScientific", "DisplayFixed", "DisplayGeneral"};
 };
 
 template <DisplayTypeE E>
@@ -142,8 +142,8 @@ struct DisplayType<DisplayTypeE::General> {
 // *** Alignment ***
 enum class AlignmentE { None, Left, Center, Right, ZeroPadding };
 struct AllAlignments : EnumValuesAsTuple<AllAlignments, AlignmentE, 5> {
-  static constexpr const char* Names[] = {"AlignNone", "AlignmentLeft", "AlignmentCenter", "AlignmentRight",
-                                          "ZeroPadding"};
+  static constexpr const char* Names[] = {
+      "AlignNone", "AlignmentLeft", "AlignmentCenter", "AlignmentRight", "ZeroPadding"};
 };
 
 template <AlignmentE E>
@@ -244,7 +244,12 @@ int main(int argc, char** argv) {
   if (benchmark::ReportUnrecognizedArguments(argc, argv))
     return 1;
 
-  makeCartesianProductBenchmark<FloatingPoint, AllLocalizations, AllDisplayTypes, AllTypes, AllValues, AllAlignments,
+  makeCartesianProductBenchmark<FloatingPoint,
+                                AllLocalizations,
+                                AllDisplayTypes,
+                                AllTypes,
+                                AllValues,
+                                AllAlignments,
                                 AllPrecisions>();
 
   benchmark::RunSpecifiedBenchmarks();
diff --git a/libcxx/benchmarks/formatter_int.bench.cpp b/libcxx/benchmarks/formatter_int.bench.cpp
index a42d96491a2b3..7cd794a4d4d66 100644
--- a/libcxx/benchmarks/formatter_int.bench.cpp
+++ b/libcxx/benchmarks/formatter_int.bench.cpp
@@ -9,14 +9,13 @@
 #include <format>
 #include <random>
 
-#include "benchmark/benchmark.h"
 #include "CartesianBenchmarks.h"
+#include "benchmark/benchmark.h"
 
 // Tests the full range of the value.
 template <class T>
-static std::array<T, 1000>
-generate(std::uniform_int_distribution<T> distribution = std::uniform_int_distribution<T>{
-             std::numeric_limits<T>::min(), std::numeric_limits<T>::max()}) {
+static std::array<T, 1000> generate(std::uniform_int_distribution<T> distribution = std::uniform_int_distribution<T>{
+                                        std::numeric_limits<T>::min(), std::numeric_limits<T>::max()}) {
   std::mt19937 generator;
   std::array<T, 1000> result;
   std::generate_n(result.begin(), result.size(), [&] { return distribution(generator); });
diff --git a/libcxx/benchmarks/function.bench.cpp b/libcxx/benchmarks/function.bench.cpp
index 7fabc937dd930..dd397bcfa5b07 100644
--- a/libcxx/benchmarks/function.bench.cpp
+++ b/libcxx/benchmarks/function.bench.cpp
@@ -29,14 +29,15 @@ enum class FunctionType {
 };
 
 struct AllFunctionTypes : EnumValuesAsTuple<AllFunctionTypes, FunctionType, 8> {
-  static constexpr const char* Names[] = {"Null",
-                                          "FuncPtr",
-                                          "MemFuncPtr",
-                                          "MemPtr",
-                                          "SmallTrivialFunctor",
-                                          "SmallNonTrivialFunctor",
-                                          "LargeTrivialFunctor",
-                                          "LargeNonTrivialFunctor"};
+  static constexpr const char* Names[] = {
+      "Null",
+      "FuncPtr",
+      "MemFuncPtr",
+      "MemPtr",
+      "SmallTrivialFunctor",
+      "SmallNonTrivialFunctor",
+      "LargeTrivialFunctor",
+      "LargeNonTrivialFunctor"};
 };
 
 enum class Opacity { kOpaque, kTransparent };
@@ -63,7 +64,7 @@ struct SmallNonTrivialFunctor {
 };
 struct LargeTrivialFunctor {
   LargeTrivialFunctor() {
-      // Do not spend time initializing the padding.
+    // Do not spend time initializing the padding.
   }
   int padding[16];
   int operator()(const S*) const { return 0; }
@@ -71,7 +72,7 @@ struct LargeTrivialFunctor {
 struct LargeNonTrivialFunctor {
   int padding[16];
   LargeNonTrivialFunctor() {
-      // Do not spend time initializing the padding.
+    // Do not spend time initializing the padding.
   }
   LargeNonTrivialFunctor(const LargeNonTrivialFunctor&) {}
   ~LargeNonTrivialFunctor() {}
@@ -83,22 +84,22 @@ using Function = std::function<int(const S*)>;
 TEST_ALWAYS_INLINE
 inline Function MakeFunction(FunctionType type, bool opaque = false) {
   switch (type) {
-    case FunctionType::Null:
-      return nullptr;
-    case FunctionType::FunctionPointer:
-      return maybeOpaque(FunctionWithS, opaque);
-    case FunctionType::MemberFunctionPointer:
-      return maybeOpaque(&S::function, opaque);
-    case FunctionType::MemberPointer:
-      return maybeOpaque(&S::field, opaque);
-    case FunctionType::SmallTrivialFunctor:
-      return maybeOpaque(SmallTrivialFunctor{}, opaque);
-    case FunctionType::SmallNonTrivialFunctor:
-      return maybeOpaque(SmallNonTrivialFunctor{}, opaque);
-    case FunctionType::LargeTrivialFunctor:
-      return maybeOpaque(LargeTrivialFunctor{}, opaque);
-    case FunctionType::LargeNonTrivialFunctor:
-      return maybeOpaque(LargeNonTrivialFunctor{}, opaque);
+  case FunctionType::Null:
+    return nullptr;
+  case FunctionType::FunctionPointer:
+    return maybeOpaque(FunctionWithS, opaque);
+  case FunctionType::MemberFunctionPointer:
+    return maybeOpaque(&S::function, opaque);
+  case FunctionType::MemberPointer:
+    return maybeOpaque(&S::field, opaque);
+  case FunctionType::SmallTrivialFunctor:
+    return maybeOpaque(SmallTrivialFunctor{}, opaque);
+  case FunctionType::SmallNonTrivialFunctor:
+    return maybeOpaque(SmallNonTrivialFunctor{}, opaque);
+  case FunctionType::LargeTrivialFunctor:
+    return maybeOpaque(LargeTrivialFunctor{}, opaque);
+  case FunctionType::LargeNonTrivialFunctor:
+    return maybeOpaque(LargeNonTrivialFunctor{}, opaque);
   }
 }
 
@@ -114,9 +115,7 @@ struct ConstructAndDestroy {
     }
   }
 
-  static std::string name() {
-    return "BM_ConstructAndDestroy" + FunctionType::name() + Opacity::name();
-  }
+  static std::string name() { return "BM_ConstructAndDestroy" + FunctionType::name() + Opacity::name(); }
 };
 
 template <class FunctionType>
@@ -125,7 +124,7 @@ struct Copy {
     auto value = MakeFunction(FunctionType());
     for (auto _ : state) {
       benchmark::DoNotOptimize(value);
-      auto copy = value;  // NOLINT
+      auto copy = value; // NOLINT
       benchmark::DoNotOptimize(copy);
     }
   }
@@ -137,7 +136,7 @@ template <class FunctionType>
 struct Move {
   static void run(benchmark::State& state) {
     Function values[2] = {MakeFunction(FunctionType())};
-    int i = 0;
+    int i              = 0;
     for (auto _ : state) {
       benchmark::DoNotOptimize(values);
       benchmark::DoNotOptimize(values[i ^ 1] = std::move(values[i]));
@@ -145,9 +144,7 @@ struct Move {
     }
   }
 
-  static std::string name() {
-    return "BM_Move" + FunctionType::name();
-  }
+  static std::string name() { return "BM_Move" + FunctionType::name(); }
 };
 
 template <class Function1, class Function2>
@@ -162,9 +159,7 @@ struct Swap {
 
   static bool skip() { return Function1() > Function2(); }
 
-  static std::string name() {
-    return "BM_Swap" + Function1::name() + Function2::name();
-  }
+  static std::string name() { return "BM_Swap" + Function1::name() + Function2::name(); }
 };
 
 template <class FunctionType>
@@ -207,20 +202,17 @@ struct InvokeInlined {
 
   static bool skip() { return FunctionType() == ::FunctionType::Null; }
 
-  static std::string name() {
-    return "BM_InvokeInlined" + FunctionType::name();
-  }
+  static std::string name() { return "BM_InvokeInlined" + FunctionType::name(); }
 };
 
-}  // namespace
+} // namespace
 
 int main(int argc, char** argv) {
   benchmark::Initialize(&argc, argv);
   if (benchmark::ReportUnrecognizedArguments(argc, argv))
     return 1;
 
-  makeCartesianProductBenchmark<ConstructAndDestroy, AllOpacity,
-                                AllFunctionTypes>();
+  makeCartesianProductBenchmark<ConstructAndDestroy, AllOpacity, AllFunctionTypes>();
   makeCartesianProductBenchmark<Copy, AllFunctionTypes>();
   makeCartesianProductBenchmark<Move, AllFunctionTypes>();
   makeCartesianProductBenchmark<Swap, AllFunctionTypes, AllFunctionTypes>();
diff --git a/libcxx/benchmarks/map.bench.cpp b/libcxx/benchmarks/map.bench.cpp
index dd1884f65032e..255164b22aa80 100644
--- a/libcxx/benchmarks/map.bench.cpp
+++ b/libcxx/benchmarks/map.bench.cpp
@@ -21,7 +21,7 @@
 // exists. To avoid errors in the benchmark these operations have a validation
 // mode to test the benchmark. Since they are not meant to be benchmarked the
 // number of sizes tested is limited to 1.
-//#define VALIDATE
+// #define VALIDATE
 
 namespace {
 
@@ -50,15 +50,12 @@ struct AllOrders : EnumValuesAsTuple<AllOrders, Order, 2> {
 struct TestSets {
   std::vector<uint64_t> Keys;
   std::vector<std::map<uint64_t, int64_t> > Maps;
-  std::vector<
-      std::vector<typename std::map<uint64_t, int64_t>::const_iterator> >
-      Hints;
+  std::vector<std::vector<typename std::map<uint64_t, int64_t>::const_iterator> > Hints;
 };
 
 enum class Shuffle { None, Keys, Hints };
 
-TestSets makeTestingSets(size_t MapSize, Mode mode, Shuffle shuffle,
-                         size_t max_maps) {
+TestSets makeTestingSets(size_t MapSize, Mode mode, Shuffle shuffle, size_t max_maps) {
   /*
    * The shuffle does not retain the random number generator to use the same
    * set of random numbers for every iteration.
@@ -74,7 +71,7 @@ TestSets makeTestingSets(size_t MapSize, Mode mode, Shuffle shuffle,
     std::shuffle(R.Keys.begin(), R.Keys.end(), std::mt19937());
 
   for (int M = 0; M < MapCount; ++M) {
-    auto& map = R.Maps.emplace_back();
+    auto& map   = R.Maps.emplace_back();
     auto& hints = R.Hints.emplace_back();
     for (uint64_t I = 0; I < MapSize; ++I) {
       hints.push_back(map.insert(std::make_pair(2 * I + 2, 0)).first);
@@ -115,8 +112,7 @@ struct ConstructorIterator : Base {
     auto& Map = Data.Maps.front();
     while (State.KeepRunningBatch(MapSize)) {
 #ifndef VALIDATE
-      benchmark::DoNotOptimize(
-          std::map<uint64_t, int64_t>(Map.begin(), Map.end()));
+      benchmark::DoNotOptimize(std::map<uint64_t, int64_t>(Map.begin(), Map.end()));
 #else
       std::map<uint64_t, int64_t> M{Map.begin(), Map.end()};
       if (M != Map)
@@ -238,9 +234,7 @@ struct Insert : Base {
   using Base::Base;
 
   void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(
-        MapSize, Mode(),
-        Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1000);
+    auto Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1000);
     while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
       for (auto& Map : Data.Maps) {
         for (auto K : Data.Keys) {
@@ -260,17 +254,12 @@ struct Insert : Base {
       }
 
       State.PauseTiming();
-      Data = makeTestingSets(MapSize, Mode(),
-                             Order::value == ::Order::Random ? Shuffle::Keys
-                                                             : Shuffle::None,
-                             1000);
+      Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1000);
       State.ResumeTiming();
     }
   }
 
-  std::string name() const {
-    return "BM_Insert" + baseName() + Mode::name() + Order::name();
-  }
+  std::string name() const { return "BM_Insert" + baseName() + Mode::name() + Order::name(); }
 };
 
 template <class Mode, class Hint>
@@ -278,13 +267,12 @@ struct InsertHint : Base {
   using Base::Base;
 
   template < ::Hint hint>
-  typename std::enable_if<hint == ::Hint::Correct>::type
-  run(benchmark::State& State) const {
+  typename std::enable_if<hint == ::Hint::Correct>::type run(benchmark::State& State) const {
     auto Data = makeTestingSets(MapSize, Mode(), Shuffle::None, 1000);
     while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
       for (size_t I = 0; I < Data.Maps.size(); ++I) {
         auto& Map = Data.Maps[I];
-        auto H = Data.Hints[I].begin();
+        auto H    = Data.Hints[I].begin();
         for (auto K : Data.Keys) {
 #ifndef VALIDATE
           benchmark::DoNotOptimize(Map.insert(*H, std::make_pair(K, 1)));
@@ -309,17 +297,14 @@ struct InsertHint : Base {
   }
 
   template < ::Hint hint>
-  typename std::enable_if<hint != ::Hint::Correct>::type
-  run(benchmark::State& State) const {
+  typename std::enable_if<hint != ::Hint::Correct>::type run(benchmark::State& State) const {
     auto Data = makeTestingSets(MapSize, Mode(), Shuffle::None, 1000);
     while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
       for (size_t I = 0; I < Data.Maps.size(); ++I) {
-        auto& Map = Data.Maps[I];
+        auto& Map  = Data.Maps[I];
         auto Third = *(Data.Hints[I].begin() + 2);
         for (auto K : Data.Keys) {
-          auto Itor = hint == ::Hint::Begin
-                          ? Map.begin()
-                          : hint == ::Hint::Third ? Third : Map.end();
+          auto Itor = hint == ::Hint::Begin ? Map.begin() : hint == ::Hint::Third ? Third : Map.end();
 #ifndef VALIDATE
           benchmark::DoNotOptimize(Map.insert(Itor, std::make_pair(K, 1)));
 #else
@@ -347,9 +332,7 @@ struct InsertHint : Base {
     run<h>(State);
   }
 
-  std::string name() const {
-    return "BM_InsertHint" + baseName() + Mode::name() + Hint::name();
-  }
+  std::string name() const { return "BM_InsertHint" + baseName() + Mode::name() + Hint::name(); }
 };
 
 template <class Mode, class Order>
@@ -357,9 +340,7 @@ struct InsertAssign : Base {
   using Base::Base;
 
   void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(
-        MapSize, Mode(),
-        Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1000);
+    auto Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1000);
     while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
       for (auto& Map : Data.Maps) {
         for (auto K : Data.Keys) {
@@ -379,17 +360,12 @@ struct InsertAssign : Base {
       }
 
       State.PauseTiming();
-      Data = makeTestingSets(MapSize, Mode(),
-                             Order::value == ::Order::Random ? Shuffle::Keys
-                                                             : Shuffle::None,
-                             1000);
+      Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1000);
       State.ResumeTiming();
     }
   }
 
-  std::string name() const {
-    return "BM_InsertAssign" + baseName() + Mode::name() + Order::name();
-  }
+  std::string name() const { return "BM_InsertAssign" + baseName() + Mode::name() + Order::name(); }
 };
 
 template <class Mode, class Hint>
@@ -397,13 +373,12 @@ struct InsertAssignHint : Base {
   using Base::Base;
 
   template < ::Hint hint>
-  typename std::enable_if<hint == ::Hint::Correct>::type
-  run(benchmark::State& State) const {
+  typename std::enable_if<hint == ::Hint::Correct>::type run(benchmark::State& State) const {
     auto Data = makeTestingSets(MapSize, Mode(), Shuffle::None, 1000);
     while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
       for (size_t I = 0; I < Data.Maps.size(); ++I) {
         auto& Map = Data.Maps[I];
-        auto H = Data.Hints[I].begin();
+        auto H    = Data.Hints[I].begin();
         for (auto K : Data.Keys) {
 #ifndef VALIDATE
           benchmark::DoNotOptimize(Map.insert_or_assign(*H, K, 1));
@@ -428,17 +403,14 @@ struct InsertAssignHint : Base {
   }
 
   template < ::Hint hint>
-  typename std::enable_if<hint != ::Hint::Correct>::type
-  run(benchmark::State& State) const {
+  typename std::enable_if<hint != ::Hint::Correct>::type run(benchmark::State& State) const {
     auto Data = makeTestingSets(MapSize, Mode(), Shuffle::None, 1000);
     while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
       for (size_t I = 0; I < Data.Maps.size(); ++I) {
-        auto& Map = Data.Maps[I];
+        auto& Map  = Data.Maps[I];
         auto Third = *(Data.Hints[I].begin() + 2);
         for (auto K : Data.Keys) {
-          auto Itor = hint == ::Hint::Begin
-                          ? Map.begin()
-                          : hint == ::Hint::Third ? Third : Map.end();
+          auto Itor = hint == ::Hint::Begin ? Map.begin() : hint == ::Hint::Third ? Third : Map.end();
 #ifndef VALIDATE
           benchmark::DoNotOptimize(Map.insert_or_assign(Itor, K, 1));
 #else
@@ -466,9 +438,7 @@ struct InsertAssignHint : Base {
     run<h>(State);
   }
 
-  std::string name() const {
-    return "BM_InsertAssignHint" + baseName() + Mode::name() + Hint::name();
-  }
+  std::string name() const { return "BM_InsertAssignHint" + baseName() + Mode::name() + Hint::name(); }
 };
 
 template <class Mode, class Order>
@@ -476,10 +446,7 @@ struct Emplace : Base {
   using Base::Base;
 
   void run(benchmark::State& State) const {
-
-    auto Data = makeTestingSets(
-        MapSize, Mode(),
-        Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1000);
+    auto Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1000);
     while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
       for (auto& Map : Data.Maps) {
         for (auto K : Data.Keys) {
@@ -499,17 +466,12 @@ struct Emplace : Base {
       }
 
       State.PauseTiming();
-      Data = makeTestingSets(MapSize, Mode(),
-                             Order::value == ::Order::Random ? Shuffle::Keys
-                                                             : Shuffle::None,
-                             1000);
+      Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1000);
       State.ResumeTiming();
     }
   }
 
-  std::string name() const {
-    return "BM_Emplace" + baseName() + Mode::name() + Order::name();
-  }
+  std::string name() const { return "BM_Emplace" + baseName() + Mode::name() + Order::name(); }
 };
 
 template <class Mode, class Hint>
@@ -517,13 +479,12 @@ struct EmplaceHint : Base {
   using Base::Base;
 
   template < ::Hint hint>
-  typename std::enable_if<hint == ::Hint::Correct>::type
-  run(benchmark::State& State) const {
+  typename std::enable_if<hint == ::Hint::Correct>::type run(benchmark::State& State) const {
     auto Data = makeTestingSets(MapSize, Mode(), Shuffle::None, 1000);
     while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
       for (size_t I = 0; I < Data.Maps.size(); ++I) {
         auto& Map = Data.Maps[I];
-        auto H = Data.Hints[I].begin();
+        auto H    = Data.Hints[I].begin();
         for (auto K : Data.Keys) {
 #ifndef VALIDATE
           benchmark::DoNotOptimize(Map.emplace_hint(*H, K, 1));
@@ -548,17 +509,14 @@ struct EmplaceHint : Base {
   }
 
   template < ::Hint hint>
-  typename std::enable_if<hint != ::Hint::Correct>::type
-  run(benchmark::State& State) const {
+  typename std::enable_if<hint != ::Hint::Correct>::type run(benchmark::State& State) const {
     auto Data = makeTestingSets(MapSize, Mode(), Shuffle::None, 1000);
     while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
       for (size_t I = 0; I < Data.Maps.size(); ++I) {
-        auto& Map = Data.Maps[I];
+        auto& Map  = Data.Maps[I];
         auto Third = *(Data.Hints[I].begin() + 2);
         for (auto K : Data.Keys) {
-          auto Itor = hint == ::Hint::Begin
-                          ? Map.begin()
-                          : hint == ::Hint::Third ? Third : Map.end();
+          auto Itor = hint == ::Hint::Begin ? Map.begin() : hint == ::Hint::Third ? Third : Map.end();
 #ifndef VALIDATE
           benchmark::DoNotOptimize(Map.emplace_hint(Itor, K, 1));
 #else
@@ -586,9 +544,7 @@ struct EmplaceHint : Base {
     run<h>(State);
   }
 
-  std::string name() const {
-    return "BM_EmplaceHint" + baseName() + Mode::name() + Hint::name();
-  }
+  std::string name() const { return "BM_EmplaceHint" + baseName() + Mode::name() + Hint::name(); }
 };
 
 template <class Mode, class Order>
@@ -596,10 +552,7 @@ struct TryEmplace : Base {
   using Base::Base;
 
   void run(benchmark::State& State) const {
-
-    auto Data = makeTestingSets(
-        MapSize, Mode(),
-        Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1000);
+    auto Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1000);
     while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
       for (auto& Map : Data.Maps) {
         for (auto K : Data.Keys) {
@@ -619,17 +572,12 @@ struct TryEmplace : Base {
       }
 
       State.PauseTiming();
-      Data = makeTestingSets(MapSize, Mode(),
-                             Order::value == ::Order::Random ? Shuffle::Keys
-                                                             : Shuffle::None,
-                             1000);
+      Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1000);
       State.ResumeTiming();
     }
   }
 
-  std::string name() const {
-    return "BM_TryEmplace" + baseName() + Mode::name() + Order::name();
-  }
+  std::string name() const { return "BM_TryEmplace" + baseName() + Mode::name() + Order::name(); }
 };
 
 template <class Mode, class Hint>
@@ -637,13 +585,12 @@ struct TryEmplaceHint : Base {
   using Base::Base;
 
   template < ::Hint hint>
-  typename std::enable_if<hint == ::Hint::Correct>::type
-  run(benchmark::State& State) const {
+  typename std::enable_if<hint == ::Hint::Correct>::type run(benchmark::State& State) const {
     auto Data = makeTestingSets(MapSize, Mode(), Shuffle::None, 1000);
     while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
       for (size_t I = 0; I < Data.Maps.size(); ++I) {
         auto& Map = Data.Maps[I];
-        auto H = Data.Hints[I].begin();
+        auto H    = Data.Hints[I].begin();
         for (auto K : Data.Keys) {
 #ifndef VALIDATE
           benchmark::DoNotOptimize(Map.try_emplace(*H, K, 1));
@@ -668,17 +615,14 @@ struct TryEmplaceHint : Base {
   }
 
   template < ::Hint hint>
-  typename std::enable_if<hint != ::Hint::Correct>::type
-  run(benchmark::State& State) const {
+  typename std::enable_if<hint != ::Hint::Correct>::type run(benchmark::State& State) const {
     auto Data = makeTestingSets(MapSize, Mode(), Shuffle::None, 1000);
     while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
       for (size_t I = 0; I < Data.Maps.size(); ++I) {
-        auto& Map = Data.Maps[I];
+        auto& Map  = Data.Maps[I];
         auto Third = *(Data.Hints[I].begin() + 2);
         for (auto K : Data.Keys) {
-          auto Itor = hint == ::Hint::Begin
-                          ? Map.begin()
-                          : hint == ::Hint::Third ? Third : Map.end();
+          auto Itor = hint == ::Hint::Begin ? Map.begin() : hint == ::Hint::Third ? Third : Map.end();
 #ifndef VALIDATE
           benchmark::DoNotOptimize(Map.try_emplace(Itor, K, 1));
 #else
@@ -706,9 +650,7 @@ struct TryEmplaceHint : Base {
     run<h>(State);
   }
 
-  std::string name() const {
-    return "BM_TryEmplaceHint" + baseName() + Mode::name() + Hint::name();
-  }
+  std::string name() const { return "BM_TryEmplaceHint" + baseName() + Mode::name() + Hint::name(); }
 };
 
 template <class Mode, class Order>
@@ -716,9 +658,7 @@ struct Erase : Base {
   using Base::Base;
 
   void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(
-        MapSize, Mode(),
-        Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1000);
+    auto Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1000);
     while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
       for (auto& Map : Data.Maps) {
         for (auto K : Data.Keys) {
@@ -738,17 +678,12 @@ struct Erase : Base {
       }
 
       State.PauseTiming();
-      Data = makeTestingSets(MapSize, Mode(),
-                             Order::value == ::Order::Random ? Shuffle::Keys
-                                                             : Shuffle::None,
-                             1000);
+      Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1000);
       State.ResumeTiming();
     }
   }
 
-  std::string name() const {
-    return "BM_Erase" + baseName() + Mode::name() + Order::name();
-  }
+  std::string name() const { return "BM_Erase" + baseName() + Mode::name() + Order::name(); }
 };
 
 template <class Order>
@@ -756,9 +691,8 @@ struct EraseIterator : Base {
   using Base::Base;
 
   void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(
-        MapSize, Mode::Hit,
-        Order::value == ::Order::Random ? Shuffle::Hints : Shuffle::None, 1000);
+    auto Data =
+        makeTestingSets(MapSize, Mode::Hit, Order::value == ::Order::Random ? Shuffle::Hints : Shuffle::None, 1000);
     while (State.KeepRunningBatch(MapSize * Data.Maps.size())) {
       for (size_t I = 0; I < Data.Maps.size(); ++I) {
         auto& Map = Data.Maps[I];
@@ -772,17 +706,13 @@ struct EraseIterator : Base {
       }
 
       State.PauseTiming();
-      Data = makeTestingSets(MapSize, Mode::Hit,
-                             Order::value == ::Order::Random ? Shuffle::Hints
-                                                             : Shuffle::None,
-                             1000);
+      Data =
+          makeTestingSets(MapSize, Mode::Hit, Order::value == ::Order::Random ? Shuffle::Hints : Shuffle::None, 1000);
       State.ResumeTiming();
     }
   }
 
-  std::string name() const {
-    return "BM_EraseIterator" + baseName() + Order::name();
-  }
+  std::string name() const { return "BM_EraseIterator" + baseName() + Order::name(); }
 };
 
 struct EraseRange : Base {
@@ -819,9 +749,7 @@ struct Count : Base {
   using Base::Base;
 
   void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(
-        MapSize, Mode(),
-        Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1);
+    auto Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1);
     auto& Map = Data.Maps.front();
     while (State.KeepRunningBatch(MapSize)) {
       for (auto K : Data.Keys) {
@@ -841,9 +769,7 @@ struct Count : Base {
     }
   }
 
-  std::string name() const {
-    return "BM_Count" + baseName() + Mode::name() + Order::name();
-  }
+  std::string name() const { return "BM_Count" + baseName() + Mode::name() + Order::name(); }
 };
 
 template <class Mode, class Order>
@@ -851,9 +777,7 @@ struct Find : Base {
   using Base::Base;
 
   void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(
-        MapSize, Mode(),
-        Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1);
+    auto Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1);
     auto& Map = Data.Maps.front();
     while (State.KeepRunningBatch(MapSize)) {
       for (auto K : Data.Keys) {
@@ -873,9 +797,7 @@ struct Find : Base {
     }
   }
 
-  std::string name() const {
-    return "BM_Find" + baseName() + Mode::name() + Order::name();
-  }
+  std::string name() const { return "BM_Find" + baseName() + Mode::name() + Order::name(); }
 };
 
 template <class Mode, class Order>
@@ -883,9 +805,7 @@ struct EqualRange : Base {
   using Base::Base;
 
   void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(
-        MapSize, Mode(),
-        Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1);
+    auto Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1);
     auto& Map = Data.Maps.front();
     while (State.KeepRunningBatch(MapSize)) {
       for (auto K : Data.Keys) {
@@ -900,12 +820,12 @@ struct EqualRange : Base {
             --Range.second;
             Key -= 2;
           }
-          if (Range.first == Map.end() || Range.first->first != K ||
-              Range.second == Map.end() || Range.second->first - 2 != Key)
+          if (Range.first == Map.end() || Range.first->first != K || Range.second == Map.end() ||
+              Range.second->first - 2 != Key)
             State.SkipWithError("Did not find the existing element");
         } else {
-          if (Range.first == Map.end() || Range.first->first - 1 != K ||
-              Range.second == Map.end() || Range.second->first - 1 != K)
+          if (Range.first == Map.end() || Range.first->first - 1 != K || Range.second == Map.end() ||
+              Range.second->first - 1 != K)
             State.SkipWithError("Did find the non-existing element");
         }
 #endif
@@ -913,9 +833,7 @@ struct EqualRange : Base {
     }
   }
 
-  std::string name() const {
-    return "BM_EqualRange" + baseName() + Mode::name() + Order::name();
-  }
+  std::string name() const { return "BM_EqualRange" + baseName() + Mode::name() + Order::name(); }
 };
 
 template <class Mode, class Order>
@@ -923,9 +841,7 @@ struct LowerBound : Base {
   using Base::Base;
 
   void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(
-        MapSize, Mode(),
-        Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1);
+    auto Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1);
     auto& Map = Data.Maps.front();
     while (State.KeepRunningBatch(MapSize)) {
       for (auto K : Data.Keys) {
@@ -945,9 +861,7 @@ struct LowerBound : Base {
     }
   }
 
-  std::string name() const {
-    return "BM_LowerBound" + baseName() + Mode::name() + Order::name();
-  }
+  std::string name() const { return "BM_LowerBound" + baseName() + Mode::name() + Order::name(); }
 };
 
 template <class Mode, class Order>
@@ -955,9 +869,7 @@ struct UpperBound : Base {
   using Base::Base;
 
   void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(
-        MapSize, Mode(),
-        Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1);
+    auto Data = makeTestingSets(MapSize, Mode(), Order::value == ::Order::Random ? Shuffle::Keys : Shuffle::None, 1);
     auto& Map = Data.Maps.front();
     while (State.KeepRunningBatch(MapSize)) {
       for (auto K : Data.Keys) {
@@ -983,9 +895,7 @@ struct UpperBound : Base {
     }
   }
 
-  std::string name() const {
-    return "BM_UpperBound" + baseName() + Mode::name() + Order::name();
-  }
+  std::string name() const { return "BM_UpperBound" + baseName() + Mode::name() + Order::name(); }
 };
 
 } // namespace
diff --git a/libcxx/benchmarks/ordered_set.bench.cpp b/libcxx/benchmarks/ordered_set.bench.cpp
index a9d9c929ac64d..22540d8ab7e0c 100644
--- a/libcxx/benchmarks/ordered_set.bench.cpp
+++ b/libcxx/benchmarks/ordered_set.bench.cpp
@@ -28,8 +28,7 @@ struct AllHitTypes : EnumValuesAsTuple<AllHitTypes, HitType, 2> {
 
 enum class AccessPattern { Ordered, Random };
 
-struct AllAccessPattern
-    : EnumValuesAsTuple<AllAccessPattern, AccessPattern, 2> {
+struct AllAccessPattern : EnumValuesAsTuple<AllAccessPattern, AccessPattern, 2> {
   static constexpr const char* Names[] = {"Ordered", "Random"};
 };
 
@@ -46,8 +45,7 @@ struct TestSets {
   std::vector<uint64_t> Keys;
 };
 
-TestSets makeTestingSets(size_t TableSize, size_t NumTables, HitType Hit,
-                         AccessPattern Access) {
+TestSets makeTestingSets(size_t TableSize, size_t NumTables, HitType Hit, AccessPattern Access) {
   TestSets R;
   R.Sets.resize(1);
 
@@ -72,8 +70,7 @@ struct Base {
   }
 
   std::string baseName() const {
-    return "_TableSize" + std::to_string(TableSize) + "_NumTables" +
-           std::to_string(NumTables);
+    return "_TableSize" + std::to_string(TableSize) + "_NumTables" + std::to_string(NumTables);
   }
 };
 
@@ -96,9 +93,7 @@ struct Create : Base {
     }
   }
 
-  std::string name() const {
-    return "BM_Create" + Access::name() + baseName();
-  }
+  std::string name() const { return "BM_Create" + Access::name() + baseName(); }
 };
 
 template <class Hit, class Access>
@@ -117,9 +112,7 @@ struct Find : Base {
     }
   }
 
-  std::string name() const {
-    return "BM_Find" + Hit::name() + Access::name() + baseName();
-  }
+  std::string name() const { return "BM_Find" + Hit::name() + Access::name() + baseName(); }
 };
 
 template <class Hit, class Access>
@@ -138,9 +131,7 @@ struct FindNeEnd : Base {
     }
   }
 
-  std::string name() const {
-    return "BM_FindNeEnd" + Hit::name() + Access::name() + baseName();
-  }
+  std::string name() const { return "BM_FindNeEnd" + Hit::name() + Access::name() + baseName(); }
 };
 
 template <class Access>
@@ -159,9 +150,7 @@ struct InsertHit : Base {
     }
   }
 
-  std::string name() const {
-    return "BM_InsertHit" + Access::name() + baseName();
-  }
+  std::string name() const { return "BM_InsertHit" + Access::name() + baseName(); }
 };
 
 template <class Access>
@@ -180,17 +169,14 @@ struct InsertMissAndErase : Base {
     }
   }
 
-  std::string name() const {
-    return "BM_InsertMissAndErase" + Access::name() + baseName();
-  }
+  std::string name() const { return "BM_InsertMissAndErase" + Access::name() + baseName(); }
 };
 
 struct IterateRangeFor : Base {
   using Base::Base;
 
   void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(TableSize, NumTables, HitType::Miss,
-                                AccessPattern::Ordered);
+    auto Data = makeTestingSets(TableSize, NumTables, HitType::Miss, AccessPattern::Ordered);
 
     while (State.KeepRunningBatch(TableSize * NumTables)) {
       for (auto& Set : Data.Sets) {
@@ -208,8 +194,7 @@ struct IterateBeginEnd : Base {
   using Base::Base;
 
   void run(benchmark::State& State) const {
-    auto Data = makeTestingSets(TableSize, NumTables, HitType::Miss,
-                                AccessPattern::Ordered);
+    auto Data = makeTestingSets(TableSize, NumTables, HitType::Miss, AccessPattern::Ordered);
 
     while (State.KeepRunningBatch(TableSize * NumTables)) {
       for (auto& Set : Data.Sets) {
@@ -223,7 +208,7 @@ struct IterateBeginEnd : Base {
   std::string name() const { return "BM_IterateBeginEnd" + baseName(); }
 };
 
-}  // namespace
+} // namespace
 
 int main(int argc, char** argv) {
   benchmark::Initialize(&argc, argv);
@@ -234,14 +219,10 @@ int main(int argc, char** argv) {
   const std::vector<size_t> NumTables{1, 10, 100, 1000, 10000, 100000, 1000000};
 
   makeCartesianProductBenchmark<Create, AllAccessPattern>(TableSize, NumTables);
-  makeCartesianProductBenchmark<Find, AllHitTypes, AllAccessPattern>(
-      TableSize, NumTables);
-  makeCartesianProductBenchmark<FindNeEnd, AllHitTypes, AllAccessPattern>(
-      TableSize, NumTables);
-  makeCartesianProductBenchmark<InsertHit, AllAccessPattern>(
-      TableSize, NumTables);
-  makeCartesianProductBenchmark<InsertMissAndErase, AllAccessPattern>(
-      TableSize, NumTables);
+  makeCartesianProductBenchmark<Find, AllHitTypes, AllAccessPattern>(TableSize, NumTables);
+  makeCartesianProductBenchmark<FindNeEnd, AllHitTypes, AllAccessPattern>(TableSize, NumTables);
+  makeCartesianProductBenchmark<InsertHit, AllAccessPattern>(TableSize, NumTables);
+  makeCartesianProductBenchmark<InsertMissAndErase, AllAccessPattern>(TableSize, NumTables);
   makeCartesianProductBenchmark<IterateRangeFor>(TableSize, NumTables);
   makeCartesianProductBenchmark<IterateBeginEnd>(TableSize, NumTables);
   benchmark::RunSpecifiedBenchmarks();
diff --git a/libcxx/benchmarks/random.bench.cpp b/libcxx/benchmarks/random.bench.cpp
index e437849321da0..fe2eb6672ae79 100644
--- a/libcxx/benchmarks/random.bench.cpp
+++ b/libcxx/benchmarks/random.bench.cpp
@@ -14,7 +14,7 @@
 #include "benchmark/benchmark.h"
 
 constexpr std::size_t MAX_BUFFER_LEN = 256;
-constexpr std::size_t MAX_SEED_LEN = 16;
+constexpr std::size_t MAX_SEED_LEN   = 16;
 
 static void BM_SeedSeq_Generate(benchmark::State& state) {
   std::array<std::uint32_t, MAX_BUFFER_LEN> buffer;
diff --git a/libcxx/benchmarks/string.bench.cpp b/libcxx/benchmarks/string.bench.cpp
index e43ad3212fff5..92018b0e48f53 100644
--- a/libcxx/benchmarks/string.bench.cpp
+++ b/libcxx/benchmarks/string.bench.cpp
@@ -11,7 +11,7 @@
 constexpr std::size_t MAX_STRING_LEN = 8 << 14;
 
 // Benchmark when there is no match.
-static void BM_StringFindNoMatch(benchmark::State &state) {
+static void BM_StringFindNoMatch(benchmark::State& state) {
   std::string s1(state.range(0), '-');
   std::string s2(8, '*');
   for (auto _ : state)
@@ -20,7 +20,7 @@ static void BM_StringFindNoMatch(benchmark::State &state) {
 BENCHMARK(BM_StringFindNoMatch)->Range(10, MAX_STRING_LEN);
 
 // Benchmark when the string matches first time.
-static void BM_StringFindAllMatch(benchmark::State &state) {
+static void BM_StringFindAllMatch(benchmark::State& state) {
   std::string s1(MAX_STRING_LEN, '-');
   std::string s2(state.range(0), '-');
   for (auto _ : state)
@@ -29,7 +29,7 @@ static void BM_StringFindAllMatch(benchmark::State &state) {
 BENCHMARK(BM_StringFindAllMatch)->Range(1, MAX_STRING_LEN);
 
 // Benchmark when the string matches somewhere in the end.
-static void BM_StringFindMatch1(benchmark::State &state) {
+static void BM_StringFindMatch1(benchmark::State& state) {
   std::string s1(MAX_STRING_LEN / 2, '*');
   s1 += std::string(state.range(0), '-');
   std::string s2(state.range(0), '-');
@@ -39,7 +39,7 @@ static void BM_StringFindMatch1(benchmark::State &state) {
 BENCHMARK(BM_StringFindMatch1)->Range(1, MAX_STRING_LEN / 4);
 
 // Benchmark when the string matches somewhere from middle to the end.
-static void BM_StringFindMatch2(benchmark::State &state) {
+static void BM_StringFindMatch2(benchmark::State& state) {
   std::string s1(MAX_STRING_LEN / 2, '*');
   s1 += std::string(state.range(0), '-');
   s1 += std::string(state.range(0), '*');
@@ -49,7 +49,7 @@ static void BM_StringFindMatch2(benchmark::State &state) {
 }
 BENCHMARK(BM_StringFindMatch2)->Range(1, MAX_STRING_LEN / 4);
 
-static void BM_StringCtorDefault(benchmark::State &state) {
+static void BM_StringCtorDefault(benchmark::State& state) {
   for (auto _ : state) {
     std::string Default;
     benchmark::DoNotOptimize(Default);
@@ -69,40 +69,38 @@ struct AllOpacity : EnumValuesAsTuple<AllOpacity, Opacity, 2> {
 
 enum class DiffType { Control, ChangeFirst, ChangeMiddle, ChangeLast };
 struct AllDiffTypes : EnumValuesAsTuple<AllDiffTypes, DiffType, 4> {
-  static constexpr const char* Names[] = {"Control", "ChangeFirst",
-                                          "ChangeMiddle", "ChangeLast"};
+  static constexpr const char* Names[] = {"Control", "ChangeFirst", "ChangeMiddle", "ChangeLast"};
 };
 
 static constexpr char SmallStringLiteral[] = "012345678";
 
 TEST_ALWAYS_INLINE const char* getSmallString(DiffType D) {
   switch (D) {
-    case DiffType::Control:
-      return SmallStringLiteral;
-    case DiffType::ChangeFirst:
-      return "-12345678";
-    case DiffType::ChangeMiddle:
-      return "0123-5678";
-    case DiffType::ChangeLast:
-      return "01234567-";
+  case DiffType::Control:
+    return SmallStringLiteral;
+  case DiffType::ChangeFirst:
+    return "-12345678";
+  case DiffType::ChangeMiddle:
+    return "0123-5678";
+  case DiffType::ChangeLast:
+    return "01234567-";
   }
 }
 
-static constexpr char LargeStringLiteral[] =
-    "012345678901234567890123456789012345678901234567890123456789012";
+static constexpr char LargeStringLiteral[] = "012345678901234567890123456789012345678901234567890123456789012";
 
 TEST_ALWAYS_INLINE const char* getLargeString(DiffType D) {
 #define LARGE_STRING_FIRST "123456789012345678901234567890"
 #define LARGE_STRING_SECOND "234567890123456789012345678901"
   switch (D) {
-    case DiffType::Control:
-      return "0" LARGE_STRING_FIRST "1" LARGE_STRING_SECOND "2";
-    case DiffType::ChangeFirst:
-      return "-" LARGE_STRING_FIRST "1" LARGE_STRING_SECOND "2";
-    case DiffType::ChangeMiddle:
-      return "0" LARGE_STRING_FIRST "-" LARGE_STRING_SECOND "2";
-    case DiffType::ChangeLast:
-      return "0" LARGE_STRING_FIRST "1" LARGE_STRING_SECOND "-";
+  case DiffType::Control:
+    return "0" LARGE_STRING_FIRST "1" LARGE_STRING_SECOND "2";
+  case DiffType::ChangeFirst:
+    return "-" LARGE_STRING_FIRST "1" LARGE_STRING_SECOND "2";
+  case DiffType::ChangeMiddle:
+    return "0" LARGE_STRING_FIRST "-" LARGE_STRING_SECOND "2";
+  case DiffType::ChangeLast:
+    return "0" LARGE_STRING_FIRST "1" LARGE_STRING_SECOND "-";
   }
 }
 
@@ -113,19 +111,18 @@ TEST_ALWAYS_INLINE const char* getHugeString(DiffType D) {
 #define HUGE_STRING3 HUGE_STRING2 HUGE_STRING2 HUGE_STRING2 HUGE_STRING2
 #define HUGE_STRING4 HUGE_STRING3 HUGE_STRING3 HUGE_STRING3 HUGE_STRING3
   switch (D) {
-    case DiffType::Control:
-      return "0123456789" HUGE_STRING4 "0123456789" HUGE_STRING4 "0123456789";
-    case DiffType::ChangeFirst:
-      return "-123456789" HUGE_STRING4 "0123456789" HUGE_STRING4 "0123456789";
-    case DiffType::ChangeMiddle:
-      return "0123456789" HUGE_STRING4 "01234-6789" HUGE_STRING4 "0123456789";
-    case DiffType::ChangeLast:
-      return "0123456789" HUGE_STRING4 "0123456789" HUGE_STRING4 "012345678-";
+  case DiffType::Control:
+    return "0123456789" HUGE_STRING4 "0123456789" HUGE_STRING4 "0123456789";
+  case DiffType::ChangeFirst:
+    return "-123456789" HUGE_STRING4 "0123456789" HUGE_STRING4 "0123456789";
+  case DiffType::ChangeMiddle:
+    return "0123456789" HUGE_STRING4 "01234-6789" HUGE_STRING4 "0123456789";
+  case DiffType::ChangeLast:
+    return "0123456789" HUGE_STRING4 "0123456789" HUGE_STRING4 "012345678-";
   }
 }
 
-TEST_ALWAYS_INLINE const char* getString(Length L,
-                                         DiffType D = DiffType::Control) {
+TEST_ALWAYS_INLINE const char* getString(Length L, DiffType D = DiffType::Control) {
   switch (L) {
   case Length::Empty:
     return "";
@@ -138,9 +135,7 @@ TEST_ALWAYS_INLINE const char* getString(Length L,
   }
 }
 
-TEST_ALWAYS_INLINE std::string makeString(Length L,
-                                          DiffType D = DiffType::Control,
-                                          Opacity O = Opacity::Transparent) {
+TEST_ALWAYS_INLINE std::string makeString(Length L, DiffType D = DiffType::Control, Opacity O = Opacity::Transparent) {
   switch (L) {
   case Length::Empty:
     return maybeOpaque("", O == Opacity::Opaque);
@@ -157,20 +152,17 @@ template <class Length, class Opaque>
 struct StringConstructDestroyCStr {
   static void run(benchmark::State& state) {
     for (auto _ : state) {
-      benchmark::DoNotOptimize(
-          makeString(Length(), DiffType::Control, Opaque()));
+      benchmark::DoNotOptimize(makeString(Length(), DiffType::Control, Opaque()));
     }
   }
 
-  static std::string name() {
-    return "BM_StringConstructDestroyCStr" + Length::name() + Opaque::name();
-  }
+  static std::string name() { return "BM_StringConstructDestroyCStr" + Length::name() + Opaque::name(); }
 };
 
 template <class Length, bool MeasureCopy, bool MeasureDestroy>
 static void StringCopyAndDestroy(benchmark::State& state) {
   static constexpr size_t NumStrings = 1024;
-  auto Orig = makeString(Length());
+  auto Orig                          = makeString(Length());
   std::aligned_storage<sizeof(std::string)>::type Storage[NumStrings];
 
   while (state.KeepRunningBatch(NumStrings)) {
@@ -194,18 +186,14 @@ static void StringCopyAndDestroy(benchmark::State& state) {
 
 template <class Length>
 struct StringCopy {
-  static void run(benchmark::State& state) {
-    StringCopyAndDestroy<Length, true, false>(state);
-  }
+  static void run(benchmark::State& state) { StringCopyAndDestroy<Length, true, false>(state); }
 
   static std::string name() { return "BM_StringCopy" + Length::name(); }
 };
 
 template <class Length>
 struct StringDestroy {
-  static void run(benchmark::State& state) {
-    StringCopyAndDestroy<Length, false, true>(state);
-  }
+  static void run(benchmark::State& state) { StringCopyAndDestroy<Length, false, true>(state); }
 
   static std::string name() { return "BM_StringDestroy" + Length::name(); }
 };
@@ -215,15 +203,15 @@ struct StringMove {
   static void run(benchmark::State& state) {
     // Keep two object locations and move construct back and forth.
     std::aligned_storage<sizeof(std::string), alignof(std::string)>::type Storage[2];
-    using S = std::string;
+    using S  = std::string;
     size_t I = 0;
-    S *newS = new (static_cast<void*>(Storage)) std::string(makeString(Length()));
+    S* newS  = new (static_cast<void*>(Storage)) std::string(makeString(Length()));
     for (auto _ : state) {
       // Switch locations.
       I ^= 1;
       benchmark::DoNotOptimize(Storage);
       // Move construct into the new location,
-      S *tmpS = new (static_cast<void*>(Storage + I)) S(std::move(*newS));
+      S* tmpS = new (static_cast<void*>(Storage + I)) S(std::move(*newS));
       // then destroy the old one.
       newS->~S();
       newS = tmpS;
@@ -237,9 +225,9 @@ struct StringMove {
 template <class Length, class Opaque>
 struct StringResizeDefaultInit {
   static void run(benchmark::State& state) {
-    constexpr bool opaque = Opaque{} == Opacity::Opaque;
+    constexpr bool opaque     = Opaque{} == Opacity::Opaque;
     constexpr int kNumStrings = 4 << 10;
-    size_t length = makeString(Length()).size();
+    size_t length             = makeString(Length()).size();
     std::string strings[kNumStrings];
     while (state.KeepRunningBatch(kNumStrings)) {
       state.PauseTiming();
@@ -254,17 +242,15 @@ struct StringResizeDefaultInit {
     }
   }
 
-  static std::string name() {
-    return "BM_StringResizeDefaultInit" + Length::name() + Opaque::name();
-  }
+  static std::string name() { return "BM_StringResizeDefaultInit" + Length::name() + Opaque::name(); }
 };
 
 template <class Length, class Opaque>
 struct StringAssignStr {
   static void run(benchmark::State& state) {
-    constexpr bool opaque = Opaque{} == Opacity::Opaque;
+    constexpr bool opaque     = Opaque{} == Opacity::Opaque;
     constexpr int kNumStrings = 4 << 10;
-    std::string src = makeString(Length());
+    std::string src           = makeString(Length());
     std::string strings[kNumStrings];
     while (state.KeepRunningBatch(kNumStrings)) {
       state.PauseTiming();
@@ -279,15 +265,13 @@ struct StringAssignStr {
     }
   }
 
-  static std::string name() {
-    return "BM_StringAssignStr" + Length::name() + Opaque::name();
-  }
+  static std::string name() { return "BM_StringAssignStr" + Length::name() + Opaque::name(); }
 };
 
 template <class Length, class Opaque>
 struct StringAssignAsciiz {
   static void run(benchmark::State& state) {
-    constexpr bool opaque = Opaque{} == Opacity::Opaque;
+    constexpr bool opaque     = Opaque{} == Opacity::Opaque;
     constexpr int kNumStrings = 4 << 10;
     std::string strings[kNumStrings];
     while (state.KeepRunningBatch(kNumStrings)) {
@@ -303,15 +287,13 @@ struct StringAssignAsciiz {
     }
   }
 
-  static std::string name() {
-    return "BM_StringAssignAsciiz" + Length::name() + Opaque::name();
-  }
+  static std::string name() { return "BM_StringAssignAsciiz" + Length::name() + Opaque::name(); }
 };
 
 template <class Length, class Opaque>
 struct StringEraseToEnd {
   static void run(benchmark::State& state) {
-    constexpr bool opaque = Opaque{} == Opacity::Opaque;
+    constexpr bool opaque     = Opaque{} == Opacity::Opaque;
     constexpr int kNumStrings = 4 << 10;
     std::string strings[kNumStrings];
     const int mid = makeString(Length()).size() / 2;
@@ -323,24 +305,21 @@ struct StringEraseToEnd {
       benchmark::DoNotOptimize(strings);
       state.ResumeTiming();
       for (int i = 0; i < kNumStrings; ++i) {
-        strings[i].erase(maybeOpaque(mid, opaque),
-                         maybeOpaque(std::string::npos, opaque));
+        strings[i].erase(maybeOpaque(mid, opaque), maybeOpaque(std::string::npos, opaque));
       }
     }
   }
 
-  static std::string name() {
-    return "BM_StringEraseToEnd" + Length::name() + Opaque::name();
-  }
+  static std::string name() { return "BM_StringEraseToEnd" + Length::name() + Opaque::name(); }
 };
 
 template <class Length, class Opaque>
 struct StringEraseWithMove {
   static void run(benchmark::State& state) {
-    constexpr bool opaque = Opaque{} == Opacity::Opaque;
+    constexpr bool opaque     = Opaque{} == Opacity::Opaque;
     constexpr int kNumStrings = 4 << 10;
     std::string strings[kNumStrings];
-    const int n = makeString(Length()).size() / 2;
+    const int n   = makeString(Length()).size() / 2;
     const int pos = n / 2;
     while (state.KeepRunningBatch(kNumStrings)) {
       state.PauseTiming();
@@ -355,16 +334,14 @@ struct StringEraseWithMove {
     }
   }
 
-  static std::string name() {
-    return "BM_StringEraseWithMove" + Length::name() + Opaque::name();
-  }
+  static std::string name() { return "BM_StringEraseWithMove" + Length::name() + Opaque::name(); }
 };
 
 template <class Opaque>
 struct StringAssignAsciizMix {
   static void run(benchmark::State& state) {
-    constexpr auto O = Opaque{};
-    constexpr auto D = DiffType::Control;
+    constexpr auto O          = Opaque{};
+    constexpr auto D          = DiffType::Control;
     constexpr int kNumStrings = 4 << 10;
     std::string strings[kNumStrings];
     while (state.KeepRunningBatch(kNumStrings)) {
@@ -387,9 +364,7 @@ struct StringAssignAsciizMix {
     }
   }
 
-  static std::string name() {
-    return "BM_StringAssignAsciizMix" + Opaque::name();
-  }
+  static std::string name() { return "BM_StringAssignAsciizMix" + Opaque::name(); }
 };
 
 enum class Relation { Eq, Less, Compare };
@@ -433,8 +408,7 @@ struct StringRelational {
   }
 
   static std::string name() {
-    return "BM_StringRelational" + Rel::name() + LHLength::name() +
-           RHLength::name() + DiffType::name();
+    return "BM_StringRelational" + Rel::name() + LHLength::name() + RHLength::name() + DiffType::name();
   }
 };
 
@@ -444,11 +418,11 @@ struct StringRelationalLiteral {
     auto Lhs = makeString(LHLength(), DiffType());
     for (auto _ : state) {
       benchmark::DoNotOptimize(Lhs);
-      constexpr const char* Literal = RHLength::value == Length::Empty
-                                          ? ""
-                                          : RHLength::value == Length::Small
-                                                ? SmallStringLiteral
-                                                : LargeStringLiteral;
+      constexpr const char* Literal =
+          RHLength::value == Length::Empty ? ""
+          : RHLength::value == Length::Small
+              ? SmallStringLiteral
+              : LargeStringLiteral;
       switch (Rel()) {
       case Relation::Eq:
         benchmark::DoNotOptimize(Lhs == Literal);
@@ -474,8 +448,7 @@ struct StringRelationalLiteral {
   }
 
   static std::string name() {
-    return "BM_StringRelationalLiteral" + Rel::name() + LHLength::name() +
-           RHLength::name() + DiffType::name();
+    return "BM_StringRelationalLiteral" + Rel::name() + LHLength::name() + RHLength::name() + DiffType::name();
   }
 };
 
@@ -493,25 +466,22 @@ template <class Temperature, class Depth, class Length>
 struct StringRead {
   void run(benchmark::State& state) const {
     static constexpr size_t NumStrings =
-        Temperature() == ::Temperature::Hot
-            ? 1 << 10
-            : /* Enough strings to overflow the cache */ 1 << 20;
-    static_assert((NumStrings & (NumStrings - 1)) == 0,
-                  "NumStrings should be a power of two to reduce overhead.");
+        Temperature() == ::Temperature::Hot ? 1 << 10 : /* Enough strings to overflow the cache */ 1 << 20;
+    static_assert((NumStrings & (NumStrings - 1)) == 0, "NumStrings should be a power of two to reduce overhead.");
 
     std::vector<std::string> Values(NumStrings, makeString(Length()));
     size_t I = 0;
     for (auto _ : state) {
       // Jump long enough to defeat cache locality, and use a value that is
       // coprime with NumStrings to ensure we visit every element.
-      I = (I + 17) % NumStrings;
+      I             = (I + 17) % NumStrings;
       const auto& V = Values[I];
 
       // Read everything first. Escaping data() through DoNotOptimize might
       // cause the compiler to have to recalculate information about `V` due to
       // aliasing.
       const char* const Data = V.data();
-      const size_t Size = V.size();
+      const size_t Size      = V.size();
       benchmark::DoNotOptimize(Data);
       benchmark::DoNotOptimize(Size);
       if (Depth() == ::Depth::Deep) {
@@ -530,25 +500,21 @@ struct StringRead {
     return false;
   }
 
-  std::string name() const {
-    return "BM_StringRead" + Temperature::name() + Depth::name() +
-           Length::name();
-  }
+  std::string name() const { return "BM_StringRead" + Temperature::name() + Depth::name() + Length::name(); }
 };
 
 void sanityCheckGeneratedStrings() {
   for (auto Lhs : {Length::Empty, Length::Small, Length::Large, Length::Huge}) {
     const auto LhsString = makeString(Lhs);
-    for (auto Rhs :
-         {Length::Empty, Length::Small, Length::Large, Length::Huge}) {
+    for (auto Rhs : {Length::Empty, Length::Small, Length::Large, Length::Huge}) {
       if (Lhs > Rhs)
         continue;
       const auto RhsString = makeString(Rhs);
 
       // The smaller one must be a prefix of the larger one.
       if (RhsString.find(LhsString) != 0) {
-        fprintf(stderr, "Invalid autogenerated strings for sizes (%d,%d).\n",
-                static_cast<int>(Lhs), static_cast<int>(Rhs));
+        fprintf(
+            stderr, "Invalid autogenerated strings for sizes (%d,%d).\n", static_cast<int>(Lhs), static_cast<int>(Rhs));
         std::abort();
       }
     }
@@ -556,15 +522,14 @@ void sanityCheckGeneratedStrings() {
   // Verify the autogenerated diffs
   for (auto L : {Length::Small, Length::Large, Length::Huge}) {
     const auto Control = makeString(L);
-    const auto Verify = [&](std::string Exp, size_t Pos) {
+    const auto Verify  = [&](std::string Exp, size_t Pos) {
       // Only change on the Pos char.
       if (Control[Pos] != Exp[Pos]) {
         Exp[Pos] = Control[Pos];
         if (Control == Exp)
           return;
       }
-      fprintf(stderr, "Invalid autogenerated diff with size %d\n",
-              static_cast<int>(L));
+      fprintf(stderr, "Invalid autogenerated diff with size %d\n", static_cast<int>(L));
       std::abort();
     };
     Verify(makeString(L, DiffType::ChangeFirst), 0);
@@ -574,20 +539,12 @@ void sanityCheckGeneratedStrings() {
 }
 
 // Some small codegen thunks to easily see generated code.
-bool StringEqString(const std::string& a, const std::string& b) {
-  return a == b;
-}
+bool StringEqString(const std::string& a, const std::string& b) { return a == b; }
 bool StringEqCStr(const std::string& a, const char* b) { return a == b; }
 bool CStrEqString(const char* a, const std::string& b) { return a == b; }
-bool StringEqCStrLiteralEmpty(const std::string& a) {
-  return a == "";
-}
-bool StringEqCStrLiteralSmall(const std::string& a) {
-  return a == SmallStringLiteral;
-}
-bool StringEqCStrLiteralLarge(const std::string& a) {
-  return a == LargeStringLiteral;
-}
+bool StringEqCStrLiteralEmpty(const std::string& a) { return a == ""; }
+bool StringEqCStrLiteralSmall(const std::string& a) { return a == SmallStringLiteral; }
+bool StringEqCStrLiteralLarge(const std::string& a) { return a == LargeStringLiteral; }
 
 int main(int argc, char** argv) {
   benchmark::Initialize(&argc, argv);
@@ -596,8 +553,7 @@ int main(int argc, char** argv) {
 
   sanityCheckGeneratedStrings();
 
-  makeCartesianProductBenchmark<StringConstructDestroyCStr, AllLengths,
-                                AllOpacity>();
+  makeCartesianProductBenchmark<StringConstructDestroyCStr, AllLengths, AllOpacity>();
 
   makeCartesianProductBenchmark<StringAssignStr, AllLengths, AllOpacity>();
   makeCartesianProductBenchmark<StringAssignAsciiz, AllLengths, AllOpacity>();
@@ -606,23 +562,23 @@ int main(int argc, char** argv) {
   makeCartesianProductBenchmark<StringCopy, AllLengths>();
   makeCartesianProductBenchmark<StringMove, AllLengths>();
   makeCartesianProductBenchmark<StringDestroy, AllLengths>();
-  makeCartesianProductBenchmark<StringResizeDefaultInit, AllLengths,
-                                AllOpacity>();
+  makeCartesianProductBenchmark<StringResizeDefaultInit, AllLengths, AllOpacity>();
   makeCartesianProductBenchmark<StringEraseToEnd, AllLengths, AllOpacity>();
   makeCartesianProductBenchmark<StringEraseWithMove, AllLengths, AllOpacity>();
-  makeCartesianProductBenchmark<StringRelational, AllRelations, AllLengths,
-                                AllLengths, AllDiffTypes>();
-  makeCartesianProductBenchmark<StringRelationalLiteral, AllRelations,
-                                AllLengths, AllLengths, AllDiffTypes>();
-  makeCartesianProductBenchmark<StringRead, AllTemperatures, AllDepths,
-                                AllLengths>();
+  makeCartesianProductBenchmark<StringRelational, AllRelations, AllLengths, AllLengths, AllDiffTypes>();
+  makeCartesianProductBenchmark<StringRelationalLiteral, AllRelations, AllLengths, AllLengths, AllDiffTypes>();
+  makeCartesianProductBenchmark<StringRead, AllTemperatures, AllDepths, AllLengths>();
   benchmark::RunSpecifiedBenchmarks();
 
   if (argc < 0) {
     // ODR-use the functions to force them being generated in the binary.
     auto functions = std::make_tuple(
-        StringEqString, StringEqCStr, CStrEqString, StringEqCStrLiteralEmpty,
-        StringEqCStrLiteralSmall, StringEqCStrLiteralLarge);
+        StringEqString,
+        StringEqCStr,
+        CStrEqString,
+        StringEqCStrLiteralEmpty,
+        StringEqCStrLiteralSmall,
+        StringEqCStrLiteralLarge);
     printf("%p", &functions);
   }
 }
diff --git a/libcxx/benchmarks/stringstream.bench.cpp b/libcxx/benchmarks/stringstream.bench.cpp
index 828ef4b405f47..ea602557ccd77 100644
--- a/libcxx/benchmarks/stringstream.bench.cpp
+++ b/libcxx/benchmarks/stringstream.bench.cpp
@@ -6,31 +6,21 @@
 TEST_NOINLINE double istream_numbers();
 
 double istream_numbers() {
-  const char *a[] = {
-    "-6  69 -71  2.4882e-02 -100 101 -2.00005 5000000 -50000000",
-    "-25 71   7 -9.3262e+01 -100 101 -2.00005 5000000 -50000000",
-    "-14 53  46 -6.7026e-02 -100 101 -2.00005 5000000 -50000000"
-  };
+  const char* a[] = {"-6  69 -71  2.4882e-02 -100 101 -2.00005 5000000 -50000000",
+                     "-25 71   7 -9.3262e+01 -100 101 -2.00005 5000000 -50000000",
+                     "-14 53  46 -6.7026e-02 -100 101 -2.00005 5000000 -50000000"};
 
   int a1, a2, a3, a4, a5, a6, a7;
   double f1 = 0.0, f2 = 0.0, q = 0.0;
-  for (int i=0; i < 3; i++) {
+  for (int i = 0; i < 3; i++) {
     std::istringstream s(a[i]);
-    s >> a1
-      >> a2
-      >> a3
-      >> f1
-      >> a4
-      >> a5
-      >> f2
-      >> a6
-      >> a7;
-    q += (a1 + a2 + a3 + a4 + a5 + a6 + a7 + f1 + f2)/1000000;
+    s >> a1 >> a2 >> a3 >> f1 >> a4 >> a5 >> f2 >> a6 >> a7;
+    q += (a1 + a2 + a3 + a4 + a5 + a6 + a7 + f1 + f2) / 1000000;
   }
   return q;
 }
 
-static void BM_Istream_numbers(benchmark::State &state) {
+static void BM_Istream_numbers(benchmark::State& state) {
   double i = 0;
   while (state.KeepRunning())
     benchmark::DoNotOptimize(i += istream_numbers());
diff --git a/libcxx/benchmarks/unordered_set_operations.bench.cpp b/libcxx/benchmarks/unordered_set_operations.bench.cpp
index acabc73f30fb3..775caeb6078b8 100644
--- a/libcxx/benchmarks/unordered_set_operations.bench.cpp
+++ b/libcxx/benchmarks/unordered_set_operations.bench.cpp
@@ -1,9 +1,9 @@
-#include <unordered_set>
-#include <vector>
-#include <functional>
 #include <cstdint>
 #include <cstdlib>
 #include <cstring>
+#include <functional>
+#include <unordered_set>
+#include <vector>
 
 #include "benchmark/benchmark.h"
 
@@ -16,91 +16,81 @@ using namespace ContainerBenchmarks;
 constexpr std::size_t TestNumInputs = 1024;
 
 template <class _Size>
-inline TEST_ALWAYS_INLINE
-_Size loadword(const void* __p) {
-    _Size __r;
-    std::memcpy(&__r, __p, sizeof(__r));
-    return __r;
+inline TEST_ALWAYS_INLINE _Size loadword(const void* __p) {
+  _Size __r;
+  std::memcpy(&__r, __p, sizeof(__r));
+  return __r;
 }
 
-inline TEST_ALWAYS_INLINE
-std::size_t rotate_by_at_least_1(std::size_t __val, int __shift) {
-    return (__val >> __shift) | (__val << (64 - __shift));
+inline TEST_ALWAYS_INLINE std::size_t rotate_by_at_least_1(std::size_t __val, int __shift) {
+  return (__val >> __shift) | (__val << (64 - __shift));
 }
 
-inline TEST_ALWAYS_INLINE
-std::size_t hash_len_16(std::size_t __u, std::size_t __v) {
-    const  std::size_t __mul = 0x9ddfea08eb382d69ULL;
-    std::size_t __a = (__u ^ __v) * __mul;
-    __a ^= (__a >> 47);
-    std::size_t __b = (__v ^ __a) * __mul;
-    __b ^= (__b >> 47);
-    __b *= __mul;
-    return __b;
+inline TEST_ALWAYS_INLINE std::size_t hash_len_16(std::size_t __u, std::size_t __v) {
+  const std::size_t __mul = 0x9ddfea08eb382d69ULL;
+  std::size_t __a         = (__u ^ __v) * __mul;
+  __a ^= (__a >> 47);
+  std::size_t __b = (__v ^ __a) * __mul;
+  __b ^= (__b >> 47);
+  __b *= __mul;
+  return __b;
 }
 
-
 template <std::size_t _Len>
-inline TEST_ALWAYS_INLINE
-std::size_t hash_len_0_to_8(const char* __s) {
-    static_assert(_Len == 4 || _Len == 8, "");
-    const uint64_t __a = loadword<uint32_t>(__s);
-    const uint64_t __b = loadword<uint32_t>(__s + _Len - 4);
-    return hash_len_16(_Len + (__a << 3), __b);
+inline TEST_ALWAYS_INLINE std::size_t hash_len_0_to_8(const char* __s) {
+  static_assert(_Len == 4 || _Len == 8, "");
+  const uint64_t __a = loadword<uint32_t>(__s);
+  const uint64_t __b = loadword<uint32_t>(__s + _Len - 4);
+  return hash_len_16(_Len + (__a << 3), __b);
 }
 
 struct UInt32Hash {
   UInt32Hash() = default;
-  inline TEST_ALWAYS_INLINE
-  std::size_t operator()(uint32_t data) const {
-      return hash_len_0_to_8<4>(reinterpret_cast<const char*>(&data));
+  inline TEST_ALWAYS_INLINE std::size_t operator()(uint32_t data) const {
+    return hash_len_0_to_8<4>(reinterpret_cast<const char*>(&data));
   }
 };
 
 struct UInt64Hash {
   UInt64Hash() = default;
-  inline TEST_ALWAYS_INLINE
-  std::size_t operator()(uint64_t data) const {
-      return hash_len_0_to_8<8>(reinterpret_cast<const char*>(&data));
+  inline TEST_ALWAYS_INLINE std::size_t operator()(uint64_t data) const {
+    return hash_len_0_to_8<8>(reinterpret_cast<const char*>(&data));
   }
 };
 
 struct UInt128Hash {
   UInt128Hash() = default;
-  inline TEST_ALWAYS_INLINE
-  std::size_t operator()(__uint128_t data) const {
-      const __uint128_t __mask = static_cast<std::size_t>(-1);
-      const std::size_t __a = (std::size_t)(data & __mask);
-      const std::size_t __b = (std::size_t)((data & (__mask << 64)) >> 64);
-      return hash_len_16(__a, rotate_by_at_least_1(__b + 16, 16)) ^ __b;
+  inline TEST_ALWAYS_INLINE std::size_t operator()(__uint128_t data) const {
+    const __uint128_t __mask = static_cast<std::size_t>(-1);
+    const std::size_t __a    = (std::size_t)(data & __mask);
+    const std::size_t __b    = (std::size_t)((data & (__mask << 64)) >> 64);
+    return hash_len_16(__a, rotate_by_at_least_1(__b + 16, 16)) ^ __b;
   }
 };
 
 struct UInt32Hash2 {
   UInt32Hash2() = default;
-  inline TEST_ALWAYS_INLINE
-  std::size_t operator()(uint32_t data) const {
-      const uint32_t __m = 0x5bd1e995;
-      const uint32_t __r = 24;
-      uint32_t __h = 4;
-      uint32_t __k = data;
-        __k *= __m;
-        __k ^= __k >> __r;
-        __k *= __m;
-        __h *= __m;
-        __h ^= __k;
-        __h ^= __h >> 13;
-        __h *= __m;
-        __h ^= __h >> 15;
+  inline TEST_ALWAYS_INLINE std::size_t operator()(uint32_t data) const {
+    const uint32_t __m = 0x5bd1e995;
+    const uint32_t __r = 24;
+    uint32_t __h       = 4;
+    uint32_t __k       = data;
+    __k *= __m;
+    __k ^= __k >> __r;
+    __k *= __m;
+    __h *= __m;
+    __h ^= __k;
+    __h ^= __h >> 13;
+    __h *= __m;
+    __h ^= __h >> 15;
     return __h;
   }
 };
 
 struct UInt64Hash2 {
   UInt64Hash2() = default;
-  inline TEST_ALWAYS_INLINE
-  std::size_t operator()(uint64_t data) const {
-      return hash_len_0_to_8<8>(reinterpret_cast<const char*>(&data));
+  inline TEST_ALWAYS_INLINE std::size_t operator()(uint64_t data) const {
+    return hash_len_0_to_8<8>(reinterpret_cast<const char*>(&data));
   }
 };
 
@@ -113,15 +103,15 @@ struct UInt64Hash2 {
 // of equal length in BM_Rehash.
 struct SlowStringEq {
   SlowStringEq() = default;
-  inline TEST_ALWAYS_INLINE
-  bool operator()(const std::string& lhs, const std::string& rhs) const {
-      if (lhs.size() != rhs.size()) return false;
-
-      bool eq = true;
-      for (size_t i = 0; i < lhs.size(); ++i) {
-          eq &= lhs[i] == rhs[i];
-      }
-      return eq;
+  inline TEST_ALWAYS_INLINE bool operator()(const std::string& lhs, const std::string& rhs) const {
+    if (lhs.size() != rhs.size())
+      return false;
+
+    bool eq = true;
+    for (size_t i = 0; i < lhs.size(); ++i) {
+      eq &= lhs[i] == rhs[i];
+    }
+    return eq;
   }
 };
 
@@ -131,212 +121,181 @@ struct SlowStringEq {
 
 template <class HashFn, class GenInputs>
 void BM_Hash(benchmark::State& st, HashFn fn, GenInputs gen) {
-    auto in = gen(st.range(0));
-    const auto end = in.data() + in.size();
-    std::size_t last_hash = 0;
-    benchmark::DoNotOptimize(&last_hash);
-    while (st.KeepRunning()) {
-        for (auto it = in.data(); it != end; ++it) {
-            benchmark::DoNotOptimize(last_hash += fn(*it));
-        }
-        benchmark::ClobberMemory();
+  auto in               = gen(st.range(0));
+  const auto end        = in.data() + in.size();
+  std::size_t last_hash = 0;
+  benchmark::DoNotOptimize(&last_hash);
+  while (st.KeepRunning()) {
+    for (auto it = in.data(); it != end; ++it) {
+      benchmark::DoNotOptimize(last_hash += fn(*it));
     }
+    benchmark::ClobberMemory();
+  }
 }
 
-BENCHMARK_CAPTURE(BM_Hash,
-    uint32_random_std_hash,
-    std::hash<uint32_t>{},
-    getRandomIntegerInputs<uint32_t>) -> Arg(TestNumInputs);
+BENCHMARK_CAPTURE(BM_Hash, uint32_random_std_hash, std::hash<uint32_t>{}, getRandomIntegerInputs<uint32_t>)
+    ->Arg(TestNumInputs);
 
-BENCHMARK_CAPTURE(BM_Hash,
-    uint32_random_custom_hash,
-    UInt32Hash{},
-    getRandomIntegerInputs<uint32_t>) -> Arg(TestNumInputs);
+BENCHMARK_CAPTURE(BM_Hash, uint32_random_custom_hash, UInt32Hash{}, getRandomIntegerInputs<uint32_t>)
+    ->Arg(TestNumInputs);
 
-BENCHMARK_CAPTURE(BM_Hash,
-    uint32_top_std_hash,
-    std::hash<uint32_t>{},
-    getSortedTopBitsIntegerInputs<uint32_t>) -> Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_Hash,
-    uint32_top_custom_hash,
-    UInt32Hash{},
-    getSortedTopBitsIntegerInputs<uint32_t>) -> Arg(TestNumInputs);
+BENCHMARK_CAPTURE(BM_Hash, uint32_top_std_hash, std::hash<uint32_t>{}, getSortedTopBitsIntegerInputs<uint32_t>)
+    ->Arg(TestNumInputs);
 
+BENCHMARK_CAPTURE(BM_Hash, uint32_top_custom_hash, UInt32Hash{}, getSortedTopBitsIntegerInputs<uint32_t>)
+    ->Arg(TestNumInputs);
 
 //----------------------------------------------------------------------------//
 //                       BM_InsertValue
 // ---------------------------------------------------------------------------//
 
-
 // Sorted Ascending //
-BENCHMARK_CAPTURE(BM_InsertValue,
-    unordered_set_uint32,
-    std::unordered_set<uint32_t>{},
-    getRandomIntegerInputs<uint32_t>)->Arg(TestNumInputs);
+BENCHMARK_CAPTURE(
+    BM_InsertValue, unordered_set_uint32, std::unordered_set<uint32_t>{}, getRandomIntegerInputs<uint32_t>)
+    ->Arg(TestNumInputs);
 
-BENCHMARK_CAPTURE(BM_InsertValue,
-    unordered_set_uint32_sorted,
-    std::unordered_set<uint32_t>{},
-    getSortedIntegerInputs<uint32_t>)->Arg(TestNumInputs);
+BENCHMARK_CAPTURE(
+    BM_InsertValue, unordered_set_uint32_sorted, std::unordered_set<uint32_t>{}, getSortedIntegerInputs<uint32_t>)
+    ->Arg(TestNumInputs);
 
 // Top Bytes //
 BENCHMARK_CAPTURE(BM_InsertValue,
-    unordered_set_top_bits_uint32,
-    std::unordered_set<uint32_t>{},
-    getSortedTopBitsIntegerInputs<uint32_t>)->Arg(TestNumInputs);
+                  unordered_set_top_bits_uint32,
+                  std::unordered_set<uint32_t>{},
+                  getSortedTopBitsIntegerInputs<uint32_t>)
+    ->Arg(TestNumInputs);
 
 BENCHMARK_CAPTURE(BM_InsertValueRehash,
-    unordered_set_top_bits_uint32,
-    std::unordered_set<uint32_t, UInt32Hash>{},
-    getSortedTopBitsIntegerInputs<uint32_t>)->Arg(TestNumInputs);
+                  unordered_set_top_bits_uint32,
+                  std::unordered_set<uint32_t, UInt32Hash>{},
+                  getSortedTopBitsIntegerInputs<uint32_t>)
+    ->Arg(TestNumInputs);
 
 // String //
-BENCHMARK_CAPTURE(BM_InsertValue,
-    unordered_set_string,
-    std::unordered_set<std::string>{},
-    getRandomStringInputs)->Arg(TestNumInputs);
+BENCHMARK_CAPTURE(BM_InsertValue, unordered_set_string, std::unordered_set<std::string>{}, getRandomStringInputs)
+    ->Arg(TestNumInputs);
 
-BENCHMARK_CAPTURE(BM_InsertValueRehash,
-    unordered_set_string,
-    std::unordered_set<std::string>{},
-    getRandomStringInputs)->Arg(TestNumInputs);
+BENCHMARK_CAPTURE(BM_InsertValueRehash, unordered_set_string, std::unordered_set<std::string>{}, getRandomStringInputs)
+    ->Arg(TestNumInputs);
 
 //----------------------------------------------------------------------------//
 //                         BM_Find
 // ---------------------------------------------------------------------------//
 
 // Random //
-BENCHMARK_CAPTURE(BM_Find,
-    unordered_set_random_uint64,
-    std::unordered_set<uint64_t>{},
-    getRandomIntegerInputs<uint64_t>)->Arg(TestNumInputs);
+BENCHMARK_CAPTURE(
+    BM_Find, unordered_set_random_uint64, std::unordered_set<uint64_t>{}, getRandomIntegerInputs<uint64_t>)
+    ->Arg(TestNumInputs);
 
 BENCHMARK_CAPTURE(BM_FindRehash,
-    unordered_set_random_uint64,
-    std::unordered_set<uint64_t, UInt64Hash>{},
-    getRandomIntegerInputs<uint64_t>)->Arg(TestNumInputs);
+                  unordered_set_random_uint64,
+                  std::unordered_set<uint64_t, UInt64Hash>{},
+                  getRandomIntegerInputs<uint64_t>)
+    ->Arg(TestNumInputs);
 
 // Sorted //
-BENCHMARK_CAPTURE(BM_Find,
-    unordered_set_sorted_uint64,
-    std::unordered_set<uint64_t>{},
-    getSortedIntegerInputs<uint64_t>)->Arg(TestNumInputs);
+BENCHMARK_CAPTURE(
+    BM_Find, unordered_set_sorted_uint64, std::unordered_set<uint64_t>{}, getSortedIntegerInputs<uint64_t>)
+    ->Arg(TestNumInputs);
 
 BENCHMARK_CAPTURE(BM_FindRehash,
-    unordered_set_sorted_uint64,
-    std::unordered_set<uint64_t, UInt64Hash>{},
-    getSortedIntegerInputs<uint64_t>)->Arg(TestNumInputs);
-
+                  unordered_set_sorted_uint64,
+                  std::unordered_set<uint64_t, UInt64Hash>{},
+                  getSortedIntegerInputs<uint64_t>)
+    ->Arg(TestNumInputs);
 
 // Sorted //
 #if 1
 BENCHMARK_CAPTURE(BM_Find,
-    unordered_set_sorted_uint128,
-    std::unordered_set<__uint128_t, UInt128Hash>{},
-    getSortedTopBitsIntegerInputs<__uint128_t>)->Arg(TestNumInputs);
+                  unordered_set_sorted_uint128,
+                  std::unordered_set<__uint128_t, UInt128Hash>{},
+                  getSortedTopBitsIntegerInputs<__uint128_t>)
+    ->Arg(TestNumInputs);
 
 BENCHMARK_CAPTURE(BM_FindRehash,
-    unordered_set_sorted_uint128,
-    std::unordered_set<__uint128_t, UInt128Hash>{},
-    getSortedTopBitsIntegerInputs<__uint128_t>)->Arg(TestNumInputs);
+                  unordered_set_sorted_uint128,
+                  std::unordered_set<__uint128_t, UInt128Hash>{},
+                  getSortedTopBitsIntegerInputs<__uint128_t>)
+    ->Arg(TestNumInputs);
 #endif
 
 // Sorted //
-BENCHMARK_CAPTURE(BM_Find,
-    unordered_set_sorted_uint32,
-    std::unordered_set<uint32_t>{},
-    getSortedIntegerInputs<uint32_t>)->Arg(TestNumInputs);
+BENCHMARK_CAPTURE(
+    BM_Find, unordered_set_sorted_uint32, std::unordered_set<uint32_t>{}, getSortedIntegerInputs<uint32_t>)
+    ->Arg(TestNumInputs);
 
 BENCHMARK_CAPTURE(BM_FindRehash,
-    unordered_set_sorted_uint32,
-    std::unordered_set<uint32_t, UInt32Hash2>{},
-    getSortedIntegerInputs<uint32_t>)->Arg(TestNumInputs);
+                  unordered_set_sorted_uint32,
+                  std::unordered_set<uint32_t, UInt32Hash2>{},
+                  getSortedIntegerInputs<uint32_t>)
+    ->Arg(TestNumInputs);
 
 // Sorted Ascending //
-BENCHMARK_CAPTURE(BM_Find,
-    unordered_set_sorted_large_uint64,
-    std::unordered_set<uint64_t>{},
-    getSortedLargeIntegerInputs<uint64_t>)->Arg(TestNumInputs);
+BENCHMARK_CAPTURE(
+    BM_Find, unordered_set_sorted_large_uint64, std::unordered_set<uint64_t>{}, getSortedLargeIntegerInputs<uint64_t>)
+    ->Arg(TestNumInputs);
 
 BENCHMARK_CAPTURE(BM_FindRehash,
-    unordered_set_sorted_large_uint64,
-    std::unordered_set<uint64_t, UInt64Hash>{},
-    getSortedLargeIntegerInputs<uint64_t>)->Arg(TestNumInputs);
-
+                  unordered_set_sorted_large_uint64,
+                  std::unordered_set<uint64_t, UInt64Hash>{},
+                  getSortedLargeIntegerInputs<uint64_t>)
+    ->Arg(TestNumInputs);
 
 // Top Bits //
-BENCHMARK_CAPTURE(BM_Find,
-    unordered_set_top_bits_uint64,
-    std::unordered_set<uint64_t>{},
-    getSortedTopBitsIntegerInputs<uint64_t>)->Arg(TestNumInputs);
+BENCHMARK_CAPTURE(
+    BM_Find, unordered_set_top_bits_uint64, std::unordered_set<uint64_t>{}, getSortedTopBitsIntegerInputs<uint64_t>)
+    ->Arg(TestNumInputs);
 
 BENCHMARK_CAPTURE(BM_FindRehash,
-    unordered_set_top_bits_uint64,
-    std::unordered_set<uint64_t, UInt64Hash>{},
-    getSortedTopBitsIntegerInputs<uint64_t>)->Arg(TestNumInputs);
+                  unordered_set_top_bits_uint64,
+                  std::unordered_set<uint64_t, UInt64Hash>{},
+                  getSortedTopBitsIntegerInputs<uint64_t>)
+    ->Arg(TestNumInputs);
 
 // String //
-BENCHMARK_CAPTURE(BM_Find,
-    unordered_set_string,
-    std::unordered_set<std::string>{},
-    getRandomStringInputs)->Arg(TestNumInputs);
+BENCHMARK_CAPTURE(BM_Find, unordered_set_string, std::unordered_set<std::string>{}, getRandomStringInputs)
+    ->Arg(TestNumInputs);
 
-BENCHMARK_CAPTURE(BM_FindRehash,
-    unordered_set_string,
-    std::unordered_set<std::string>{},
-    getRandomStringInputs)->Arg(TestNumInputs);
+BENCHMARK_CAPTURE(BM_FindRehash, unordered_set_string, std::unordered_set<std::string>{}, getRandomStringInputs)
+    ->Arg(TestNumInputs);
 
 //----------------------------------------------------------------------------//
 //                         BM_Rehash
 // ---------------------------------------------------------------------------//
 
 BENCHMARK_CAPTURE(BM_Rehash,
-    unordered_set_string_arg,
-    std::unordered_set<std::string, std::hash<std::string>, SlowStringEq>{},
-    getRandomStringInputs)->Arg(TestNumInputs);
+                  unordered_set_string_arg,
+                  std::unordered_set<std::string, std::hash<std::string>, SlowStringEq>{},
+                  getRandomStringInputs)
+    ->Arg(TestNumInputs);
 
-BENCHMARK_CAPTURE(BM_Rehash,
-    unordered_set_int_arg,
-    std::unordered_set<int>{},
-    getRandomIntegerInputs<int>)->Arg(TestNumInputs);
+BENCHMARK_CAPTURE(BM_Rehash, unordered_set_int_arg, std::unordered_set<int>{}, getRandomIntegerInputs<int>)
+    ->Arg(TestNumInputs);
 
 ///////////////////////////////////////////////////////////////////////////////
-BENCHMARK_CAPTURE(BM_InsertDuplicate,
-    unordered_set_int,
-    std::unordered_set<int>{},
-    getRandomIntegerInputs<int>)->Arg(TestNumInputs);
-BENCHMARK_CAPTURE(BM_InsertDuplicate,
-    unordered_set_string,
-    std::unordered_set<std::string>{},
-    getRandomStringInputs)->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_EmplaceDuplicate,
-    unordered_set_int,
-    std::unordered_set<int>{},
-    getRandomIntegerInputs<int>)->Arg(TestNumInputs);
-BENCHMARK_CAPTURE(BM_EmplaceDuplicate,
-    unordered_set_string,
-    std::unordered_set<std::string>{},
-    getRandomStringInputs)->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_InsertDuplicate,
-    unordered_set_int_insert_arg,
-    std::unordered_set<int>{},
-    getRandomIntegerInputs<int>)->Arg(TestNumInputs);
-BENCHMARK_CAPTURE(BM_InsertDuplicate,
-    unordered_set_string_insert_arg,
-    std::unordered_set<std::string>{},
-    getRandomStringInputs)->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_EmplaceDuplicate,
-    unordered_set_int_insert_arg,
-    std::unordered_set<int>{},
-    getRandomIntegerInputs<unsigned>)->Arg(TestNumInputs);
-
-BENCHMARK_CAPTURE(BM_EmplaceDuplicate,
-    unordered_set_string_arg,
-    std::unordered_set<std::string>{},
-    getRandomCStringInputs)->Arg(TestNumInputs);
+BENCHMARK_CAPTURE(BM_InsertDuplicate, unordered_set_int, std::unordered_set<int>{}, getRandomIntegerInputs<int>)
+    ->Arg(TestNumInputs);
+BENCHMARK_CAPTURE(BM_InsertDuplicate, unordered_set_string, std::unordered_set<std::string>{}, getRandomStringInputs)
+    ->Arg(TestNumInputs);
+
+BENCHMARK_CAPTURE(BM_EmplaceDuplicate, unordered_set_int, std::unordered_set<int>{}, getRandomIntegerInputs<int>)
+    ->Arg(TestNumInputs);
+BENCHMARK_CAPTURE(BM_EmplaceDuplicate, unordered_set_string, std::unordered_set<std::string>{}, getRandomStringInputs)
+    ->Arg(TestNumInputs);
+
+BENCHMARK_CAPTURE(
+    BM_InsertDuplicate, unordered_set_int_insert_arg, std::unordered_set<int>{}, getRandomIntegerInputs<int>)
+    ->Arg(TestNumInputs);
+BENCHMARK_CAPTURE(
+    BM_InsertDuplicate, unordered_set_string_insert_arg, std::unordered_set<std::string>{}, getRandomStringInputs)
+    ->Arg(TestNumInputs);
+
+BENCHMARK_CAPTURE(
+    BM_EmplaceDuplicate, unordered_set_int_insert_arg, std::unordered_set<int>{}, getRandomIntegerInputs<unsigned>)
+    ->Arg(TestNumInputs);
+
+BENCHMARK_CAPTURE(
+    BM_EmplaceDuplicate, unordered_set_string_arg, std::unordered_set<std::string>{}, getRandomCStringInputs)
+    ->Arg(TestNumInputs);
 
 BENCHMARK_MAIN();
diff --git a/libcxx/benchmarks/vector_operations.bench.cpp b/libcxx/benchmarks/vector_operations.bench.cpp
index 2e707c0a1b3c5..e02f0ee0c11ca 100644
--- a/libcxx/benchmarks/vector_operations.bench.cpp
+++ b/libcxx/benchmarks/vector_operations.bench.cpp
@@ -1,8 +1,8 @@
-#include <vector>
-#include <functional>
 #include <cstdint>
 #include <cstdlib>
 #include <cstring>
+#include <functional>
+#include <vector>
 
 #include "benchmark/benchmark.h"
 
@@ -13,36 +13,21 @@ using namespace ContainerBenchmarks;
 
 constexpr std::size_t TestNumInputs = 1024;
 
-BENCHMARK_CAPTURE(BM_ConstructSize,
-    vector_byte,
-    std::vector<unsigned char>{})->Arg(5140480);
-
-BENCHMARK_CAPTURE(BM_CopyConstruct,
-    vector_int,
-    std::vector<int>{})->Arg(5140480);
+BENCHMARK_CAPTURE(BM_ConstructSize, vector_byte, std::vector<unsigned char>{})->Arg(5140480);
 
-BENCHMARK_CAPTURE(BM_Assignment,
-    vector_int,
-    std::vector<int>{})->Arg(5140480);
+BENCHMARK_CAPTURE(BM_CopyConstruct, vector_int, std::vector<int>{})->Arg(5140480);
 
-BENCHMARK_CAPTURE(BM_ConstructSizeValue,
-    vector_byte,
-    std::vector<unsigned char>{}, 0)->Arg(5140480);
+BENCHMARK_CAPTURE(BM_Assignment, vector_int, std::vector<int>{})->Arg(5140480);
 
-BENCHMARK_CAPTURE(BM_ConstructIterIter,
-  vector_char,
-  std::vector<char>{},
-  getRandomIntegerInputs<char>)->Arg(TestNumInputs);
+BENCHMARK_CAPTURE(BM_ConstructSizeValue, vector_byte, std::vector<unsigned char>{}, 0)->Arg(5140480);
 
-BENCHMARK_CAPTURE(BM_ConstructIterIter,
-  vector_size_t,
-  std::vector<size_t>{},
-  getRandomIntegerInputs<size_t>)->Arg(TestNumInputs);
+BENCHMARK_CAPTURE(BM_ConstructIterIter, vector_char, std::vector<char>{}, getRandomIntegerInputs<char>)
+    ->Arg(TestNumInputs);
 
-BENCHMARK_CAPTURE(BM_ConstructIterIter,
-  vector_string,
-  std::vector<std::string>{},
-  getRandomStringInputs)->Arg(TestNumInputs);
+BENCHMARK_CAPTURE(BM_ConstructIterIter, vector_size_t, std::vector<size_t>{}, getRandomIntegerInputs<size_t>)
+    ->Arg(TestNumInputs);
 
+BENCHMARK_CAPTURE(BM_ConstructIterIter, vector_string, std::vector<std::string>{}, getRandomStringInputs)
+    ->Arg(TestNumInputs);
 
 BENCHMARK_MAIN();
diff --git a/libcxx/include/__algorithm/ranges_adjacent_find.h b/libcxx/include/__algorithm/ranges_adjacent_find.h
index e3de36bbc28d0..5f33192666ae9 100644
--- a/libcxx/include/__algorithm/ranges_adjacent_find.h
+++ b/libcxx/include/__algorithm/ranges_adjacent_find.h
@@ -31,10 +31,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 namespace __adjacent_find {
 struct __fn {
-
   template <class _Iter, class _Sent, class _Proj, class _Pred>
-  _LIBCPP_HIDE_FROM_ABI constexpr static
-  _Iter __adjacent_find_impl(_Iter __first, _Sent __last, _Pred& __pred, _Proj& __proj) {
+  _LIBCPP_HIDE_FROM_ABI constexpr static _Iter
+  __adjacent_find_impl(_Iter __first, _Sent __last, _Pred& __pred, _Proj& __proj) {
     if (__first == __last)
       return __first;
 
@@ -47,27 +46,28 @@ struct __fn {
     return __i;
   }
 
-  template <forward_iterator _Iter, sentinel_for<_Iter> _Sent,
-            class _Proj = identity,
+  template <forward_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj                                                                       = identity,
             indirect_binary_predicate<projected<_Iter, _Proj>, projected<_Iter, _Proj>> _Pred = ranges::equal_to>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  _Iter operator()(_Iter __first, _Sent __last, _Pred __pred = {}, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, _Sent __last, _Pred __pred = {}, _Proj __proj = {}) const {
     return __adjacent_find_impl(std::move(__first), std::move(__last), __pred, __proj);
   }
 
   template <forward_range _Range,
             class _Proj = identity,
-            indirect_binary_predicate<projected<iterator_t<_Range>, _Proj>,
-                                      projected<iterator_t<_Range>, _Proj>> _Pred = ranges::equal_to>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_iterator_t<_Range> operator()(_Range&& __range, _Pred __pred = {}, _Proj __proj = {}) const {
+            indirect_binary_predicate<projected<iterator_t<_Range>, _Proj>, projected<iterator_t<_Range>, _Proj>>
+                _Pred = ranges::equal_to>
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __range, _Pred __pred = {}, _Proj __proj = {}) const {
     return __adjacent_find_impl(ranges::begin(__range), ranges::end(__range), __pred, __proj);
   }
 };
 } // namespace __adjacent_find
 
 inline namespace __cpo {
-  inline constexpr auto adjacent_find = __adjacent_find::__fn{};
+inline constexpr auto adjacent_find = __adjacent_find::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_all_of.h b/libcxx/include/__algorithm/ranges_all_of.h
index 494a77d74c7ea..39a2ae4de01e9 100644
--- a/libcxx/include/__algorithm/ranges_all_of.h
+++ b/libcxx/include/__algorithm/ranges_all_of.h
@@ -29,10 +29,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 namespace __all_of {
 struct __fn {
-
   template <class _Iter, class _Sent, class _Proj, class _Pred>
-  _LIBCPP_HIDE_FROM_ABI constexpr static
-  bool __all_of_impl(_Iter __first, _Sent __last, _Pred& __pred, _Proj& __proj) {
+  _LIBCPP_HIDE_FROM_ABI constexpr static bool __all_of_impl(_Iter __first, _Sent __last, _Pred& __pred, _Proj& __proj) {
     for (; __first != __last; ++__first) {
       if (!std::invoke(__pred, std::invoke(__proj, *__first)))
         return false;
@@ -40,24 +38,27 @@ struct __fn {
     return true;
   }
 
-  template <input_iterator _Iter, sentinel_for<_Iter> _Sent, class _Proj = identity,
+  template <input_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj = identity,
             indirect_unary_predicate<projected<_Iter, _Proj>> _Pred>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  bool operator()(_Iter __first, _Sent __last, _Pred __pred, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Iter __first, _Sent __last, _Pred __pred, _Proj __proj = {}) const {
     return __all_of_impl(std::move(__first), std::move(__last), __pred, __proj);
   }
 
-  template <input_range _Range, class _Proj = identity,
+  template <input_range _Range,
+            class _Proj = identity,
             indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  bool operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) const {
     return __all_of_impl(ranges::begin(__range), ranges::end(__range), __pred, __proj);
   }
 };
 } // namespace __all_of
 
 inline namespace __cpo {
-  inline constexpr auto all_of = __all_of::__fn{};
+inline constexpr auto all_of = __all_of::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_any_of.h b/libcxx/include/__algorithm/ranges_any_of.h
index eb102bd858b00..2ca8531102eac 100644
--- a/libcxx/include/__algorithm/ranges_any_of.h
+++ b/libcxx/include/__algorithm/ranges_any_of.h
@@ -29,10 +29,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 namespace __any_of {
 struct __fn {
-
   template <class _Iter, class _Sent, class _Proj, class _Pred>
-  _LIBCPP_HIDE_FROM_ABI constexpr static
-  bool __any_of_impl(_Iter __first, _Sent __last, _Pred& __pred, _Proj& __proj) {
+  _LIBCPP_HIDE_FROM_ABI constexpr static bool __any_of_impl(_Iter __first, _Sent __last, _Pred& __pred, _Proj& __proj) {
     for (; __first != __last; ++__first) {
       if (std::invoke(__pred, std::invoke(__proj, *__first)))
         return true;
@@ -40,24 +38,27 @@ struct __fn {
     return false;
   }
 
-  template <input_iterator _Iter, sentinel_for<_Iter> _Sent, class _Proj = identity,
+  template <input_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj = identity,
             indirect_unary_predicate<projected<_Iter, _Proj>> _Pred>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  bool operator()(_Iter __first, _Sent __last, _Pred __pred = {}, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Iter __first, _Sent __last, _Pred __pred = {}, _Proj __proj = {}) const {
     return __any_of_impl(std::move(__first), std::move(__last), __pred, __proj);
   }
 
-  template <input_range _Range, class _Proj = identity,
+  template <input_range _Range,
+            class _Proj = identity,
             indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  bool operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) const {
     return __any_of_impl(ranges::begin(__range), ranges::end(__range), __pred, __proj);
   }
 };
 } // namespace __any_of
 
 inline namespace __cpo {
-  inline constexpr auto any_of = __any_of::__fn{};
+inline constexpr auto any_of = __any_of::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_binary_search.h b/libcxx/include/__algorithm/ranges_binary_search.h
index 728eb8b68ef76..22008e0f1bc8f 100644
--- a/libcxx/include/__algorithm/ranges_binary_search.h
+++ b/libcxx/include/__algorithm/ranges_binary_search.h
@@ -31,28 +31,33 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 namespace __binary_search {
 struct __fn {
-  template <forward_iterator _Iter, sentinel_for<_Iter> _Sent, class _Type, class _Proj = identity,
+  template <forward_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Type,
+            class _Proj                                                             = identity,
             indirect_strict_weak_order<const _Type*, projected<_Iter, _Proj>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  bool operator()(_Iter __first, _Sent __last, const _Type& __value, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Iter __first, _Sent __last, const _Type& __value, _Comp __comp = {}, _Proj __proj = {}) const {
     auto __ret = std::__lower_bound<_RangeAlgPolicy>(__first, __last, __value, __comp, __proj);
     return __ret != __last && !std::invoke(__comp, __value, std::invoke(__proj, *__ret));
   }
 
-  template <forward_range _Range, class _Type, class _Proj = identity,
+  template <forward_range _Range,
+            class _Type,
+            class _Proj                                                                          = identity,
             indirect_strict_weak_order<const _Type*, projected<iterator_t<_Range>, _Proj>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  bool operator()(_Range&& __r, const _Type& __value, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Range&& __r, const _Type& __value, _Comp __comp = {}, _Proj __proj = {}) const {
     auto __first = ranges::begin(__r);
-    auto __last = ranges::end(__r);
-    auto __ret = std::__lower_bound<_RangeAlgPolicy>(__first, __last, __value, __comp, __proj);
+    auto __last  = ranges::end(__r);
+    auto __ret   = std::__lower_bound<_RangeAlgPolicy>(__first, __last, __value, __comp, __proj);
     return __ret != __last && !std::invoke(__comp, __value, std::invoke(__proj, *__ret));
   }
 };
 } // namespace __binary_search
 
 inline namespace __cpo {
-  inline constexpr auto binary_search = __binary_search::__fn{};
+inline constexpr auto binary_search = __binary_search::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_clamp.h b/libcxx/include/__algorithm/ranges_clamp.h
index 45a8464c98a23..45bbebb97ef87 100644
--- a/libcxx/include/__algorithm/ranges_clamp.h
+++ b/libcxx/include/__algorithm/ranges_clamp.h
@@ -29,16 +29,11 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 namespace __clamp {
 struct __fn {
-
   template <class _Type,
-            class _Proj = identity,
+            class _Proj                                                      = identity,
             indirect_strict_weak_order<projected<const _Type*, _Proj>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  const _Type& operator()(const _Type& __value,
-                          const _Type& __low,
-                          const _Type& __high,
-                          _Comp __comp = {},
-                          _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr const _Type& operator()(
+      const _Type& __value, const _Type& __low, const _Type& __high, _Comp __comp = {}, _Proj __proj = {}) const {
     _LIBCPP_ASSERT(!bool(std::invoke(__comp, std::invoke(__proj, __high), std::invoke(__proj, __low))),
                    "Bad bounds passed to std::ranges::clamp");
 
@@ -49,12 +44,11 @@ struct __fn {
     else
       return __value;
   }
-
 };
 } // namespace __clamp
 
 inline namespace __cpo {
-  inline constexpr auto clamp = __clamp::__fn{};
+inline constexpr auto clamp = __clamp::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_copy.h b/libcxx/include/__algorithm/ranges_copy.h
index f5700c3d3c000..1c87f074e7cab 100644
--- a/libcxx/include/__algorithm/ranges_copy.h
+++ b/libcxx/include/__algorithm/ranges_copy.h
@@ -36,19 +36,18 @@ using copy_result = in_out_result<_InIter, _OutIter>;
 
 namespace __copy {
 struct __fn {
-
   template <input_iterator _InIter, sentinel_for<_InIter> _Sent, weakly_incrementable _OutIter>
     requires indirectly_copyable<_InIter, _OutIter>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  copy_result<_InIter, _OutIter> operator()(_InIter __first, _Sent __last, _OutIter __result) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr copy_result<_InIter, _OutIter>
+  operator()(_InIter __first, _Sent __last, _OutIter __result) const {
     auto __ret = std::__copy<_RangeAlgPolicy>(std::move(__first), std::move(__last), std::move(__result));
     return {std::move(__ret.first), std::move(__ret.second)};
   }
 
   template <input_range _Range, weakly_incrementable _OutIter>
     requires indirectly_copyable<iterator_t<_Range>, _OutIter>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  copy_result<borrowed_iterator_t<_Range>, _OutIter> operator()(_Range&& __r, _OutIter __result) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr copy_result<borrowed_iterator_t<_Range>, _OutIter>
+  operator()(_Range&& __r, _OutIter __result) const {
     auto __ret = std::__copy<_RangeAlgPolicy>(ranges::begin(__r), ranges::end(__r), std::move(__result));
     return {std::move(__ret.first), std::move(__ret.second)};
   }
@@ -56,7 +55,7 @@ struct __fn {
 } // namespace __copy
 
 inline namespace __cpo {
-  inline constexpr auto copy = __copy::__fn{};
+inline constexpr auto copy = __copy::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_copy_backward.h b/libcxx/include/__algorithm/ranges_copy_backward.h
index b7b5b88792b3f..865e944d4384d 100644
--- a/libcxx/include/__algorithm/ranges_copy_backward.h
+++ b/libcxx/include/__algorithm/ranges_copy_backward.h
@@ -29,24 +29,23 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 namespace ranges {
 
-template<class _Ip, class _Op>
+template <class _Ip, class _Op>
 using copy_backward_result = in_out_result<_Ip, _Op>;
 
 namespace __copy_backward {
 struct __fn {
-
   template <bidirectional_iterator _InIter1, sentinel_for<_InIter1> _Sent1, bidirectional_iterator _InIter2>
     requires indirectly_copyable<_InIter1, _InIter2>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  copy_backward_result<_InIter1, _InIter2> operator()(_InIter1 __first, _Sent1 __last, _InIter2 __result) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr copy_backward_result<_InIter1, _InIter2>
+  operator()(_InIter1 __first, _Sent1 __last, _InIter2 __result) const {
     auto __ret = std::__copy_backward<_RangeAlgPolicy>(std::move(__first), std::move(__last), std::move(__result));
     return {std::move(__ret.first), std::move(__ret.second)};
   }
 
   template <bidirectional_range _Range, bidirectional_iterator _Iter>
     requires indirectly_copyable<iterator_t<_Range>, _Iter>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  copy_backward_result<borrowed_iterator_t<_Range>, _Iter> operator()(_Range&& __r, _Iter __result) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr copy_backward_result<borrowed_iterator_t<_Range>, _Iter>
+  operator()(_Range&& __r, _Iter __result) const {
     auto __ret = std::__copy_backward<_RangeAlgPolicy>(ranges::begin(__r), ranges::end(__r), std::move(__result));
     return {std::move(__ret.first), std::move(__ret.second)};
   }
@@ -54,7 +53,7 @@ struct __fn {
 } // namespace __copy_backward
 
 inline namespace __cpo {
-  inline constexpr auto copy_backward = __copy_backward::__fn{};
+inline constexpr auto copy_backward = __copy_backward::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_copy_if.h b/libcxx/include/__algorithm/ranges_copy_if.h
index b714e4cd1979b..b77dbd37fcee3 100644
--- a/libcxx/include/__algorithm/ranges_copy_if.h
+++ b/libcxx/include/__algorithm/ranges_copy_if.h
@@ -30,15 +30,13 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 namespace ranges {
 
-template<class _Ip, class _Op>
+template <class _Ip, class _Op>
 using copy_if_result = in_out_result<_Ip, _Op>;
 
 namespace __copy_if {
 struct __fn {
-
   template <class _InIter, class _Sent, class _OutIter, class _Proj, class _Pred>
-  _LIBCPP_HIDE_FROM_ABI static constexpr
-  copy_if_result <_InIter, _OutIter>
+  _LIBCPP_HIDE_FROM_ABI static constexpr copy_if_result<_InIter, _OutIter>
   __copy_if_impl(_InIter __first, _Sent __last, _OutIter __result, _Pred& __pred, _Proj& __proj) {
     for (; __first != __last; ++__first) {
       if (std::invoke(__pred, std::invoke(__proj, *__first))) {
@@ -49,20 +47,23 @@ struct __fn {
     return {std::move(__first), std::move(__result)};
   }
 
-  template <input_iterator _Iter, sentinel_for<_Iter> _Sent, weakly_incrementable _OutIter, class _Proj = identity,
+  template <input_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            weakly_incrementable _OutIter,
+            class _Proj = identity,
             indirect_unary_predicate<projected<_Iter, _Proj>> _Pred>
     requires indirectly_copyable<_Iter, _OutIter>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  copy_if_result<_Iter, _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr copy_if_result<_Iter, _OutIter>
   operator()(_Iter __first, _Sent __last, _OutIter __result, _Pred __pred, _Proj __proj = {}) const {
     return __copy_if_impl(std::move(__first), std::move(__last), std::move(__result), __pred, __proj);
   }
 
-  template <input_range _Range, weakly_incrementable _OutIter, class _Proj = identity,
+  template <input_range _Range,
+            weakly_incrementable _OutIter,
+            class _Proj = identity,
             indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
     requires indirectly_copyable<iterator_t<_Range>, _OutIter>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  copy_if_result<borrowed_iterator_t<_Range>, _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr copy_if_result<borrowed_iterator_t<_Range>, _OutIter>
   operator()(_Range&& __r, _OutIter __result, _Pred __pred, _Proj __proj = {}) const {
     return __copy_if_impl(ranges::begin(__r), ranges::end(__r), std::move(__result), __pred, __proj);
   }
@@ -70,7 +71,7 @@ struct __fn {
 } // namespace __copy_if
 
 inline namespace __cpo {
-  inline constexpr auto copy_if = __copy_if::__fn{};
+inline constexpr auto copy_if = __copy_if::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_copy_n.h b/libcxx/include/__algorithm/ranges_copy_n.h
index 30ee019ac9bce..99e8eee14d0f8 100644
--- a/libcxx/include/__algorithm/ranges_copy_n.h
+++ b/libcxx/include/__algorithm/ranges_copy_n.h
@@ -36,10 +36,9 @@ using copy_n_result = in_out_result<_Ip, _Op>;
 
 namespace __copy_n {
 struct __fn {
-
   template <class _InIter, class _DiffType, class _OutIter>
-  _LIBCPP_HIDE_FROM_ABI constexpr static
-  copy_n_result<_InIter, _OutIter> __go(_InIter __first, _DiffType __n, _OutIter __result) {
+  _LIBCPP_HIDE_FROM_ABI constexpr static copy_n_result<_InIter, _OutIter>
+  __go(_InIter __first, _DiffType __n, _OutIter __result) {
     while (__n != 0) {
       *__result = *__first;
       ++__first;
@@ -50,23 +49,23 @@ struct __fn {
   }
 
   template <random_access_iterator _InIter, class _DiffType, random_access_iterator _OutIter>
-  _LIBCPP_HIDE_FROM_ABI constexpr static
-  copy_n_result<_InIter, _OutIter> __go(_InIter __first, _DiffType __n, _OutIter __result) {
+  _LIBCPP_HIDE_FROM_ABI constexpr static copy_n_result<_InIter, _OutIter>
+  __go(_InIter __first, _DiffType __n, _OutIter __result) {
     auto __ret = std::__copy<_RangeAlgPolicy>(__first, __first + __n, __result);
     return {__ret.first, __ret.second};
   }
 
   template <input_iterator _Ip, weakly_incrementable _Op>
     requires indirectly_copyable<_Ip, _Op>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  copy_n_result<_Ip, _Op> operator()(_Ip __first, iter_difference_t<_Ip> __n, _Op __result) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr copy_n_result<_Ip, _Op>
+  operator()(_Ip __first, iter_difference_t<_Ip> __n, _Op __result) const {
     return __go(std::move(__first), __n, std::move(__result));
   }
 };
 } // namespace __copy_n
 
 inline namespace __cpo {
-  inline constexpr auto copy_n = __copy_n::__fn{};
+inline constexpr auto copy_n = __copy_n::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_count.h b/libcxx/include/__algorithm/ranges_count.h
index 677ee38185fab..82f54567561f2 100644
--- a/libcxx/include/__algorithm/ranges_count.h
+++ b/libcxx/include/__algorithm/ranges_count.h
@@ -34,16 +34,16 @@ namespace __count {
 struct __fn {
   template <input_iterator _Iter, sentinel_for<_Iter> _Sent, class _Type, class _Proj = identity>
     requires indirect_binary_predicate<ranges::equal_to, projected<_Iter, _Proj>, const _Type*>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  iter_difference_t<_Iter> operator()(_Iter __first, _Sent __last, const _Type& __value, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr iter_difference_t<_Iter>
+  operator()(_Iter __first, _Sent __last, const _Type& __value, _Proj __proj = {}) const {
     auto __pred = [&](auto&& __e) { return __e == __value; };
     return ranges::__count_if_impl(std::move(__first), std::move(__last), __pred, __proj);
   }
 
   template <input_range _Range, class _Type, class _Proj = identity>
     requires indirect_binary_predicate<ranges::equal_to, projected<iterator_t<_Range>, _Proj>, const _Type*>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  range_difference_t<_Range> operator()(_Range&& __r, const _Type& __value, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr range_difference_t<_Range>
+  operator()(_Range&& __r, const _Type& __value, _Proj __proj = {}) const {
     auto __pred = [&](auto&& __e) { return __e == __value; };
     return ranges::__count_if_impl(ranges::begin(__r), ranges::end(__r), __pred, __proj);
   }
@@ -51,7 +51,7 @@ struct __fn {
 } // namespace __count
 
 inline namespace __cpo {
-  inline constexpr auto count = __count::__fn{};
+inline constexpr auto count = __count::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_count_if.h b/libcxx/include/__algorithm/ranges_count_if.h
index 48c4370eb288e..92f37d049e0c4 100644
--- a/libcxx/include/__algorithm/ranges_count_if.h
+++ b/libcxx/include/__algorithm/ranges_count_if.h
@@ -31,9 +31,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 namespace ranges {
 template <class _Iter, class _Sent, class _Proj, class _Pred>
-_LIBCPP_HIDE_FROM_ABI constexpr
-iter_difference_t<_Iter> __count_if_impl(_Iter __first, _Sent __last,
-                                             _Pred& __pred, _Proj& __proj) {
+_LIBCPP_HIDE_FROM_ABI constexpr iter_difference_t<_Iter>
+__count_if_impl(_Iter __first, _Sent __last, _Pred& __pred, _Proj& __proj) {
   iter_difference_t<_Iter> __counter(0);
   for (; __first != __last; ++__first) {
     if (std::invoke(__pred, std::invoke(__proj, *__first)))
@@ -44,24 +43,27 @@ iter_difference_t<_Iter> __count_if_impl(_Iter __first, _Sent __last,
 
 namespace __count_if {
 struct __fn {
-  template <input_iterator _Iter, sentinel_for<_Iter> _Sent, class _Proj = identity,
+  template <input_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj = identity,
             indirect_unary_predicate<projected<_Iter, _Proj>> _Predicate>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  iter_difference_t<_Iter> operator()(_Iter __first, _Sent __last, _Predicate __pred, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr iter_difference_t<_Iter>
+  operator()(_Iter __first, _Sent __last, _Predicate __pred, _Proj __proj = {}) const {
     return ranges::__count_if_impl(std::move(__first), std::move(__last), __pred, __proj);
   }
 
-  template <input_range _Range, class _Proj = identity,
+  template <input_range _Range,
+            class _Proj = identity,
             indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Predicate>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  range_difference_t<_Range> operator()(_Range&& __r, _Predicate __pred, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr range_difference_t<_Range>
+  operator()(_Range&& __r, _Predicate __pred, _Proj __proj = {}) const {
     return ranges::__count_if_impl(ranges::begin(__r), ranges::end(__r), __pred, __proj);
   }
 };
 } // namespace __count_if
 
 inline namespace __cpo {
-  inline constexpr auto count_if = __count_if::__fn{};
+inline constexpr auto count_if = __count_if::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_equal.h b/libcxx/include/__algorithm/ranges_equal.h
index 87544531c70bc..4cb1f7df1952e 100644
--- a/libcxx/include/__algorithm/ranges_equal.h
+++ b/libcxx/include/__algorithm/ranges_equal.h
@@ -33,61 +33,67 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 namespace __equal {
 struct __fn {
-  template <input_iterator _Iter1, sentinel_for<_Iter1> _Sent1,
-            input_iterator _Iter2, sentinel_for<_Iter2> _Sent2,
-            class _Pred = ranges::equal_to,
+  template <input_iterator _Iter1,
+            sentinel_for<_Iter1> _Sent1,
+            input_iterator _Iter2,
+            sentinel_for<_Iter2> _Sent2,
+            class _Pred  = ranges::equal_to,
             class _Proj1 = identity,
             class _Proj2 = identity>
     requires indirectly_comparable<_Iter1, _Iter2, _Pred, _Proj1, _Proj2>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  bool operator()(_Iter1 __first1, _Sent1 __last1,
-                  _Iter2 __first2, _Sent2 __last2,
-                  _Pred __pred = {},
-                  _Proj1 __proj1 = {},
-                  _Proj2 __proj2 = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(
+      _Iter1 __first1,
+      _Sent1 __last1,
+      _Iter2 __first2,
+      _Sent2 __last2,
+      _Pred __pred   = {},
+      _Proj1 __proj1 = {},
+      _Proj2 __proj2 = {}) const {
     if constexpr (sized_sentinel_for<_Sent1, _Iter1> && sized_sentinel_for<_Sent2, _Iter2>) {
       if (__last1 - __first1 != __last2 - __first2)
         return false;
     }
     auto __unwrapped1 = std::__unwrap_range(std::move(__first1), std::move(__last1));
     auto __unwrapped2 = std::__unwrap_range(std::move(__first2), std::move(__last2));
-    return std::__equal_impl(std::move(__unwrapped1.first), std::move(__unwrapped1.second),
-                             std::move(__unwrapped2.first), std::move(__unwrapped2.second),
-                             __pred,
-                             __proj1,
-                             __proj2);
+    return std::__equal_impl(
+        std::move(__unwrapped1.first),
+        std::move(__unwrapped1.second),
+        std::move(__unwrapped2.first),
+        std::move(__unwrapped2.second),
+        __pred,
+        __proj1,
+        __proj2);
   }
 
   template <input_range _Range1,
             input_range _Range2,
-            class _Pred = ranges::equal_to,
+            class _Pred  = ranges::equal_to,
             class _Proj1 = identity,
             class _Proj2 = identity>
     requires indirectly_comparable<iterator_t<_Range1>, iterator_t<_Range2>, _Pred, _Proj1, _Proj2>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  bool operator()(_Range1&& __range1,
-                  _Range2&& __range2,
-                  _Pred __pred = {},
-                  _Proj1 __proj1 = {},
-                  _Proj2 __proj2 = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(
+      _Range1&& __range1, _Range2&& __range2, _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
     if constexpr (sized_range<_Range1> && sized_range<_Range2>) {
       if (ranges::distance(__range1) != ranges::distance(__range2))
         return false;
     }
     auto __unwrapped1 = std::__unwrap_range(ranges::begin(__range1), ranges::end(__range1));
     auto __unwrapped2 = std::__unwrap_range(ranges::begin(__range2), ranges::end(__range2));
-    return std::__equal_impl(std::move(__unwrapped1.first), std::move(__unwrapped1.second),
-                             std::move(__unwrapped2.first), std::move(__unwrapped2.second),
-                             __pred,
-                             __proj1,
-                             __proj2);
+    return std::__equal_impl(
+        std::move(__unwrapped1.first),
+        std::move(__unwrapped1.second),
+        std::move(__unwrapped2.first),
+        std::move(__unwrapped2.second),
+        __pred,
+        __proj1,
+        __proj2);
     return false;
   }
 };
 } // namespace __equal
 
 inline namespace __cpo {
-  inline constexpr auto equal = __equal::__fn{};
+inline constexpr auto equal = __equal::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_equal_range.h b/libcxx/include/__algorithm/ranges_equal_range.h
index 075634ae9dd89..ed78cf312c6b2 100644
--- a/libcxx/include/__algorithm/ranges_equal_range.h
+++ b/libcxx/include/__algorithm/ranges_equal_range.h
@@ -38,28 +38,25 @@ namespace ranges {
 namespace __equal_range {
 
 struct __fn {
-  template <
-      forward_iterator _Iter,
-      sentinel_for<_Iter> _Sent,
-      class _Tp,
-      class _Proj                                                           = identity,
-      indirect_strict_weak_order<const _Tp*, projected<_Iter, _Proj>> _Comp = ranges::less>
+  template <forward_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Tp,
+            class _Proj                                                           = identity,
+            indirect_strict_weak_order<const _Tp*, projected<_Iter, _Proj>> _Comp = ranges::less>
   _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr subrange<_Iter>
   operator()(_Iter __first, _Sent __last, const _Tp& __value, _Comp __comp = {}, _Proj __proj = {}) const {
-    auto __ret = std::__equal_range<_RangeAlgPolicy>(
-        std::move(__first), std::move(__last), __value, __comp, __proj);
+    auto __ret = std::__equal_range<_RangeAlgPolicy>(std::move(__first), std::move(__last), __value, __comp, __proj);
     return {std::move(__ret.first), std::move(__ret.second)};
   }
 
-  template <
-      forward_range _Range,
-      class _Tp,
-      class _Proj                                                                        = identity,
-      indirect_strict_weak_order<const _Tp*, projected<iterator_t<_Range>, _Proj>> _Comp = ranges::less>
+  template <forward_range _Range,
+            class _Tp,
+            class _Proj                                                                        = identity,
+            indirect_strict_weak_order<const _Tp*, projected<iterator_t<_Range>, _Proj>> _Comp = ranges::less>
   _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr borrowed_subrange_t<_Range>
   operator()(_Range&& __range, const _Tp& __value, _Comp __comp = {}, _Proj __proj = {}) const {
-    auto __ret = std::__equal_range<_RangeAlgPolicy>(
-        ranges::begin(__range), ranges::end(__range), __value, __comp, __proj);
+    auto __ret =
+        std::__equal_range<_RangeAlgPolicy>(ranges::begin(__range), ranges::end(__range), __value, __comp, __proj);
     return {std::move(__ret.first), std::move(__ret.second)};
   }
 };
@@ -67,7 +64,7 @@ struct __fn {
 } // namespace __equal_range
 
 inline namespace __cpo {
-  inline constexpr auto equal_range = __equal_range::__fn{};
+inline constexpr auto equal_range = __equal_range::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_fill.h b/libcxx/include/__algorithm/ranges_fill.h
index 4e0594c6d2d7f..88a892f5c2786 100644
--- a/libcxx/include/__algorithm/ranges_fill.h
+++ b/libcxx/include/__algorithm/ranges_fill.h
@@ -28,9 +28,8 @@ namespace ranges {
 namespace __fill {
 struct __fn {
   template <class _Type, output_iterator<const _Type&> _Iter, sentinel_for<_Iter> _Sent>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  _Iter operator()(_Iter __first, _Sent __last, const _Type& __value) const {
-    if constexpr(random_access_iterator<_Iter> && sized_sentinel_for<_Sent, _Iter>) {
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter operator()(_Iter __first, _Sent __last, const _Type& __value) const {
+    if constexpr (random_access_iterator<_Iter> && sized_sentinel_for<_Sent, _Iter>) {
       return ranges::fill_n(__first, __last - __first, __value);
     } else {
       for (; __first != __last; ++__first)
@@ -40,15 +39,14 @@ struct __fn {
   }
 
   template <class _Type, output_range<const _Type&> _Range>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_iterator_t<_Range> operator()(_Range&& __range, const _Type& __value) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range> operator()(_Range&& __range, const _Type& __value) const {
     return (*this)(ranges::begin(__range), ranges::end(__range), __value);
   }
 };
 } // namespace __fill
 
 inline namespace __cpo {
-  inline constexpr auto fill = __fill::__fn{};
+inline constexpr auto fill = __fill::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_fill_n.h b/libcxx/include/__algorithm/ranges_fill_n.h
index c5f7e6eadf098..dbd8ec27aef9f 100644
--- a/libcxx/include/__algorithm/ranges_fill_n.h
+++ b/libcxx/include/__algorithm/ranges_fill_n.h
@@ -25,8 +25,8 @@ namespace ranges {
 namespace __fill_n {
 struct __fn {
   template <class _Type, output_iterator<const _Type&> _Iter>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  _Iter operator()(_Iter __first, iter_difference_t<_Iter> __n, const _Type& __value) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, iter_difference_t<_Iter> __n, const _Type& __value) const {
     for (; __n != 0; --__n) {
       *__first = __value;
       ++__first;
@@ -37,7 +37,7 @@ struct __fn {
 } // namespace __fill_n
 
 inline namespace __cpo {
-  inline constexpr auto fill_n = __fill_n::__fn{};
+inline constexpr auto fill_n = __fill_n::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_find.h b/libcxx/include/__algorithm/ranges_find.h
index 084cdfff0c9c6..de870e381184c 100644
--- a/libcxx/include/__algorithm/ranges_find.h
+++ b/libcxx/include/__algorithm/ranges_find.h
@@ -49,22 +49,22 @@ struct __fn {
 
   template <input_iterator _Ip, sentinel_for<_Ip> _Sp, class _Tp, class _Proj = identity>
     requires indirect_binary_predicate<ranges::equal_to, projected<_Ip, _Proj>, const _Tp*>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  _Ip operator()(_Ip __first, _Sp __last, const _Tp& __value, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr _Ip
+  operator()(_Ip __first, _Sp __last, const _Tp& __value, _Proj __proj = {}) const {
     return __find_unwrap(std::move(__first), std::move(__last), __value, __proj);
   }
 
   template <input_range _Rp, class _Tp, class _Proj = identity>
     requires indirect_binary_predicate<ranges::equal_to, projected<iterator_t<_Rp>, _Proj>, const _Tp*>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_iterator_t<_Rp> operator()(_Rp&& __r, const _Tp& __value, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Rp>
+  operator()(_Rp&& __r, const _Tp& __value, _Proj __proj = {}) const {
     return __find_unwrap(ranges::begin(__r), ranges::end(__r), __value, __proj);
   }
 };
 } // namespace __find
 
 inline namespace __cpo {
-  inline constexpr auto find = __find::__fn{};
+inline constexpr auto find = __find::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_find_end.h b/libcxx/include/__algorithm/ranges_find_end.h
index 2d46e8c38c6e2..2c57ad424bfde 100644
--- a/libcxx/include/__algorithm/ranges_find_end.h
+++ b/libcxx/include/__algorithm/ranges_find_end.h
@@ -34,18 +34,22 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 namespace __find_end {
 struct __fn {
-  template <forward_iterator _Iter1, sentinel_for<_Iter1> _Sent1,
-            forward_iterator _Iter2, sentinel_for<_Iter2> _Sent2,
-            class _Pred = ranges::equal_to,
+  template <forward_iterator _Iter1,
+            sentinel_for<_Iter1> _Sent1,
+            forward_iterator _Iter2,
+            sentinel_for<_Iter2> _Sent2,
+            class _Pred  = ranges::equal_to,
             class _Proj1 = identity,
             class _Proj2 = identity>
     requires indirectly_comparable<_Iter1, _Iter2, _Pred, _Proj1, _Proj2>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  subrange<_Iter1> operator()(_Iter1 __first1, _Sent1 __last1,
-                              _Iter2 __first2, _Sent2 __last2,
-                              _Pred __pred = {},
-                              _Proj1 __proj1 = {},
-                              _Proj2 __proj2 = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr subrange<_Iter1> operator()(
+      _Iter1 __first1,
+      _Sent1 __last1,
+      _Iter2 __first2,
+      _Sent2 __last2,
+      _Pred __pred   = {},
+      _Proj1 __proj1 = {},
+      _Proj2 __proj2 = {}) const {
     auto __ret = std::__find_end_impl<_RangeAlgPolicy>(
         __first1,
         __last1,
@@ -61,16 +65,12 @@ struct __fn {
 
   template <forward_range _Range1,
             forward_range _Range2,
-            class _Pred = ranges::equal_to,
+            class _Pred  = ranges::equal_to,
             class _Proj1 = identity,
             class _Proj2 = identity>
     requires indirectly_comparable<iterator_t<_Range1>, iterator_t<_Range2>, _Pred, _Proj1, _Proj2>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_subrange_t<_Range1> operator()(_Range1&& __range1,
-                                          _Range2&& __range2,
-                                          _Pred __pred = {},
-                                          _Proj1 __proj1 = {},
-                                          _Proj2 __proj2 = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr borrowed_subrange_t<_Range1> operator()(
+      _Range1&& __range1, _Range2&& __range2, _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
     auto __ret = std::__find_end_impl<_RangeAlgPolicy>(
         ranges::begin(__range1),
         ranges::end(__range1),
@@ -87,7 +87,7 @@ struct __fn {
 } // namespace __find_end
 
 inline namespace __cpo {
-  inline constexpr auto find_end = __find_end::__fn{};
+inline constexpr auto find_end = __find_end::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_find_first_of.h b/libcxx/include/__algorithm/ranges_find_first_of.h
index 5699583ba408e..ec6d52c63250b 100644
--- a/libcxx/include/__algorithm/ranges_find_first_of.h
+++ b/libcxx/include/__algorithm/ranges_find_first_of.h
@@ -31,14 +31,15 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 namespace __find_first_of {
 struct __fn {
-
   template <class _Iter1, class _Sent1, class _Iter2, class _Sent2, class _Pred, class _Proj1, class _Proj2>
-  _LIBCPP_HIDE_FROM_ABI constexpr static
-  _Iter1 __find_first_of_impl(_Iter1 __first1, _Sent1 __last1,
-                              _Iter2 __first2, _Sent2 __last2,
-                              _Pred& __pred,
-                              _Proj1& __proj1,
-                              _Proj2& __proj2) {
+  _LIBCPP_HIDE_FROM_ABI constexpr static _Iter1 __find_first_of_impl(
+      _Iter1 __first1,
+      _Sent1 __last1,
+      _Iter2 __first2,
+      _Sent2 __last2,
+      _Pred& __pred,
+      _Proj1& __proj1,
+      _Proj2& __proj2) {
     for (; __first1 != __last1; ++__first1) {
       for (auto __j = __first2; __j != __last2; ++__j) {
         if (std::invoke(__pred, std::invoke(__proj1, *__first1), std::invoke(__proj2, *__j)))
@@ -48,49 +49,48 @@ struct __fn {
     return __first1;
   }
 
-  template <input_iterator _Iter1, sentinel_for<_Iter1> _Sent1,
-            forward_iterator _Iter2, sentinel_for<_Iter2> _Sent2,
-            class _Pred = ranges::equal_to,
+  template <input_iterator _Iter1,
+            sentinel_for<_Iter1> _Sent1,
+            forward_iterator _Iter2,
+            sentinel_for<_Iter2> _Sent2,
+            class _Pred  = ranges::equal_to,
             class _Proj1 = identity,
             class _Proj2 = identity>
     requires indirectly_comparable<_Iter1, _Iter2, _Pred, _Proj1, _Proj2>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  _Iter1 operator()(_Iter1 __first1, _Sent1 __last1,
-                    _Iter2 __first2, _Sent2 __last2,
-                    _Pred __pred = {},
-                    _Proj1 __proj1 = {},
-                    _Proj2 __proj2 = {}) const {
-    return __find_first_of_impl(std::move(__first1), std::move(__last1),
-                                std::move(__first2), std::move(__last2),
-                                __pred,
-                                __proj1,
-                                __proj2);
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr _Iter1 operator()(
+      _Iter1 __first1,
+      _Sent1 __last1,
+      _Iter2 __first2,
+      _Sent2 __last2,
+      _Pred __pred   = {},
+      _Proj1 __proj1 = {},
+      _Proj2 __proj2 = {}) const {
+    return __find_first_of_impl(
+        std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), __pred, __proj1, __proj2);
   }
 
   template <input_range _Range1,
             forward_range _Range2,
-            class _Pred = ranges::equal_to,
+            class _Pred  = ranges::equal_to,
             class _Proj1 = identity,
             class _Proj2 = identity>
     requires indirectly_comparable<iterator_t<_Range1>, iterator_t<_Range2>, _Pred, _Proj1, _Proj2>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_iterator_t<_Range1> operator()(_Range1&& __range1,
-                                          _Range2&& __range2,
-                                          _Pred __pred = {},
-                                          _Proj1 __proj1 = {},
-                                          _Proj2 __proj2 = {}) const {
-    return __find_first_of_impl(ranges::begin(__range1), ranges::end(__range1),
-                                ranges::begin(__range2), ranges::end(__range2),
-                                __pred,
-                                __proj1,
-                                __proj2);
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range1> operator()(
+      _Range1&& __range1, _Range2&& __range2, _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
+    return __find_first_of_impl(
+        ranges::begin(__range1),
+        ranges::end(__range1),
+        ranges::begin(__range2),
+        ranges::end(__range2),
+        __pred,
+        __proj1,
+        __proj2);
   }
-
 };
 } // namespace __find_first_of
 
 inline namespace __cpo {
-  inline constexpr auto find_first_of = __find_first_of::__fn{};
+inline constexpr auto find_first_of = __find_first_of::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_find_if.h b/libcxx/include/__algorithm/ranges_find_if.h
index 6e2bd4c3ddd4a..af54a5007ee25 100644
--- a/libcxx/include/__algorithm/ranges_find_if.h
+++ b/libcxx/include/__algorithm/ranges_find_if.h
@@ -31,8 +31,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 
 template <class _Ip, class _Sp, class _Pred, class _Proj>
-_LIBCPP_HIDE_FROM_ABI constexpr
-_Ip __find_if_impl(_Ip __first, _Sp __last, _Pred& __pred, _Proj& __proj) {
+_LIBCPP_HIDE_FROM_ABI constexpr _Ip __find_if_impl(_Ip __first, _Sp __last, _Pred& __pred, _Proj& __proj) {
   for (; __first != __last; ++__first) {
     if (std::invoke(__pred, std::invoke(__proj, *__first)))
       break;
@@ -42,25 +41,25 @@ _Ip __find_if_impl(_Ip __first, _Sp __last, _Pred& __pred, _Proj& __proj) {
 
 namespace __find_if {
 struct __fn {
-
-  template <input_iterator _Ip, sentinel_for<_Ip> _Sp, class _Proj = identity,
+  template <input_iterator _Ip,
+            sentinel_for<_Ip> _Sp,
+            class _Proj = identity,
             indirect_unary_predicate<projected<_Ip, _Proj>> _Pred>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  _Ip operator()(_Ip __first, _Sp __last, _Pred __pred, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr _Ip
+  operator()(_Ip __first, _Sp __last, _Pred __pred, _Proj __proj = {}) const {
     return ranges::__find_if_impl(std::move(__first), std::move(__last), __pred, __proj);
   }
 
-  template <input_range _Rp, class _Proj = identity,
-            indirect_unary_predicate<projected<iterator_t<_Rp>, _Proj>> _Pred>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_iterator_t<_Rp> operator()(_Rp&& __r, _Pred __pred, _Proj __proj = {}) const {
+  template <input_range _Rp, class _Proj = identity, indirect_unary_predicate<projected<iterator_t<_Rp>, _Proj>> _Pred>
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Rp>
+  operator()(_Rp&& __r, _Pred __pred, _Proj __proj = {}) const {
     return ranges::__find_if_impl(ranges::begin(__r), ranges::end(__r), __pred, __proj);
   }
 };
 } // namespace __find_if
 
 inline namespace __cpo {
-  inline constexpr auto find_if = __find_if::__fn{};
+inline constexpr auto find_if = __find_if::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_find_if_not.h b/libcxx/include/__algorithm/ranges_find_if_not.h
index e60ee2b49cffa..6beade1462e09 100644
--- a/libcxx/include/__algorithm/ranges_find_if_not.h
+++ b/libcxx/include/__algorithm/ranges_find_if_not.h
@@ -33,18 +33,19 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 namespace __find_if_not {
 struct __fn {
-  template <input_iterator _Ip, sentinel_for<_Ip> _Sp, class _Proj = identity,
+  template <input_iterator _Ip,
+            sentinel_for<_Ip> _Sp,
+            class _Proj = identity,
             indirect_unary_predicate<projected<_Ip, _Proj>> _Pred>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  _Ip operator()(_Ip __first, _Sp __last, _Pred __pred, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr _Ip
+  operator()(_Ip __first, _Sp __last, _Pred __pred, _Proj __proj = {}) const {
     auto __pred2 = [&](auto&& __e) { return !std::invoke(__pred, std::forward<decltype(__e)>(__e)); };
     return ranges::__find_if_impl(std::move(__first), std::move(__last), __pred2, __proj);
   }
 
-  template <input_range _Rp, class _Proj = identity,
-            indirect_unary_predicate<projected<iterator_t<_Rp>, _Proj>> _Pred>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_iterator_t<_Rp> operator()(_Rp&& __r, _Pred __pred, _Proj __proj = {}) const {
+  template <input_range _Rp, class _Proj = identity, indirect_unary_predicate<projected<iterator_t<_Rp>, _Proj>> _Pred>
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Rp>
+  operator()(_Rp&& __r, _Pred __pred, _Proj __proj = {}) const {
     auto __pred2 = [&](auto&& __e) { return !std::invoke(__pred, std::forward<decltype(__e)>(__e)); };
     return ranges::__find_if_impl(ranges::begin(__r), ranges::end(__r), __pred2, __proj);
   }
@@ -52,7 +53,7 @@ struct __fn {
 } // namespace __find_if_not
 
 inline namespace __cpo {
-  inline constexpr auto find_if_not = __find_if_not::__fn{};
+inline constexpr auto find_if_not = __find_if_not::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_for_each.h b/libcxx/include/__algorithm/ranges_for_each.h
index a72f77963759f..7878ed26709fb 100644
--- a/libcxx/include/__algorithm/ranges_for_each.h
+++ b/libcxx/include/__algorithm/ranges_for_each.h
@@ -37,37 +37,35 @@ namespace __for_each {
 struct __fn {
 private:
   template <class _Iter, class _Sent, class _Proj, class _Func>
-  _LIBCPP_HIDE_FROM_ABI constexpr static
-  for_each_result<_Iter, _Func> __for_each_impl(_Iter __first, _Sent __last, _Func& __func, _Proj& __proj) {
+  _LIBCPP_HIDE_FROM_ABI constexpr static for_each_result<_Iter, _Func>
+  __for_each_impl(_Iter __first, _Sent __last, _Func& __func, _Proj& __proj) {
     for (; __first != __last; ++__first)
       std::invoke(__func, std::invoke(__proj, *__first));
     return {std::move(__first), std::move(__func)};
   }
 
 public:
-  template <input_iterator _Iter, sentinel_for<_Iter> _Sent,
+  template <input_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
             class _Proj = identity,
             indirectly_unary_invocable<projected<_Iter, _Proj>> _Func>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  for_each_result<_Iter, _Func> operator()(_Iter __first, _Sent __last, _Func __func, _Proj __proj = {}) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr for_each_result<_Iter, _Func>
+  operator()(_Iter __first, _Sent __last, _Func __func, _Proj __proj = {}) const {
     return __for_each_impl(std::move(__first), std::move(__last), __func, __proj);
   }
 
   template <input_range _Range,
             class _Proj = identity,
             indirectly_unary_invocable<projected<iterator_t<_Range>, _Proj>> _Func>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  for_each_result<borrowed_iterator_t<_Range>, _Func> operator()(_Range&& __range,
-                                                                 _Func __func,
-                                                                 _Proj __proj = {}) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr for_each_result<borrowed_iterator_t<_Range>, _Func>
+  operator()(_Range&& __range, _Func __func, _Proj __proj = {}) const {
     return __for_each_impl(ranges::begin(__range), ranges::end(__range), __func, __proj);
   }
-
 };
 } // namespace __for_each
 
 inline namespace __cpo {
-  inline constexpr auto for_each = __for_each::__fn{};
+inline constexpr auto for_each = __for_each::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_for_each_n.h b/libcxx/include/__algorithm/ranges_for_each_n.h
index daf0a5d828299..53ccb9a6035a4 100644
--- a/libcxx/include/__algorithm/ranges_for_each_n.h
+++ b/libcxx/include/__algorithm/ranges_for_each_n.h
@@ -35,27 +35,20 @@ using for_each_n_result = in_fun_result<_Iter, _Func>;
 
 namespace __for_each_n {
 struct __fn {
-
-  template <input_iterator _Iter,
-            class _Proj = identity,
-            indirectly_unary_invocable<projected<_Iter, _Proj>> _Func>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  for_each_n_result<_Iter, _Func> operator()(_Iter __first,
-                                             iter_difference_t<_Iter> __count,
-                                             _Func __func,
-                                             _Proj __proj = {}) const {
+  template <input_iterator _Iter, class _Proj = identity, indirectly_unary_invocable<projected<_Iter, _Proj>> _Func>
+  _LIBCPP_HIDE_FROM_ABI constexpr for_each_n_result<_Iter, _Func>
+  operator()(_Iter __first, iter_difference_t<_Iter> __count, _Func __func, _Proj __proj = {}) const {
     while (__count-- > 0) {
       std::invoke(__func, std::invoke(__proj, *__first));
       ++__first;
     }
     return {std::move(__first), std::move(__func)};
   }
-
 };
 } // namespace __for_each_n
 
 inline namespace __cpo {
-  inline constexpr auto for_each_n = __for_each_n::__fn{};
+inline constexpr auto for_each_n = __for_each_n::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_generate.h b/libcxx/include/__algorithm/ranges_generate.h
index de0db1665f1e0..3ff1e13c42209 100644
--- a/libcxx/include/__algorithm/ranges_generate.h
+++ b/libcxx/include/__algorithm/ranges_generate.h
@@ -32,10 +32,8 @@ namespace ranges {
 namespace __generate {
 
 struct __fn {
-
   template <class _OutIter, class _Sent, class _Func>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  static _OutIter __generate_fn_impl(_OutIter __first, _Sent __last, _Func& __gen) {
+  _LIBCPP_HIDE_FROM_ABI constexpr static _OutIter __generate_fn_impl(_OutIter __first, _Sent __last, _Func& __gen) {
     for (; __first != __last; ++__first) {
       *__first = __gen();
     }
@@ -44,25 +42,22 @@ struct __fn {
   }
 
   template <input_or_output_iterator _OutIter, sentinel_for<_OutIter> _Sent, copy_constructible _Func>
-  requires invocable<_Func&> && indirectly_writable<_OutIter, invoke_result_t<_Func&>>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  _OutIter operator()(_OutIter __first, _Sent __last, _Func __gen) const {
+    requires invocable<_Func&> && indirectly_writable<_OutIter, invoke_result_t<_Func&>>
+  _LIBCPP_HIDE_FROM_ABI constexpr _OutIter operator()(_OutIter __first, _Sent __last, _Func __gen) const {
     return __generate_fn_impl(std::move(__first), std::move(__last), __gen);
   }
 
   template <class _Range, copy_constructible _Func>
-  requires invocable<_Func&> && output_range<_Range, invoke_result_t<_Func&>>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_iterator_t<_Range> operator()(_Range&& __range, _Func __gen) const {
+    requires invocable<_Func&> && output_range<_Range, invoke_result_t<_Func&>>
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range> operator()(_Range&& __range, _Func __gen) const {
     return __generate_fn_impl(ranges::begin(__range), ranges::end(__range), __gen);
   }
-
 };
 
 } // namespace __generate
 
 inline namespace __cpo {
-  inline constexpr auto generate = __generate::__fn{};
+inline constexpr auto generate = __generate::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_generate_n.h b/libcxx/include/__algorithm/ranges_generate_n.h
index 122cd8db79494..c025c621a191c 100644
--- a/libcxx/include/__algorithm/ranges_generate_n.h
+++ b/libcxx/include/__algorithm/ranges_generate_n.h
@@ -33,11 +33,10 @@ namespace ranges {
 namespace __generate_n {
 
 struct __fn {
-
   template <input_or_output_iterator _OutIter, copy_constructible _Func>
-  requires invocable<_Func&> && indirectly_writable<_OutIter, invoke_result_t<_Func&>>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  _OutIter operator()(_OutIter __first, iter_difference_t<_OutIter> __n, _Func __gen) const {
+    requires invocable<_Func&> && indirectly_writable<_OutIter, invoke_result_t<_Func&>>
+  _LIBCPP_HIDE_FROM_ABI constexpr _OutIter
+  operator()(_OutIter __first, iter_difference_t<_OutIter> __n, _Func __gen) const {
     for (; __n > 0; --__n) {
       *__first = __gen();
       ++__first;
@@ -45,13 +44,12 @@ struct __fn {
 
     return __first;
   }
-
 };
 
 } // namespace __generate_n
 
 inline namespace __cpo {
-  inline constexpr auto generate_n = __generate_n::__fn{};
+inline constexpr auto generate_n = __generate_n::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_includes.h b/libcxx/include/__algorithm/ranges_includes.h
index 314a923771f33..aa35080c8cfd4 100644
--- a/libcxx/include/__algorithm/ranges_includes.h
+++ b/libcxx/include/__algorithm/ranges_includes.h
@@ -35,14 +35,13 @@ namespace ranges {
 namespace __includes {
 
 struct __fn {
-  template <
-      input_iterator _Iter1,
-      sentinel_for<_Iter1> _Sent1,
-      input_iterator _Iter2,
-      sentinel_for<_Iter2> _Sent2,
-      class _Proj1                                                                           = identity,
-      class _Proj2                                                                           = identity,
-      indirect_strict_weak_order<projected<_Iter1, _Proj1>, projected<_Iter2, _Proj2>> _Comp = ranges::less>
+  template <input_iterator _Iter1,
+            sentinel_for<_Iter1> _Sent1,
+            input_iterator _Iter2,
+            sentinel_for<_Iter2> _Sent2,
+            class _Proj1                                                                           = identity,
+            class _Proj2                                                                           = identity,
+            indirect_strict_weak_order<projected<_Iter1, _Proj1>, projected<_Iter2, _Proj2>> _Comp = ranges::less>
   _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(
       _Iter1 __first1,
       _Sent1 __last1,
@@ -61,13 +60,12 @@ struct __fn {
         std::move(__proj2));
   }
 
-  template <
-      input_range _Range1,
-      input_range _Range2,
-      class _Proj1 = identity,
-      class _Proj2 = identity,
-      indirect_strict_weak_order<projected<iterator_t<_Range1>, _Proj1>, projected<iterator_t<_Range2>, _Proj2>>
-          _Comp = ranges::less>
+  template <input_range _Range1,
+            input_range _Range2,
+            class _Proj1 = identity,
+            class _Proj2 = identity,
+            indirect_strict_weak_order<projected<iterator_t<_Range1>, _Proj1>, projected<iterator_t<_Range2>, _Proj2>>
+                _Comp = ranges::less>
   _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(
       _Range1&& __range1, _Range2&& __range2, _Comp __comp = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
     return std::__includes(
@@ -84,7 +82,7 @@ struct __fn {
 } // namespace __includes
 
 inline namespace __cpo {
-  inline constexpr auto includes = __includes::__fn{};
+inline constexpr auto includes = __includes::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_inplace_merge.h b/libcxx/include/__algorithm/ranges_inplace_merge.h
index 8f78975d78977..86001b003d5ca 100644
--- a/libcxx/include/__algorithm/ranges_inplace_merge.h
+++ b/libcxx/include/__algorithm/ranges_inplace_merge.h
@@ -38,43 +38,37 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 namespace __inplace_merge {
 
-  struct __fn {
-    template <class _Iter, class _Sent, class _Comp, class _Proj>
-    _LIBCPP_HIDE_FROM_ABI static constexpr auto
-    __inplace_merge_impl(_Iter __first, _Iter __middle, _Sent __last, _Comp&& __comp, _Proj&& __proj) {
-      auto __last_iter = ranges::next(__middle, __last);
-      std::__inplace_merge<_RangeAlgPolicy>(
-          std::move(__first), std::move(__middle), __last_iter, std::__make_projected(__comp, __proj));
-      return __last_iter;
-    }
+struct __fn {
+  template <class _Iter, class _Sent, class _Comp, class _Proj>
+  _LIBCPP_HIDE_FROM_ABI static constexpr auto
+  __inplace_merge_impl(_Iter __first, _Iter __middle, _Sent __last, _Comp&& __comp, _Proj&& __proj) {
+    auto __last_iter = ranges::next(__middle, __last);
+    std::__inplace_merge<_RangeAlgPolicy>(
+        std::move(__first), std::move(__middle), __last_iter, std::__make_projected(__comp, __proj));
+    return __last_iter;
+  }
 
-    template <
-        bidirectional_iterator _Iter,
-        sentinel_for<_Iter> _Sent,
-        class _Comp = ranges::less,
-        class _Proj = identity>
-      requires sortable<_Iter, _Comp, _Proj>
-    _LIBCPP_HIDE_FROM_ABI _Iter
-    operator()(_Iter __first, _Iter __middle, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
-      return __inplace_merge_impl(
-          std::move(__first), std::move(__middle), std::move(__last), std::move(__comp), std::move(__proj));
-    }
+  template <bidirectional_iterator _Iter, sentinel_for<_Iter> _Sent, class _Comp = ranges::less, class _Proj = identity>
+    requires sortable<_Iter, _Comp, _Proj>
+  _LIBCPP_HIDE_FROM_ABI _Iter
+  operator()(_Iter __first, _Iter __middle, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+    return __inplace_merge_impl(
+        std::move(__first), std::move(__middle), std::move(__last), std::move(__comp), std::move(__proj));
+  }
 
-    template <bidirectional_range _Range, class _Comp = ranges::less, class _Proj = identity>
-      requires sortable<
-          iterator_t<_Range>,
-          _Comp,
-          _Proj> _LIBCPP_HIDE_FROM_ABI borrowed_iterator_t<_Range>
-      operator()(_Range&& __range, iterator_t<_Range> __middle, _Comp __comp = {}, _Proj __proj = {}) const {
-      return __inplace_merge_impl(
-          ranges::begin(__range), std::move(__middle), ranges::end(__range), std::move(__comp), std::move(__proj));
-    }
-  };
+  template <bidirectional_range _Range, class _Comp = ranges::less, class _Proj = identity>
+    requires sortable<iterator_t<_Range>, _Comp, _Proj>
+  _LIBCPP_HIDE_FROM_ABI borrowed_iterator_t<_Range>
+  operator()(_Range&& __range, iterator_t<_Range> __middle, _Comp __comp = {}, _Proj __proj = {}) const {
+    return __inplace_merge_impl(
+        ranges::begin(__range), std::move(__middle), ranges::end(__range), std::move(__comp), std::move(__proj));
+  }
+};
 
 } // namespace __inplace_merge
 
 inline namespace __cpo {
-  inline constexpr auto inplace_merge = __inplace_merge::__fn{};
+inline constexpr auto inplace_merge = __inplace_merge::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_is_heap.h b/libcxx/include/__algorithm/ranges_is_heap.h
index 9c77e0c3f0717..f298c347b747a 100644
--- a/libcxx/include/__algorithm/ranges_is_heap.h
+++ b/libcxx/include/__algorithm/ranges_is_heap.h
@@ -34,28 +34,30 @@ namespace ranges {
 namespace __is_heap {
 
 struct __fn {
-
   template <class _Iter, class _Sent, class _Proj, class _Comp>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  static bool __is_heap_fn_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
-    auto __last_iter = ranges::next(__first, __last);
+  _LIBCPP_HIDE_FROM_ABI constexpr static bool
+  __is_heap_fn_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
+    auto __last_iter        = ranges::next(__first, __last);
     auto&& __projected_comp = std::__make_projected(__comp, __proj);
 
     auto __result = std::__is_heap_until(std::move(__first), std::move(__last_iter), __projected_comp);
     return __result == __last;
   }
 
-  template <random_access_iterator _Iter, sentinel_for<_Iter> _Sent, class _Proj = identity,
+  template <random_access_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj                                               = identity,
             indirect_strict_weak_order<projected<_Iter, _Proj>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  bool operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
     return __is_heap_fn_impl(std::move(__first), std::move(__last), __comp, __proj);
   }
 
-  template <random_access_range _Range, class _Proj = identity,
+  template <random_access_range _Range,
+            class _Proj                                                            = identity,
             indirect_strict_weak_order<projected<iterator_t<_Range>, _Proj>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  bool operator()(_Range&& __range, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Range&& __range, _Comp __comp = {}, _Proj __proj = {}) const {
     return __is_heap_fn_impl(ranges::begin(__range), ranges::end(__range), __comp, __proj);
   }
 };
@@ -63,7 +65,7 @@ struct __fn {
 } // namespace __is_heap
 
 inline namespace __cpo {
-  inline constexpr auto is_heap = __is_heap::__fn{};
+inline constexpr auto is_heap = __is_heap::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_is_heap_until.h b/libcxx/include/__algorithm/ranges_is_heap_until.h
index 3b1fec758fa64..73f13fb50440e 100644
--- a/libcxx/include/__algorithm/ranges_is_heap_until.h
+++ b/libcxx/include/__algorithm/ranges_is_heap_until.h
@@ -35,36 +35,37 @@ namespace ranges {
 namespace __is_heap_until {
 
 struct __fn {
-
   template <class _Iter, class _Sent, class _Proj, class _Comp>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  static _Iter __is_heap_until_fn_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
-    auto __last_iter = ranges::next(__first, __last);
+  _LIBCPP_HIDE_FROM_ABI constexpr static _Iter
+  __is_heap_until_fn_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
+    auto __last_iter        = ranges::next(__first, __last);
     auto&& __projected_comp = std::__make_projected(__comp, __proj);
 
     return std::__is_heap_until(std::move(__first), std::move(__last_iter), __projected_comp);
   }
 
-  template <random_access_iterator _Iter, sentinel_for<_Iter> _Sent, class _Proj = identity,
+  template <random_access_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj                                               = identity,
             indirect_strict_weak_order<projected<_Iter, _Proj>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  _Iter operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
     return __is_heap_until_fn_impl(std::move(__first), std::move(__last), __comp, __proj);
   }
 
-  template <random_access_range _Range, class _Proj = identity,
+  template <random_access_range _Range,
+            class _Proj                                                            = identity,
             indirect_strict_weak_order<projected<iterator_t<_Range>, _Proj>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_iterator_t<_Range> operator()(_Range&& __range, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __range, _Comp __comp = {}, _Proj __proj = {}) const {
     return __is_heap_until_fn_impl(ranges::begin(__range), ranges::end(__range), __comp, __proj);
   }
-
 };
 
 } // namespace __is_heap_until
 
 inline namespace __cpo {
-  inline constexpr auto is_heap_until = __is_heap_until::__fn{};
+inline constexpr auto is_heap_until = __is_heap_until::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_is_partitioned.h b/libcxx/include/__algorithm/ranges_is_partitioned.h
index 6782717ccacad..f81c9c311a5b5 100644
--- a/libcxx/include/__algorithm/ranges_is_partitioned.h
+++ b/libcxx/include/__algorithm/ranges_is_partitioned.h
@@ -30,10 +30,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 namespace __is_partitioned {
 struct __fn {
-
   template <class _Iter, class _Sent, class _Proj, class _Pred>
-  _LIBCPP_HIDE_FROM_ABI constexpr static
-  bool __is_parititioned_impl(_Iter __first, _Sent __last, _Pred& __pred, _Proj& __proj) {
+  _LIBCPP_HIDE_FROM_ABI constexpr static bool
+  __is_parititioned_impl(_Iter __first, _Sent __last, _Pred& __pred, _Proj& __proj) {
     for (; __first != __last; ++__first) {
       if (!std::invoke(__pred, std::invoke(__proj, *__first)))
         break;
@@ -51,26 +50,27 @@ struct __fn {
     return true;
   }
 
-  template <input_iterator _Iter, sentinel_for<_Iter> _Sent,
+  template <input_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
             class _Proj = identity,
             indirect_unary_predicate<projected<_Iter, _Proj>> _Pred>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  bool operator()(_Iter __first, _Sent __last, _Pred __pred, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Iter __first, _Sent __last, _Pred __pred, _Proj __proj = {}) const {
     return __is_parititioned_impl(std::move(__first), std::move(__last), __pred, __proj);
   }
 
   template <input_range _Range,
             class _Proj = identity,
             indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  bool operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) const {
     return __is_parititioned_impl(ranges::begin(__range), ranges::end(__range), __pred, __proj);
   }
 };
 } // namespace __is_partitioned
 
 inline namespace __cpo {
-  inline constexpr auto is_partitioned = __is_partitioned::__fn{};
+inline constexpr auto is_partitioned = __is_partitioned::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_is_permutation.h b/libcxx/include/__algorithm/ranges_is_permutation.h
index 95a0a82514304..2b99839bc66fa 100644
--- a/libcxx/include/__algorithm/ranges_is_permutation.h
+++ b/libcxx/include/__algorithm/ranges_is_permutation.h
@@ -32,53 +32,66 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 namespace __is_permutation {
 struct __fn {
-
-  template <class _Iter1, class _Sent1, class _Iter2, class _Sent2,
-            class _Proj1, class _Proj2, class _Pred>
-  _LIBCPP_HIDE_FROM_ABI constexpr static
-  bool __is_permutation_func_impl(_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Sent2 __last2,
-                                  _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) {
+  template <class _Iter1, class _Sent1, class _Iter2, class _Sent2, class _Proj1, class _Proj2, class _Pred>
+  _LIBCPP_HIDE_FROM_ABI constexpr static bool __is_permutation_func_impl(
+      _Iter1 __first1,
+      _Sent1 __last1,
+      _Iter2 __first2,
+      _Sent2 __last2,
+      _Pred& __pred,
+      _Proj1& __proj1,
+      _Proj2& __proj2) {
     return std::__is_permutation<_RangeAlgPolicy>(
-        std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2),
-        __pred, __proj1, __proj2);
+        std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), __pred, __proj1, __proj2);
   }
 
-  template <forward_iterator _Iter1, sentinel_for<_Iter1> _Sent1,
-            forward_iterator _Iter2, sentinel_for<_Iter2> _Sent2,
-            class _Proj1 = identity,
-            class _Proj2 = identity,
-            indirect_equivalence_relation<projected<_Iter1, _Proj1>,
-                                          projected<_Iter2, _Proj2>> _Pred = ranges::equal_to>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  bool operator()(_Iter1 __first1, _Sent1 __last1, _Iter2 __first2, _Sent2 __last2,
-                  _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
+  template <
+      forward_iterator _Iter1,
+      sentinel_for<_Iter1> _Sent1,
+      forward_iterator _Iter2,
+      sentinel_for<_Iter2> _Sent2,
+      class _Proj1                                                                              = identity,
+      class _Proj2                                                                              = identity,
+      indirect_equivalence_relation<projected<_Iter1, _Proj1>, projected<_Iter2, _Proj2>> _Pred = ranges::equal_to>
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(
+      _Iter1 __first1,
+      _Sent1 __last1,
+      _Iter2 __first2,
+      _Sent2 __last2,
+      _Pred __pred   = {},
+      _Proj1 __proj1 = {},
+      _Proj2 __proj2 = {}) const {
     return __is_permutation_func_impl(
-        std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2),
-        __pred, __proj1, __proj2);
+        std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), __pred, __proj1, __proj2);
   }
 
   template <forward_range _Range1,
             forward_range _Range2,
-            class _Proj1 = identity,
-            class _Proj2 = identity,
-            indirect_equivalence_relation<projected<iterator_t<_Range1>, _Proj1>, projected<iterator_t<_Range2>, _Proj2>> _Pred = ranges::equal_to>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  bool operator()(_Range1&& __range1, _Range2&& __range2,
-                  _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
+            class _Proj1                                                                = identity,
+            class _Proj2                                                                = identity,
+            indirect_equivalence_relation<projected<iterator_t<_Range1>, _Proj1>,
+                                          projected<iterator_t<_Range2>, _Proj2>> _Pred = ranges::equal_to>
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(
+      _Range1&& __range1, _Range2&& __range2, _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
     if constexpr (sized_range<_Range1> && sized_range<_Range2>) {
       if (ranges::distance(__range1) != ranges::distance(__range2))
         return false;
     }
 
     return __is_permutation_func_impl(
-        ranges::begin(__range1), ranges::end(__range1), ranges::begin(__range2), ranges::end(__range2),
-        __pred, __proj1, __proj2);
+        ranges::begin(__range1),
+        ranges::end(__range1),
+        ranges::begin(__range2),
+        ranges::end(__range2),
+        __pred,
+        __proj1,
+        __proj2);
   }
 };
 } // namespace __is_permutation
 
 inline namespace __cpo {
-  inline constexpr auto is_permutation = __is_permutation::__fn{};
+inline constexpr auto is_permutation = __is_permutation::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_is_sorted.h b/libcxx/include/__algorithm/ranges_is_sorted.h
index 50c97baa83c32..3eb2c768d66a2 100644
--- a/libcxx/include/__algorithm/ranges_is_sorted.h
+++ b/libcxx/include/__algorithm/ranges_is_sorted.h
@@ -30,19 +30,20 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 namespace __is_sorted {
 struct __fn {
-  template <forward_iterator _Iter, sentinel_for<_Iter> _Sent,
-            class _Proj = identity,
+  template <forward_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj                                               = identity,
             indirect_strict_weak_order<projected<_Iter, _Proj>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  bool operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
     return ranges::__is_sorted_until_impl(std::move(__first), __last, __comp, __proj) == __last;
   }
 
   template <forward_range _Range,
-            class _Proj = identity,
+            class _Proj                                                            = identity,
             indirect_strict_weak_order<projected<iterator_t<_Range>, _Proj>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  bool operator()(_Range&& __range, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Range&& __range, _Comp __comp = {}, _Proj __proj = {}) const {
     auto __last = ranges::end(__range);
     return ranges::__is_sorted_until_impl(ranges::begin(__range), __last, __comp, __proj) == __last;
   }
@@ -50,7 +51,7 @@ struct __fn {
 } // namespace __is_sorted
 
 inline namespace __cpo {
-  inline constexpr auto is_sorted = __is_sorted::__fn{};
+inline constexpr auto is_sorted = __is_sorted::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_is_sorted_until.h b/libcxx/include/__algorithm/ranges_is_sorted_until.h
index f139d0391de91..19e9875d2757d 100644
--- a/libcxx/include/__algorithm/ranges_is_sorted_until.h
+++ b/libcxx/include/__algorithm/ranges_is_sorted_until.h
@@ -31,8 +31,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 
 template <class _Iter, class _Sent, class _Proj, class _Comp>
-_LIBCPP_HIDE_FROM_ABI constexpr
-_Iter __is_sorted_until_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
+_LIBCPP_HIDE_FROM_ABI constexpr _Iter
+__is_sorted_until_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
   if (__first == __last)
     return __first;
   auto __i = __first;
@@ -46,26 +46,27 @@ _Iter __is_sorted_until_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj&
 
 namespace __is_sorted_until {
 struct __fn {
-  template <forward_iterator _Iter, sentinel_for<_Iter> _Sent,
-            class _Proj = identity,
+  template <forward_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj                                               = identity,
             indirect_strict_weak_order<projected<_Iter, _Proj>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  _Iter operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
     return ranges::__is_sorted_until_impl(std::move(__first), std::move(__last), __comp, __proj);
   }
 
   template <forward_range _Range,
-            class _Proj = identity,
+            class _Proj                                                            = identity,
             indirect_strict_weak_order<projected<iterator_t<_Range>, _Proj>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_iterator_t<_Range> operator()(_Range&& __range, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __range, _Comp __comp = {}, _Proj __proj = {}) const {
     return ranges::__is_sorted_until_impl(ranges::begin(__range), ranges::end(__range), __comp, __proj);
   }
 };
 } // namespace __is_sorted_until
 
 inline namespace __cpo {
-  inline constexpr auto is_sorted_until = __is_sorted_until::__fn{};
+inline constexpr auto is_sorted_until = __is_sorted_until::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_lexicographical_compare.h b/libcxx/include/__algorithm/ranges_lexicographical_compare.h
index c51f4d7e5ca0c..5b843dfd7b313 100644
--- a/libcxx/include/__algorithm/ranges_lexicographical_compare.h
+++ b/libcxx/include/__algorithm/ranges_lexicographical_compare.h
@@ -30,17 +30,17 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 namespace __lexicographical_compare {
 struct __fn {
-
   template <class _Iter1, class _Sent1, class _Iter2, class _Sent2, class _Proj1, class _Proj2, class _Comp>
-  _LIBCPP_HIDE_FROM_ABI constexpr static
-  bool __lexicographical_compare_impl(_Iter1 __first1, _Sent1 __last1,
-                                      _Iter2 __first2, _Sent2 __last2,
-                                      _Comp& __comp,
-                                      _Proj1& __proj1,
-                                      _Proj2& __proj2) {
+  _LIBCPP_HIDE_FROM_ABI constexpr static bool __lexicographical_compare_impl(
+      _Iter1 __first1,
+      _Sent1 __last1,
+      _Iter2 __first2,
+      _Sent2 __last2,
+      _Comp& __comp,
+      _Proj1& __proj1,
+      _Proj2& __proj2) {
     while (__first2 != __last2) {
-      if (__first1 == __last1
-       || std::invoke(__comp, std::invoke(__proj1, *__first1), std::invoke(__proj2, *__first2)))
+      if (__first1 == __last1 || std::invoke(__comp, std::invoke(__proj1, *__first1), std::invoke(__proj2, *__first2)))
         return true;
       if (std::invoke(__comp, std::invoke(__proj2, *__first2), std::invoke(__proj1, *__first1)))
         return false;
@@ -50,44 +50,47 @@ struct __fn {
     return false;
   }
 
-  template <input_iterator _Iter1, sentinel_for<_Iter1> _Sent1,
-            input_iterator _Iter2, sentinel_for<_Iter2> _Sent2,
-            class _Proj1 = identity,
-            class _Proj2 = identity,
+  template <input_iterator _Iter1,
+            sentinel_for<_Iter1> _Sent1,
+            input_iterator _Iter2,
+            sentinel_for<_Iter2> _Sent2,
+            class _Proj1                                                                           = identity,
+            class _Proj2                                                                           = identity,
             indirect_strict_weak_order<projected<_Iter1, _Proj1>, projected<_Iter2, _Proj2>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  bool operator()(_Iter1 __first1, _Sent1 __last1,
-                  _Iter2 __first2, _Sent2 __last2,
-                  _Comp __comp = {},
-                  _Proj1 __proj1 = {},
-                  _Proj2 __proj2 = {}) const {
-    return __lexicographical_compare_impl(std::move(__first1), std::move(__last1),
-                                          std::move(__first2), std::move(__last2),
-                                          __comp,
-                                          __proj1,
-                                          __proj2);
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(
+      _Iter1 __first1,
+      _Sent1 __last1,
+      _Iter2 __first2,
+      _Sent2 __last2,
+      _Comp __comp   = {},
+      _Proj1 __proj1 = {},
+      _Proj2 __proj2 = {}) const {
+    return __lexicographical_compare_impl(
+        std::move(__first1), std::move(__last1), std::move(__first2), std::move(__last2), __comp, __proj1, __proj2);
   }
 
   template <input_range _Range1,
             input_range _Range2,
             class _Proj1 = identity,
             class _Proj2 = identity,
-            indirect_strict_weak_order<projected<iterator_t<_Range1>, _Proj1>,
-                                       projected<iterator_t<_Range2>, _Proj2>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  bool operator()(_Range1&& __range1, _Range2&& __range2, _Comp __comp = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
-    return __lexicographical_compare_impl(ranges::begin(__range1), ranges::end(__range1),
-                                          ranges::begin(__range2), ranges::end(__range2),
-                                          __comp,
-                                          __proj1,
-                                          __proj2);
+            indirect_strict_weak_order<projected<iterator_t<_Range1>, _Proj1>, projected<iterator_t<_Range2>, _Proj2>>
+                _Comp = ranges::less>
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool operator()(
+      _Range1&& __range1, _Range2&& __range2, _Comp __comp = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
+    return __lexicographical_compare_impl(
+        ranges::begin(__range1),
+        ranges::end(__range1),
+        ranges::begin(__range2),
+        ranges::end(__range2),
+        __comp,
+        __proj1,
+        __proj2);
   }
-
 };
 } // namespace __lexicographical_compare
 
 inline namespace __cpo {
-  inline constexpr auto lexicographical_compare = __lexicographical_compare::__fn{};
+inline constexpr auto lexicographical_compare = __lexicographical_compare::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_lower_bound.h b/libcxx/include/__algorithm/ranges_lower_bound.h
index 3293886fda4db..58b3f815b96a4 100644
--- a/libcxx/include/__algorithm/ranges_lower_bound.h
+++ b/libcxx/include/__algorithm/ranges_lower_bound.h
@@ -35,27 +35,29 @@ namespace ranges {
 
 namespace __lower_bound {
 struct __fn {
-  template <forward_iterator _Iter, sentinel_for<_Iter> _Sent, class _Type, class _Proj = identity,
+  template <forward_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Type,
+            class _Proj                                                             = identity,
             indirect_strict_weak_order<const _Type*, projected<_Iter, _Proj>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  _Iter operator()(_Iter __first, _Sent __last, const _Type& __value, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, _Sent __last, const _Type& __value, _Comp __comp = {}, _Proj __proj = {}) const {
     return std::__lower_bound<_RangeAlgPolicy>(__first, __last, __value, __comp, __proj);
   }
 
-  template <forward_range _Range, class _Type, class _Proj = identity,
+  template <forward_range _Range,
+            class _Type,
+            class _Proj                                                                          = identity,
             indirect_strict_weak_order<const _Type*, projected<iterator_t<_Range>, _Proj>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_iterator_t<_Range> operator()(_Range&& __r,
-                                         const _Type& __value,
-                                         _Comp __comp = {},
-                                         _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __r, const _Type& __value, _Comp __comp = {}, _Proj __proj = {}) const {
     return std::__lower_bound<_RangeAlgPolicy>(ranges::begin(__r), ranges::end(__r), __value, __comp, __proj);
   }
 };
 } // namespace __lower_bound
 
 inline namespace __cpo {
-  inline constexpr auto lower_bound = __lower_bound::__fn{};
+inline constexpr auto lower_bound = __lower_bound::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_make_heap.h b/libcxx/include/__algorithm/ranges_make_heap.h
index 7f92fa3104371..f17eabff43d2a 100644
--- a/libcxx/include/__algorithm/ranges_make_heap.h
+++ b/libcxx/include/__algorithm/ranges_make_heap.h
@@ -41,8 +41,8 @@ namespace __make_heap {
 
 struct __fn {
   template <class _Iter, class _Sent, class _Comp, class _Proj>
-  _LIBCPP_HIDE_FROM_ABI constexpr static
-  _Iter __make_heap_fn_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
+  _LIBCPP_HIDE_FROM_ABI constexpr static _Iter
+  __make_heap_fn_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
     auto __last_iter = ranges::next(__first, __last);
 
     auto&& __projected_comp = std::__make_projected(__comp, __proj);
@@ -53,15 +53,15 @@ struct __fn {
 
   template <random_access_iterator _Iter, sentinel_for<_Iter> _Sent, class _Comp = ranges::less, class _Proj = identity>
     requires sortable<_Iter, _Comp, _Proj>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  _Iter operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
     return __make_heap_fn_impl(std::move(__first), std::move(__last), __comp, __proj);
   }
 
   template <random_access_range _Range, class _Comp = ranges::less, class _Proj = identity>
     requires sortable<iterator_t<_Range>, _Comp, _Proj>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_iterator_t<_Range> operator()(_Range&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
     return __make_heap_fn_impl(ranges::begin(__r), ranges::end(__r), __comp, __proj);
   }
 };
@@ -69,7 +69,7 @@ struct __fn {
 } // namespace __make_heap
 
 inline namespace __cpo {
-  inline constexpr auto make_heap = __make_heap::__fn{};
+inline constexpr auto make_heap = __make_heap::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_max.h b/libcxx/include/__algorithm/ranges_max.h
index 2fd2970bbf35d..321cac56e29b0 100644
--- a/libcxx/include/__algorithm/ranges_max.h
+++ b/libcxx/include/__algorithm/ranges_max.h
@@ -31,40 +31,43 @@
 #if _LIBCPP_STD_VER >= 20
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 namespace ranges {
 namespace __max {
 struct __fn {
-  template <class _Tp, class _Proj = identity,
+  template <class _Tp,
+            class _Proj                                                    = identity,
             indirect_strict_weak_order<projected<const _Tp*, _Proj>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  const _Tp& operator()(_LIBCPP_LIFETIMEBOUND const _Tp& __a,
-                        _LIBCPP_LIFETIMEBOUND const _Tp& __b,
-                        _Comp __comp = {},
-                        _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr const _Tp&
+  operator()(_LIBCPP_LIFETIMEBOUND const _Tp& __a,
+             _LIBCPP_LIFETIMEBOUND const _Tp& __b,
+             _Comp __comp = {},
+             _Proj __proj = {}) const {
     return std::invoke(__comp, std::invoke(__proj, __a), std::invoke(__proj, __b)) ? __b : __a;
   }
 
-  template <copyable _Tp, class _Proj = identity,
+  template <copyable _Tp,
+            class _Proj                                                    = identity,
             indirect_strict_weak_order<projected<const _Tp*, _Proj>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  _Tp operator()(initializer_list<_Tp> __il, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr _Tp
+  operator()(initializer_list<_Tp> __il, _Comp __comp = {}, _Proj __proj = {}) const {
     _LIBCPP_ASSERT(__il.begin() != __il.end(), "initializer_list must contain at least one element");
 
     auto __comp_lhs_rhs_swapped = [&](auto&& __lhs, auto&& __rhs) { return std::invoke(__comp, __rhs, __lhs); };
     return *ranges::__min_element_impl(__il.begin(), __il.end(), __comp_lhs_rhs_swapped, __proj);
   }
 
-  template <input_range _Rp, class _Proj = identity,
+  template <input_range _Rp,
+            class _Proj                                                         = identity,
             indirect_strict_weak_order<projected<iterator_t<_Rp>, _Proj>> _Comp = ranges::less>
     requires indirectly_copyable_storable<iterator_t<_Rp>, range_value_t<_Rp>*>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  range_value_t<_Rp> operator()(_Rp&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr range_value_t<_Rp>
+  operator()(_Rp&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
     auto __first = ranges::begin(__r);
-    auto __last = ranges::end(__r);
+    auto __last  = ranges::end(__r);
 
     _LIBCPP_ASSERT(__first != __last, "range must contain at least one element");
 
@@ -84,7 +87,7 @@ struct __fn {
 } // namespace __max
 
 inline namespace __cpo {
-  inline constexpr auto max = __max::__fn{};
+inline constexpr auto max = __max::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_max_element.h b/libcxx/include/__algorithm/ranges_max_element.h
index 39e86d3b9a60c..2d92661c81099 100644
--- a/libcxx/include/__algorithm/ranges_max_element.h
+++ b/libcxx/include/__algorithm/ranges_max_element.h
@@ -31,18 +31,21 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 namespace __max_element {
 struct __fn {
-  template <forward_iterator _Ip, sentinel_for<_Ip> _Sp, class _Proj = identity,
+  template <forward_iterator _Ip,
+            sentinel_for<_Ip> _Sp,
+            class _Proj                                             = identity,
             indirect_strict_weak_order<projected<_Ip, _Proj>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  _Ip operator()(_Ip __first, _Sp __last, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr _Ip
+  operator()(_Ip __first, _Sp __last, _Comp __comp = {}, _Proj __proj = {}) const {
     auto __comp_lhs_rhs_swapped = [&](auto&& __lhs, auto&& __rhs) { return std::invoke(__comp, __rhs, __lhs); };
     return ranges::__min_element_impl(__first, __last, __comp_lhs_rhs_swapped, __proj);
   }
 
-  template <forward_range _Rp, class _Proj = identity,
+  template <forward_range _Rp,
+            class _Proj                                                         = identity,
             indirect_strict_weak_order<projected<iterator_t<_Rp>, _Proj>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_iterator_t<_Rp> operator()(_Rp&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Rp>
+  operator()(_Rp&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
     auto __comp_lhs_rhs_swapped = [&](auto&& __lhs, auto&& __rhs) { return std::invoke(__comp, __rhs, __lhs); };
     return ranges::__min_element_impl(ranges::begin(__r), ranges::end(__r), __comp_lhs_rhs_swapped, __proj);
   }
@@ -50,7 +53,7 @@ struct __fn {
 } // namespace __max_element
 
 inline namespace __cpo {
-  inline constexpr auto max_element = __max_element::__fn{};
+inline constexpr auto max_element = __max_element::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_merge.h b/libcxx/include/__algorithm/ranges_merge.h
index c5797b0e71bdd..7f49154ec9221 100644
--- a/libcxx/include/__algorithm/ranges_merge.h
+++ b/libcxx/include/__algorithm/ranges_merge.h
@@ -38,25 +38,25 @@ using merge_result = in_in_out_result<_InIter1, _InIter2, _OutIter>;
 
 namespace __merge {
 
-template <
-    class _InIter1,
-    class _Sent1,
-    class _InIter2,
-    class _Sent2,
-    class _OutIter,
-    class _Comp,
-    class _Proj1,
-    class _Proj2>
-_LIBCPP_HIDE_FROM_ABI constexpr merge_result<__remove_cvref_t<_InIter1>, __remove_cvref_t<_InIter2>, __remove_cvref_t<_OutIter>>
-__merge_impl(
-    _InIter1&& __first1,
-    _Sent1&& __last1,
-    _InIter2&& __first2,
-    _Sent2&& __last2,
-    _OutIter&& __result,
-    _Comp&& __comp,
-    _Proj1&& __proj1,
-    _Proj2&& __proj2) {
+template < class _InIter1,
+           class _Sent1,
+           class _InIter2,
+           class _Sent2,
+           class _OutIter,
+           class _Comp,
+           class _Proj1,
+           class _Proj2>
+_LIBCPP_HIDE_FROM_ABI constexpr merge_result<__remove_cvref_t<_InIter1>,
+                                             __remove_cvref_t<_InIter2>,
+                                             __remove_cvref_t<_OutIter>>
+__merge_impl(_InIter1&& __first1,
+             _Sent1&& __last1,
+             _InIter2&& __first2,
+             _Sent2&& __last2,
+             _OutIter&& __result,
+             _Comp&& __comp,
+             _Proj1&& __proj1,
+             _Proj2&& __proj2) {
   for (; __first1 != __last1 && __first2 != __last2; ++__result) {
     if (std::invoke(__comp, std::invoke(__proj2, *__first2), std::invoke(__proj1, *__first1))) {
       *__result = *__first2;
@@ -72,15 +72,14 @@ __merge_impl(
 }
 
 struct __fn {
-  template <
-      input_iterator _InIter1,
-      sentinel_for<_InIter1> _Sent1,
-      input_iterator _InIter2,
-      sentinel_for<_InIter2> _Sent2,
-      weakly_incrementable _OutIter,
-      class _Comp  = less,
-      class _Proj1 = identity,
-      class _Proj2 = identity>
+  template <input_iterator _InIter1,
+            sentinel_for<_InIter1> _Sent1,
+            input_iterator _InIter2,
+            sentinel_for<_InIter2> _Sent2,
+            weakly_incrementable _OutIter,
+            class _Comp  = less,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
     requires mergeable<_InIter1, _InIter2, _OutIter, _Comp, _Proj1, _Proj2>
   _LIBCPP_HIDE_FROM_ABI constexpr merge_result<_InIter1, _InIter2, _OutIter> operator()(
       _InIter1 __first1,
@@ -94,28 +93,20 @@ struct __fn {
     return __merge::__merge_impl(__first1, __last1, __first2, __last2, __result, __comp, __proj1, __proj2);
   }
 
-  template <
-      input_range _Range1,
-      input_range _Range2,
-      weakly_incrementable _OutIter,
-      class _Comp  = less,
-      class _Proj1 = identity,
-      class _Proj2 = identity>
-    requires mergeable<
-        iterator_t<_Range1>,
-        iterator_t<_Range2>,
-        _OutIter,
-        _Comp,
-        _Proj1,
-        _Proj2>
+  template <input_range _Range1,
+            input_range _Range2,
+            weakly_incrementable _OutIter,
+            class _Comp  = less,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires mergeable<iterator_t<_Range1>, iterator_t<_Range2>, _OutIter, _Comp, _Proj1, _Proj2>
   _LIBCPP_HIDE_FROM_ABI constexpr merge_result<borrowed_iterator_t<_Range1>, borrowed_iterator_t<_Range2>, _OutIter>
-        operator()(
-            _Range1&& __range1,
-            _Range2&& __range2,
-            _OutIter __result,
-            _Comp __comp   = {},
-            _Proj1 __proj1 = {},
-            _Proj2 __proj2 = {}) const {
+  operator()(_Range1&& __range1,
+             _Range2&& __range2,
+             _OutIter __result,
+             _Comp __comp   = {},
+             _Proj1 __proj1 = {},
+             _Proj2 __proj2 = {}) const {
     return __merge::__merge_impl(
         ranges::begin(__range1),
         ranges::end(__range1),
@@ -131,7 +122,7 @@ struct __fn {
 } // namespace __merge
 
 inline namespace __cpo {
-  inline constexpr auto merge = __merge::__fn{};
+inline constexpr auto merge = __merge::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_min.h b/libcxx/include/__algorithm/ranges_min.h
index 5e941a1db06a8..63161396d9097 100644
--- a/libcxx/include/__algorithm/ranges_min.h
+++ b/libcxx/include/__algorithm/ranges_min.h
@@ -30,38 +30,41 @@
 #if _LIBCPP_STD_VER >= 20
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 namespace ranges {
 namespace __min {
 struct __fn {
-  template <class _Tp, class _Proj = identity,
+  template <class _Tp,
+            class _Proj                                                    = identity,
             indirect_strict_weak_order<projected<const _Tp*, _Proj>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  const _Tp& operator()(_LIBCPP_LIFETIMEBOUND const _Tp& __a,
-                        _LIBCPP_LIFETIMEBOUND const _Tp& __b,
-                        _Comp __comp = {},
-                        _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr const _Tp&
+  operator()(_LIBCPP_LIFETIMEBOUND const _Tp& __a,
+             _LIBCPP_LIFETIMEBOUND const _Tp& __b,
+             _Comp __comp = {},
+             _Proj __proj = {}) const {
     return std::invoke(__comp, std::invoke(__proj, __b), std::invoke(__proj, __a)) ? __b : __a;
   }
 
-  template <copyable _Tp, class _Proj = identity,
+  template <copyable _Tp,
+            class _Proj                                                    = identity,
             indirect_strict_weak_order<projected<const _Tp*, _Proj>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  _Tp operator()(initializer_list<_Tp> __il, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr _Tp
+  operator()(initializer_list<_Tp> __il, _Comp __comp = {}, _Proj __proj = {}) const {
     _LIBCPP_ASSERT(__il.begin() != __il.end(), "initializer_list must contain at least one element");
     return *ranges::__min_element_impl(__il.begin(), __il.end(), __comp, __proj);
   }
 
-  template <input_range _Rp, class _Proj = identity,
+  template <input_range _Rp,
+            class _Proj                                                         = identity,
             indirect_strict_weak_order<projected<iterator_t<_Rp>, _Proj>> _Comp = ranges::less>
     requires indirectly_copyable_storable<iterator_t<_Rp>, range_value_t<_Rp>*>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  range_value_t<_Rp> operator()(_Rp&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr range_value_t<_Rp>
+  operator()(_Rp&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
     auto __first = ranges::begin(__r);
-    auto __last = ranges::end(__r);
+    auto __last  = ranges::end(__r);
     _LIBCPP_ASSERT(__first != __last, "range must contain at least one element");
     if constexpr (forward_range<_Rp> && !__is_cheap_to_copy<range_value_t<_Rp>>) {
       return *ranges::__min_element_impl(__first, __last, __comp, __proj);
@@ -78,7 +81,7 @@ struct __fn {
 } // namespace __min
 
 inline namespace __cpo {
-  inline constexpr auto min = __min::__fn{};
+inline constexpr auto min = __min::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_min_element.h b/libcxx/include/__algorithm/ranges_min_element.h
index 4c58adb3119f8..07826a0e6b817 100644
--- a/libcxx/include/__algorithm/ranges_min_element.h
+++ b/libcxx/include/__algorithm/ranges_min_element.h
@@ -32,8 +32,7 @@ namespace ranges {
 
 // TODO(ranges): `ranges::min_element` can now simply delegate to `std::__min_element`.
 template <class _Ip, class _Sp, class _Proj, class _Comp>
-_LIBCPP_HIDE_FROM_ABI constexpr
-_Ip __min_element_impl(_Ip __first, _Sp __last, _Comp& __comp, _Proj& __proj) {
+_LIBCPP_HIDE_FROM_ABI constexpr _Ip __min_element_impl(_Ip __first, _Sp __last, _Comp& __comp, _Proj& __proj) {
   if (__first == __last)
     return __first;
 
@@ -46,24 +45,27 @@ _Ip __min_element_impl(_Ip __first, _Sp __last, _Comp& __comp, _Proj& __proj) {
 
 namespace __min_element {
 struct __fn {
-  template <forward_iterator _Ip, sentinel_for<_Ip> _Sp, class _Proj = identity,
+  template <forward_iterator _Ip,
+            sentinel_for<_Ip> _Sp,
+            class _Proj                                             = identity,
             indirect_strict_weak_order<projected<_Ip, _Proj>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  _Ip operator()(_Ip __first, _Sp __last, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr _Ip
+  operator()(_Ip __first, _Sp __last, _Comp __comp = {}, _Proj __proj = {}) const {
     return ranges::__min_element_impl(__first, __last, __comp, __proj);
   }
 
-  template <forward_range _Rp, class _Proj = identity,
+  template <forward_range _Rp,
+            class _Proj                                                         = identity,
             indirect_strict_weak_order<projected<iterator_t<_Rp>, _Proj>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_iterator_t<_Rp> operator()(_Rp&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Rp>
+  operator()(_Rp&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
     return ranges::__min_element_impl(ranges::begin(__r), ranges::end(__r), __comp, __proj);
   }
 };
 } // namespace __min_element
 
 inline namespace __cpo {
-  inline constexpr auto min_element = __min_element::__fn{};
+inline constexpr auto min_element = __min_element::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_minmax.h b/libcxx/include/__algorithm/ranges_minmax.h
index 2a966bbee84fd..9a8bd9b10d636 100644
--- a/libcxx/include/__algorithm/ranges_minmax.h
+++ b/libcxx/include/__algorithm/ranges_minmax.h
@@ -37,7 +37,7 @@
 #if _LIBCPP_STD_VER >= 20
 
 _LIBCPP_PUSH_MACROS
-#include <__undef_macros>
+#  include <__undef_macros>
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -47,7 +47,8 @@ using minmax_result = min_max_result<_T1>;
 
 namespace __minmax {
 struct __fn {
-  template <class _Type, class _Proj = identity,
+  template <class _Type,
+            class _Proj                                                      = identity,
             indirect_strict_weak_order<projected<const _Type*, _Proj>> _Comp = ranges::less>
   _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr ranges::minmax_result<const _Type&>
   operator()(_LIBCPP_LIFETIMEBOUND const _Type& __a,
@@ -59,22 +60,24 @@ struct __fn {
     return {__a, __b};
   }
 
-  template <copyable _Type, class _Proj = identity,
+  template <copyable _Type,
+            class _Proj                                                      = identity,
             indirect_strict_weak_order<projected<const _Type*, _Proj>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  ranges::minmax_result<_Type> operator()(initializer_list<_Type> __il, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr ranges::minmax_result<_Type>
+  operator()(initializer_list<_Type> __il, _Comp __comp = {}, _Proj __proj = {}) const {
     _LIBCPP_ASSERT(__il.begin() != __il.end(), "initializer_list has to contain at least one element");
     auto __iters = std::__minmax_element_impl(__il.begin(), __il.end(), __comp, __proj);
-    return ranges::minmax_result<_Type> { *__iters.first, *__iters.second };
+    return ranges::minmax_result<_Type>{*__iters.first, *__iters.second};
   }
 
-  template <input_range _Range, class _Proj = identity,
+  template <input_range _Range,
+            class _Proj                                                            = identity,
             indirect_strict_weak_order<projected<iterator_t<_Range>, _Proj>> _Comp = ranges::less>
     requires indirectly_copyable_storable<iterator_t<_Range>, range_value_t<_Range>*>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  ranges::minmax_result<range_value_t<_Range>> operator()(_Range&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
-    auto __first = ranges::begin(__r);
-    auto __last = ranges::end(__r);
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr ranges::minmax_result<range_value_t<_Range>>
+  operator()(_Range&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
+    auto __first  = ranges::begin(__r);
+    auto __last   = ranges::end(__r);
     using _ValueT = range_value_t<_Range>;
 
     _LIBCPP_ASSERT(__first != __last, "range has to contain at least one element");
@@ -98,8 +101,9 @@ struct __fn {
       // input_iterators can't be copied, so the implementation for input_iterators has to store
       // the values instead of a pointer to the correct values
       auto __less = [&](auto&& __a, auto&& __b) -> bool {
-        return std::invoke(__comp, std::invoke(__proj, std::forward<decltype(__a)>(__a)),
-                                   std::invoke(__proj, std::forward<decltype(__b)>(__b)));
+        return std::invoke(__comp,
+                           std::invoke(__proj, std::forward<decltype(__a)>(__a)),
+                           std::invoke(__proj, std::forward<decltype(__b)>(__b)));
       };
 
       // During initialization, members are allowed to refer to already initialized members
@@ -142,7 +146,7 @@ struct __fn {
 } // namespace __minmax
 
 inline namespace __cpo {
-  inline constexpr auto minmax = __minmax::__fn{};
+inline constexpr auto minmax = __minmax::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_minmax_element.h b/libcxx/include/__algorithm/ranges_minmax_element.h
index cbf350557948f..a52319f6b5d3f 100644
--- a/libcxx/include/__algorithm/ranges_minmax_element.h
+++ b/libcxx/include/__algorithm/ranges_minmax_element.h
@@ -39,18 +39,20 @@ using minmax_element_result = min_max_result<_T1>;
 
 namespace __minmax_element {
 struct __fn {
-  template <forward_iterator _Ip, sentinel_for<_Ip> _Sp, class _Proj = identity,
+  template <forward_iterator _Ip,
+            sentinel_for<_Ip> _Sp,
+            class _Proj                                             = identity,
             indirect_strict_weak_order<projected<_Ip, _Proj>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  ranges::minmax_element_result<_Ip> operator()(_Ip __first, _Sp __last, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr ranges::minmax_element_result<_Ip>
+  operator()(_Ip __first, _Sp __last, _Comp __comp = {}, _Proj __proj = {}) const {
     auto __ret = std::__minmax_element_impl(std::move(__first), std::move(__last), __comp, __proj);
     return {__ret.first, __ret.second};
   }
 
-  template <forward_range _Rp, class _Proj = identity,
+  template <forward_range _Rp,
+            class _Proj                                                         = identity,
             indirect_strict_weak_order<projected<iterator_t<_Rp>, _Proj>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  ranges::minmax_element_result<borrowed_iterator_t<_Rp>>
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr ranges::minmax_element_result<borrowed_iterator_t<_Rp>>
   operator()(_Rp&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
     auto __ret = std::__minmax_element_impl(ranges::begin(__r), ranges::end(__r), __comp, __proj);
     return {__ret.first, __ret.second};
@@ -59,7 +61,7 @@ struct __fn {
 } // namespace __minmax_element
 
 inline namespace __cpo {
-  inline constexpr auto minmax_element = __minmax_element::__fn{};
+inline constexpr auto minmax_element = __minmax_element::__fn{};
 } // namespace __cpo
 
 } // namespace ranges
diff --git a/libcxx/include/__algorithm/ranges_mismatch.h b/libcxx/include/__algorithm/ranges_mismatch.h
index 098c4151cf38e..db9bfc8e87db6 100644
--- a/libcxx/include/__algorithm/ranges_mismatch.h
+++ b/libcxx/include/__algorithm/ranges_mismatch.h
@@ -36,12 +36,9 @@ using mismatch_result = in_in_result<_I1, _I2>;
 
 namespace __mismatch {
 struct __fn {
-  template <class _I1, class _S1, class _I2, class _S2,
-            class _Pred, class _Proj1, class _Proj2>
-  static _LIBCPP_HIDE_FROM_ABI constexpr
-  mismatch_result<_I1, _I2>
-  __go(_I1 __first1, _S1 __last1, _I2 __first2, _S2 __last2,
-       _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) {
+  template <class _I1, class _S1, class _I2, class _S2, class _Pred, class _Proj1, class _Proj2>
+  static _LIBCPP_HIDE_FROM_ABI constexpr mismatch_result<_I1, _I2>
+  __go(_I1 __first1, _S1 __last1, _I2 __first2, _S2 __last2, _Pred& __pred, _Proj1& __proj1, _Proj2& __proj2) {
     while (__first1 != __last1 && __first2 != __last2) {
       if (!std::invoke(__pred, std::invoke(__proj1, *__first1), std::invoke(__proj2, *__first2)))
         break;
@@ -51,30 +48,37 @@ struct __fn {
     return {std::move(__first1), std::move(__first2)};
   }
 
-  template <input_iterator _I1, sentinel_for<_I1> _S1,
-            input_iterator _I2, sentinel_for<_I2> _S2,
-            class _Pred = ranges::equal_to, class _Proj1 = identity, class _Proj2 = identity>
+  template <input_iterator _I1,
+            sentinel_for<_I1> _S1,
+            input_iterator _I2,
+            sentinel_for<_I2> _S2,
+            class _Pred  = ranges::equal_to,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
     requires indirectly_comparable<_I1, _I2, _Pred, _Proj1, _Proj2>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  mismatch_result<_I1, _I2> operator()(_I1 __first1, _S1 __last1, _I2 __first2, _S2 __last2,
-                                       _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr mismatch_result<_I1, _I2> operator()(
+      _I1 __first1, _S1 __last1, _I2 __first2, _S2 __last2, _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {})
+      const {
     return __go(std::move(__first1), __last1, std::move(__first2), __last2, __pred, __proj1, __proj2);
   }
 
-  template <input_range _R1, input_range _R2,
-            class _Pred = ranges::equal_to, class _Proj1 = identity, class _Proj2 = identity>
+  template <input_range _R1,
+            input_range _R2,
+            class _Pred  = ranges::equal_to,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
     requires indirectly_comparable<iterator_t<_R1>, iterator_t<_R2>, _Pred, _Proj1, _Proj2>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  mismatch_result<borrowed_iterator_t<_R1>, borrowed_iterator_t<_R2>>
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr mismatch_result<borrowed_iterator_t<_R1>,
+                                                                        borrowed_iterator_t<_R2>>
   operator()(_R1&& __r1, _R2&& __r2, _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
-    return __go(ranges::begin(__r1), ranges::end(__r1), ranges::begin(__r2), ranges::end(__r2),
-                __pred, __proj1, __proj2);
+    return __go(
+        ranges::begin(__r1), ranges::end(__r1), ranges::begin(__r2), ranges::end(__r2), __pred, __proj1, __proj2);
   }
 };
 } // namespace __mismatch
 
 inline namespace __cpo {
-  constexpr inline auto mismatch = __mismatch::__fn{};
+constexpr inline auto mismatch = __mismatch::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_move.h b/libcxx/include/__algorithm/ranges_move.h
index 5e06e903f7295..8bd2409f891c0 100644
--- a/libcxx/include/__algorithm/ranges_move.h
+++ b/libcxx/include/__algorithm/ranges_move.h
@@ -34,33 +34,31 @@ using move_result = in_out_result<_InIter, _OutIter>;
 
 namespace __move {
 struct __fn {
-
   template <class _InIter, class _Sent, class _OutIter>
-  _LIBCPP_HIDE_FROM_ABI constexpr static
-  move_result<_InIter, _OutIter> __move_impl(_InIter __first, _Sent __last, _OutIter __result) {
+  _LIBCPP_HIDE_FROM_ABI constexpr static move_result<_InIter, _OutIter>
+  __move_impl(_InIter __first, _Sent __last, _OutIter __result) {
     auto __ret = std::__move<_RangeAlgPolicy>(std::move(__first), std::move(__last), std::move(__result));
     return {std::move(__ret.first), std::move(__ret.second)};
   }
 
   template <input_iterator _InIter, sentinel_for<_InIter> _Sent, weakly_incrementable _OutIter>
     requires indirectly_movable<_InIter, _OutIter>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  move_result<_InIter, _OutIter> operator()(_InIter __first, _Sent __last, _OutIter __result) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr move_result<_InIter, _OutIter>
+  operator()(_InIter __first, _Sent __last, _OutIter __result) const {
     return __move_impl(std::move(__first), std::move(__last), std::move(__result));
   }
 
   template <input_range _Range, weakly_incrementable _OutIter>
     requires indirectly_movable<iterator_t<_Range>, _OutIter>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  move_result<borrowed_iterator_t<_Range>, _OutIter> operator()(_Range&& __range, _OutIter __result) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr move_result<borrowed_iterator_t<_Range>, _OutIter>
+  operator()(_Range&& __range, _OutIter __result) const {
     return __move_impl(ranges::begin(__range), ranges::end(__range), std::move(__result));
   }
-
 };
 } // namespace __move
 
 inline namespace __cpo {
-  inline constexpr auto move = __move::__fn{};
+inline constexpr auto move = __move::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_move_backward.h b/libcxx/include/__algorithm/ranges_move_backward.h
index 95c2c6612ecd1..ee390a40e489a 100644
--- a/libcxx/include/__algorithm/ranges_move_backward.h
+++ b/libcxx/include/__algorithm/ranges_move_backward.h
@@ -36,33 +36,31 @@ using move_backward_result = in_out_result<_InIter, _OutIter>;
 
 namespace __move_backward {
 struct __fn {
-
   template <class _InIter, class _Sent, class _OutIter>
-  _LIBCPP_HIDE_FROM_ABI constexpr static
-  move_backward_result<_InIter, _OutIter> __move_backward_impl(_InIter __first, _Sent __last, _OutIter __result) {
+  _LIBCPP_HIDE_FROM_ABI constexpr static move_backward_result<_InIter, _OutIter>
+  __move_backward_impl(_InIter __first, _Sent __last, _OutIter __result) {
     auto __ret = std::__move_backward<_RangeAlgPolicy>(std::move(__first), std::move(__last), std::move(__result));
     return {std::move(__ret.first), std::move(__ret.second)};
   }
 
   template <bidirectional_iterator _InIter, sentinel_for<_InIter> _Sent, bidirectional_iterator _OutIter>
     requires indirectly_movable<_InIter, _OutIter>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  move_backward_result<_InIter, _OutIter> operator()(_InIter __first, _Sent __last, _OutIter __result) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr move_backward_result<_InIter, _OutIter>
+  operator()(_InIter __first, _Sent __last, _OutIter __result) const {
     return __move_backward_impl(std::move(__first), std::move(__last), std::move(__result));
   }
 
   template <bidirectional_range _Range, bidirectional_iterator _Iter>
     requires indirectly_movable<iterator_t<_Range>, _Iter>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  move_backward_result<borrowed_iterator_t<_Range>, _Iter> operator()(_Range&& __range, _Iter __result) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr move_backward_result<borrowed_iterator_t<_Range>, _Iter>
+  operator()(_Range&& __range, _Iter __result) const {
     return __move_backward_impl(ranges::begin(__range), ranges::end(__range), std::move(__result));
   }
-
 };
 } // namespace __move_backward
 
 inline namespace __cpo {
-  inline constexpr auto move_backward = __move_backward::__fn{};
+inline constexpr auto move_backward = __move_backward::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_none_of.h b/libcxx/include/__algorithm/ranges_none_of.h
index 39940adbb9a65..b0d363895e000 100644
--- a/libcxx/include/__algorithm/ranges_none_of.h
+++ b/libcxx/include/__algorithm/ranges_none_of.h
@@ -29,10 +29,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 namespace __none_of {
 struct __fn {
-
   template <class _Iter, class _Sent, class _Proj, class _Pred>
-  _LIBCPP_HIDE_FROM_ABI constexpr static
-  bool __none_of_impl(_Iter __first, _Sent __last, _Pred& __pred, _Proj& __proj) {
+  _LIBCPP_HIDE_FROM_ABI constexpr static bool
+  __none_of_impl(_Iter __first, _Sent __last, _Pred& __pred, _Proj& __proj) {
     for (; __first != __last; ++__first) {
       if (std::invoke(__pred, std::invoke(__proj, *__first)))
         return false;
@@ -40,24 +39,27 @@ struct __fn {
     return true;
   }
 
-  template <input_iterator _Iter, sentinel_for<_Iter> _Sent, class _Proj = identity,
+  template <input_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj = identity,
             indirect_unary_predicate<projected<_Iter, _Proj>> _Pred>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  bool operator()(_Iter __first, _Sent __last, _Pred __pred = {}, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Iter __first, _Sent __last, _Pred __pred = {}, _Proj __proj = {}) const {
     return __none_of_impl(std::move(__first), std::move(__last), __pred, __proj);
   }
 
-  template <input_range _Range, class _Proj = identity,
+  template <input_range _Range,
+            class _Proj = identity,
             indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  bool operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr bool
+  operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) const {
     return __none_of_impl(ranges::begin(__range), ranges::end(__range), __pred, __proj);
   }
 };
 } // namespace __none_of
 
 inline namespace __cpo {
-  inline constexpr auto none_of = __none_of::__fn{};
+inline constexpr auto none_of = __none_of::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_nth_element.h b/libcxx/include/__algorithm/ranges_nth_element.h
index 96bf33b079674..7abdbd0889e0c 100644
--- a/libcxx/include/__algorithm/ranges_nth_element.h
+++ b/libcxx/include/__algorithm/ranges_nth_element.h
@@ -40,8 +40,8 @@ namespace __nth_element {
 
 struct __fn {
   template <class _Iter, class _Sent, class _Comp, class _Proj>
-  _LIBCPP_HIDE_FROM_ABI constexpr static
-  _Iter __nth_element_fn_impl(_Iter __first, _Iter __nth, _Sent __last, _Comp& __comp, _Proj& __proj) {
+  _LIBCPP_HIDE_FROM_ABI constexpr static _Iter
+  __nth_element_fn_impl(_Iter __first, _Iter __nth, _Sent __last, _Comp& __comp, _Proj& __proj) {
     auto __last_iter = ranges::next(__first, __last);
 
     auto&& __projected_comp = std::__make_projected(__comp, __proj);
@@ -52,16 +52,15 @@ struct __fn {
 
   template <random_access_iterator _Iter, sentinel_for<_Iter> _Sent, class _Comp = ranges::less, class _Proj = identity>
     requires sortable<_Iter, _Comp, _Proj>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  _Iter operator()(_Iter __first, _Iter __nth, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, _Iter __nth, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
     return __nth_element_fn_impl(std::move(__first), std::move(__nth), std::move(__last), __comp, __proj);
   }
 
   template <random_access_range _Range, class _Comp = ranges::less, class _Proj = identity>
     requires sortable<iterator_t<_Range>, _Comp, _Proj>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_iterator_t<_Range> operator()(_Range&& __r, iterator_t<_Range> __nth, _Comp __comp = {},
-                                         _Proj __proj = {}) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __r, iterator_t<_Range> __nth, _Comp __comp = {}, _Proj __proj = {}) const {
     return __nth_element_fn_impl(ranges::begin(__r), std::move(__nth), ranges::end(__r), __comp, __proj);
   }
 };
@@ -69,7 +68,7 @@ struct __fn {
 } // namespace __nth_element
 
 inline namespace __cpo {
-  inline constexpr auto nth_element = __nth_element::__fn{};
+inline constexpr auto nth_element = __nth_element::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_partial_sort.h b/libcxx/include/__algorithm/ranges_partial_sort.h
index 4562c35443e69..9ec8882097d78 100644
--- a/libcxx/include/__algorithm/ranges_partial_sort.h
+++ b/libcxx/include/__algorithm/ranges_partial_sort.h
@@ -42,24 +42,23 @@ namespace __partial_sort {
 
 struct __fn {
   template <class _Iter, class _Sent, class _Comp, class _Proj>
-  _LIBCPP_HIDE_FROM_ABI constexpr static
-  _Iter __partial_sort_fn_impl(_Iter __first, _Iter __middle, _Sent __last, _Comp& __comp, _Proj& __proj) {
+  _LIBCPP_HIDE_FROM_ABI constexpr static _Iter
+  __partial_sort_fn_impl(_Iter __first, _Iter __middle, _Sent __last, _Comp& __comp, _Proj& __proj) {
     auto&& __projected_comp = std::__make_projected(__comp, __proj);
     return std::__partial_sort<_RangeAlgPolicy>(std::move(__first), std::move(__middle), __last, __projected_comp);
   }
 
   template <random_access_iterator _Iter, sentinel_for<_Iter> _Sent, class _Comp = ranges::less, class _Proj = identity>
     requires sortable<_Iter, _Comp, _Proj>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  _Iter operator()(_Iter __first, _Iter __middle, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, _Iter __middle, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
     return __partial_sort_fn_impl(std::move(__first), std::move(__middle), std::move(__last), __comp, __proj);
   }
 
   template <random_access_range _Range, class _Comp = ranges::less, class _Proj = identity>
     requires sortable<iterator_t<_Range>, _Comp, _Proj>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_iterator_t<_Range> operator()(_Range&& __r, iterator_t<_Range> __middle, _Comp __comp = {},
-                                         _Proj __proj = {}) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __r, iterator_t<_Range> __middle, _Comp __comp = {}, _Proj __proj = {}) const {
     return __partial_sort_fn_impl(ranges::begin(__r), std::move(__middle), ranges::end(__r), __comp, __proj);
   }
 };
@@ -67,7 +66,7 @@ struct __fn {
 } // namespace __partial_sort
 
 inline namespace __cpo {
-  inline constexpr auto partial_sort = __partial_sort::__fn{};
+inline constexpr auto partial_sort = __partial_sort::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_partial_sort_copy.h b/libcxx/include/__algorithm/ranges_partial_sort_copy.h
index 5401a374ec28a..eba7d9ac41657 100644
--- a/libcxx/include/__algorithm/ranges_partial_sort_copy.h
+++ b/libcxx/include/__algorithm/ranges_partial_sort_copy.h
@@ -42,46 +42,63 @@ using partial_sort_copy_result = in_out_result<_InIter, _OutIter>;
 namespace __partial_sort_copy {
 
 struct __fn {
-
-  template <input_iterator _Iter1, sentinel_for<_Iter1> _Sent1,
-            random_access_iterator _Iter2, sentinel_for<_Iter2> _Sent2,
-            class _Comp = ranges::less, class _Proj1 = identity, class _Proj2 = identity>
-  requires indirectly_copyable<_Iter1, _Iter2> && sortable<_Iter2, _Comp, _Proj2> &&
-           indirect_strict_weak_order<_Comp, projected<_Iter1, _Proj1>, projected<_Iter2, _Proj2>>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  partial_sort_copy_result<_Iter1, _Iter2>
-  operator()(_Iter1 __first, _Sent1 __last, _Iter2 __result_first, _Sent2 __result_last,
-             _Comp __comp = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
+  template <input_iterator _Iter1,
+            sentinel_for<_Iter1> _Sent1,
+            random_access_iterator _Iter2,
+            sentinel_for<_Iter2> _Sent2,
+            class _Comp  = ranges::less,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires indirectly_copyable<_Iter1, _Iter2> && sortable<_Iter2, _Comp, _Proj2> &&
+             indirect_strict_weak_order<_Comp, projected<_Iter1, _Proj1>, projected<_Iter2, _Proj2>>
+  _LIBCPP_HIDE_FROM_ABI constexpr partial_sort_copy_result<_Iter1, _Iter2> operator()(
+      _Iter1 __first,
+      _Sent1 __last,
+      _Iter2 __result_first,
+      _Sent2 __result_last,
+      _Comp __comp   = {},
+      _Proj1 __proj1 = {},
+      _Proj2 __proj2 = {}) const {
     auto __result = std::__partial_sort_copy<_RangeAlgPolicy>(
-        std::move(__first), std::move(__last), std::move(__result_first), std::move(__result_last),
-        __comp, __proj1, __proj2
-    );
+        std::move(__first),
+        std::move(__last),
+        std::move(__result_first),
+        std::move(__result_last),
+        __comp,
+        __proj1,
+        __proj2);
     return {std::move(__result.first), std::move(__result.second)};
   }
 
-  template <input_range _Range1, random_access_range _Range2, class _Comp = ranges::less,
-            class _Proj1 = identity, class _Proj2 = identity>
-  requires indirectly_copyable<iterator_t<_Range1>, iterator_t<_Range2>> &&
-           sortable<iterator_t<_Range2>, _Comp, _Proj2> &&
-           indirect_strict_weak_order<_Comp, projected<iterator_t<_Range1>, _Proj1>,
-                                      projected<iterator_t<_Range2>, _Proj2>>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  partial_sort_copy_result<borrowed_iterator_t<_Range1>, borrowed_iterator_t<_Range2>>
-  operator()(_Range1&& __range, _Range2&& __result_range, _Comp __comp = {},
-             _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
+  template <input_range _Range1,
+            random_access_range _Range2,
+            class _Comp  = ranges::less,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires indirectly_copyable<iterator_t<_Range1>, iterator_t<_Range2>> &&
+             sortable<iterator_t<_Range2>, _Comp, _Proj2> &&
+             indirect_strict_weak_order<_Comp,
+                                        projected<iterator_t<_Range1>, _Proj1>,
+                                        projected<iterator_t<_Range2>, _Proj2>>
+  _LIBCPP_HIDE_FROM_ABI constexpr partial_sort_copy_result<borrowed_iterator_t<_Range1>, borrowed_iterator_t<_Range2>>
+  operator()(
+      _Range1&& __range, _Range2&& __result_range, _Comp __comp = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
     auto __result = std::__partial_sort_copy<_RangeAlgPolicy>(
-        ranges::begin(__range), ranges::end(__range), ranges::begin(__result_range), ranges::end(__result_range),
-        __comp, __proj1, __proj2
-    );
+        ranges::begin(__range),
+        ranges::end(__range),
+        ranges::begin(__result_range),
+        ranges::end(__result_range),
+        __comp,
+        __proj1,
+        __proj2);
     return {std::move(__result.first), std::move(__result.second)};
   }
-
 };
 
 } // namespace __partial_sort_copy
 
 inline namespace __cpo {
-  inline constexpr auto partial_sort_copy = __partial_sort_copy::__fn{};
+inline constexpr auto partial_sort_copy = __partial_sort_copy::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_partition.h b/libcxx/include/__algorithm/ranges_partition.h
index de839de12bfcc..89d192b51fd32 100644
--- a/libcxx/include/__algorithm/ranges_partition.h
+++ b/libcxx/include/__algorithm/ranges_partition.h
@@ -40,38 +40,39 @@ namespace ranges {
 namespace __partition {
 
 struct __fn {
-
   template <class _Iter, class _Sent, class _Proj, class _Pred>
-  _LIBCPP_HIDE_FROM_ABI static constexpr
-  subrange<__remove_cvref_t<_Iter>> __partition_fn_impl(_Iter&& __first, _Sent&& __last, _Pred&& __pred, _Proj&& __proj) {
+  _LIBCPP_HIDE_FROM_ABI static constexpr subrange<__remove_cvref_t<_Iter>>
+  __partition_fn_impl(_Iter&& __first, _Sent&& __last, _Pred&& __pred, _Proj&& __proj) {
     auto&& __projected_pred = std::__make_projected(__pred, __proj);
-    auto __result = std::__partition<_RangeAlgPolicy>(
+    auto __result           = std::__partition<_RangeAlgPolicy>(
         std::move(__first), std::move(__last), __projected_pred, __iterator_concept<_Iter>());
 
     return {std::move(__result.first), std::move(__result.second)};
   }
 
-  template <permutable _Iter, sentinel_for<_Iter> _Sent, class _Proj = identity,
+  template <permutable _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj = identity,
             indirect_unary_predicate<projected<_Iter, _Proj>> _Pred>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  subrange<_Iter> operator()(_Iter __first, _Sent __last, _Pred __pred, _Proj __proj = {}) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr subrange<_Iter>
+  operator()(_Iter __first, _Sent __last, _Pred __pred, _Proj __proj = {}) const {
     return __partition_fn_impl(__first, __last, __pred, __proj);
   }
 
-  template <forward_range _Range, class _Proj = identity,
+  template <forward_range _Range,
+            class _Proj = identity,
             indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
-  requires permutable<iterator_t<_Range>>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_subrange_t<_Range> operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) const {
+    requires permutable<iterator_t<_Range>>
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_subrange_t<_Range>
+  operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) const {
     return __partition_fn_impl(ranges::begin(__range), ranges::end(__range), __pred, __proj);
   }
-
 };
 
 } // namespace __partition
 
 inline namespace __cpo {
-  inline constexpr auto partition = __partition::__fn{};
+inline constexpr auto partition = __partition::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_partition_copy.h b/libcxx/include/__algorithm/ranges_partition_copy.h
index e398387bf0699..6a16b02db3e55 100644
--- a/libcxx/include/__algorithm/ranges_partition_copy.h
+++ b/libcxx/include/__algorithm/ranges_partition_copy.h
@@ -38,14 +38,18 @@ using partition_copy_result = in_out_out_result<_InIter, _OutIter1, _OutIter2>;
 namespace __partition_copy {
 
 struct __fn {
-
   // TODO(ranges): delegate to the classic algorithm.
   template <class _InIter, class _Sent, class _OutIter1, class _OutIter2, class _Proj, class _Pred>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  static partition_copy_result<
-      __remove_cvref_t<_InIter>, __remove_cvref_t<_OutIter1>, __remove_cvref_t<_OutIter2>
-  > __partition_copy_fn_impl( _InIter&& __first, _Sent&& __last, _OutIter1&& __out_true, _OutIter2&& __out_false,
-      _Pred& __pred, _Proj& __proj) {
+  _LIBCPP_HIDE_FROM_ABI constexpr static partition_copy_result<__remove_cvref_t<_InIter>,
+                                                               __remove_cvref_t<_OutIter1>,
+                                                               __remove_cvref_t<_OutIter2> >
+  __partition_copy_fn_impl(
+      _InIter&& __first,
+      _Sent&& __last,
+      _OutIter1&& __out_true,
+      _OutIter2&& __out_false,
+      _Pred& __pred,
+      _Proj& __proj) {
     for (; __first != __last; ++__first) {
       if (std::invoke(__pred, std::invoke(__proj, *__first))) {
         *__out_true = *__first;
@@ -60,34 +64,37 @@ struct __fn {
     return {std::move(__first), std::move(__out_true), std::move(__out_false)};
   }
 
-  template <input_iterator _InIter, sentinel_for<_InIter> _Sent,
-            weakly_incrementable _OutIter1, weakly_incrementable _OutIter2,
-            class _Proj = identity, indirect_unary_predicate<projected<_InIter, _Proj>> _Pred>
-  requires indirectly_copyable<_InIter, _OutIter1> && indirectly_copyable<_InIter, _OutIter2>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  partition_copy_result<_InIter, _OutIter1, _OutIter2>
-  operator()(_InIter __first, _Sent __last, _OutIter1 __out_true, _OutIter2 __out_false,
-             _Pred __pred, _Proj __proj = {}) const {
+  template <input_iterator _InIter,
+            sentinel_for<_InIter> _Sent,
+            weakly_incrementable _OutIter1,
+            weakly_incrementable _OutIter2,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<_InIter, _Proj>> _Pred>
+    requires indirectly_copyable<_InIter, _OutIter1> && indirectly_copyable<_InIter, _OutIter2>
+  _LIBCPP_HIDE_FROM_ABI constexpr partition_copy_result<_InIter, _OutIter1, _OutIter2> operator()(
+      _InIter __first, _Sent __last, _OutIter1 __out_true, _OutIter2 __out_false, _Pred __pred, _Proj __proj = {})
+      const {
     return __partition_copy_fn_impl(
         std::move(__first), std::move(__last), std::move(__out_true), std::move(__out_false), __pred, __proj);
   }
 
-  template <input_range _Range, weakly_incrementable _OutIter1, weakly_incrementable _OutIter2,
-            class _Proj = identity, indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
-  requires indirectly_copyable<iterator_t<_Range>, _OutIter1> && indirectly_copyable<iterator_t<_Range>, _OutIter2>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  partition_copy_result<borrowed_iterator_t<_Range>, _OutIter1, _OutIter2>
+  template <input_range _Range,
+            weakly_incrementable _OutIter1,
+            weakly_incrementable _OutIter2,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
+    requires indirectly_copyable<iterator_t<_Range>, _OutIter1> && indirectly_copyable<iterator_t<_Range>, _OutIter2>
+  _LIBCPP_HIDE_FROM_ABI constexpr partition_copy_result<borrowed_iterator_t<_Range>, _OutIter1, _OutIter2>
   operator()(_Range&& __range, _OutIter1 __out_true, _OutIter2 __out_false, _Pred __pred, _Proj __proj = {}) const {
     return __partition_copy_fn_impl(
         ranges::begin(__range), ranges::end(__range), std::move(__out_true), std::move(__out_false), __pred, __proj);
   }
-
 };
 
 } // namespace __partition_copy
 
 inline namespace __cpo {
-  inline constexpr auto partition_copy = __partition_copy::__fn{};
+inline constexpr auto partition_copy = __partition_copy::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_partition_point.h b/libcxx/include/__algorithm/ranges_partition_point.h
index 129ebb6c68da4..6fc20e7d00e9f 100644
--- a/libcxx/include/__algorithm/ranges_partition_point.h
+++ b/libcxx/include/__algorithm/ranges_partition_point.h
@@ -35,16 +35,15 @@ namespace ranges {
 namespace __partition_point {
 
 struct __fn {
-
   // TODO(ranges): delegate to the classic algorithm.
   template <class _Iter, class _Sent, class _Proj, class _Pred>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  static _Iter __partition_point_fn_impl(_Iter&& __first, _Sent&& __last, _Pred& __pred, _Proj& __proj) {
+  _LIBCPP_HIDE_FROM_ABI constexpr static _Iter
+  __partition_point_fn_impl(_Iter&& __first, _Sent&& __last, _Pred& __pred, _Proj& __proj) {
     auto __len = ranges::distance(__first, __last);
 
     while (__len != 0) {
       auto __half_len = std::__half_positive(__len);
-      auto __mid = ranges::next(__first, __half_len);
+      auto __mid      = ranges::next(__first, __half_len);
 
       if (std::invoke(__pred, std::invoke(__proj, *__mid))) {
         __first = ++__mid;
@@ -58,26 +57,27 @@ struct __fn {
     return __first;
   }
 
-  template <forward_iterator _Iter, sentinel_for<_Iter> _Sent, class _Proj = identity,
+  template <forward_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj = identity,
             indirect_unary_predicate<projected<_Iter, _Proj>> _Pred>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  _Iter operator()(_Iter __first, _Sent __last, _Pred __pred, _Proj __proj = {}) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter operator()(_Iter __first, _Sent __last, _Pred __pred, _Proj __proj = {}) const {
     return __partition_point_fn_impl(std::move(__first), std::move(__last), __pred, __proj);
   }
 
-  template <forward_range _Range, class _Proj = identity,
+  template <forward_range _Range,
+            class _Proj = identity,
             indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_iterator_t<_Range> operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) const {
     return __partition_point_fn_impl(ranges::begin(__range), ranges::end(__range), __pred, __proj);
   }
-
 };
 
 } // namespace __partition_point
 
 inline namespace __cpo {
-  inline constexpr auto partition_point = __partition_point::__fn{};
+inline constexpr auto partition_point = __partition_point::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_pop_heap.h b/libcxx/include/__algorithm/ranges_pop_heap.h
index 54ea97db7a655..364cfe94b161e 100644
--- a/libcxx/include/__algorithm/ranges_pop_heap.h
+++ b/libcxx/include/__algorithm/ranges_pop_heap.h
@@ -41,10 +41,10 @@ namespace __pop_heap {
 
 struct __fn {
   template <class _Iter, class _Sent, class _Comp, class _Proj>
-  _LIBCPP_HIDE_FROM_ABI constexpr static
-  _Iter __pop_heap_fn_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
+  _LIBCPP_HIDE_FROM_ABI constexpr static _Iter
+  __pop_heap_fn_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
     auto __last_iter = ranges::next(__first, __last);
-    auto __len = __last_iter - __first;
+    auto __len       = __last_iter - __first;
 
     auto&& __projected_comp = std::__make_projected(__comp, __proj);
     std::__pop_heap<_RangeAlgPolicy>(std::move(__first), __last_iter, __projected_comp, __len);
@@ -54,15 +54,15 @@ struct __fn {
 
   template <random_access_iterator _Iter, sentinel_for<_Iter> _Sent, class _Comp = ranges::less, class _Proj = identity>
     requires sortable<_Iter, _Comp, _Proj>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  _Iter operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
     return __pop_heap_fn_impl(std::move(__first), std::move(__last), __comp, __proj);
   }
 
   template <random_access_range _Range, class _Comp = ranges::less, class _Proj = identity>
     requires sortable<iterator_t<_Range>, _Comp, _Proj>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_iterator_t<_Range> operator()(_Range&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
     return __pop_heap_fn_impl(ranges::begin(__r), ranges::end(__r), __comp, __proj);
   }
 };
@@ -70,7 +70,7 @@ struct __fn {
 } // namespace __pop_heap
 
 inline namespace __cpo {
-  inline constexpr auto pop_heap = __pop_heap::__fn{};
+inline constexpr auto pop_heap = __pop_heap::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_prev_permutation.h b/libcxx/include/__algorithm/ranges_prev_permutation.h
index bf062874a31c4..ae7a68cce5fdc 100644
--- a/libcxx/include/__algorithm/ranges_prev_permutation.h
+++ b/libcxx/include/__algorithm/ranges_prev_permutation.h
@@ -40,9 +40,7 @@ using prev_permutation_result = in_found_result<_InIter>;
 namespace __prev_permutation {
 
 struct __fn {
-
-  template <bidirectional_iterator _Iter, sentinel_for<_Iter> _Sent,
-            class _Comp = ranges::less, class _Proj = identity>
+  template <bidirectional_iterator _Iter, sentinel_for<_Iter> _Sent, class _Comp = ranges::less, class _Proj = identity>
     requires sortable<_Iter, _Comp, _Proj>
   _LIBCPP_HIDE_FROM_ABI constexpr prev_permutation_result<_Iter>
   operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
@@ -51,8 +49,7 @@ struct __fn {
     return {std::move(__result.first), std::move(__result.second)};
   }
 
-  template <bidirectional_range _Range,
-            class _Comp = ranges::less, class _Proj = identity>
+  template <bidirectional_range _Range, class _Comp = ranges::less, class _Proj = identity>
     requires sortable<iterator_t<_Range>, _Comp, _Proj>
   _LIBCPP_HIDE_FROM_ABI constexpr prev_permutation_result<borrowed_iterator_t<_Range>>
   operator()(_Range&& __range, _Comp __comp = {}, _Proj __proj = {}) const {
@@ -60,7 +57,6 @@ struct __fn {
         ranges::begin(__range), ranges::end(__range), std::__make_projected(__comp, __proj));
     return {std::move(__result.first), std::move(__result.second)};
   }
-
 };
 
 } // namespace __prev_permutation
diff --git a/libcxx/include/__algorithm/ranges_push_heap.h b/libcxx/include/__algorithm/ranges_push_heap.h
index 6e65d11d01554..1ed9c953f54c3 100644
--- a/libcxx/include/__algorithm/ranges_push_heap.h
+++ b/libcxx/include/__algorithm/ranges_push_heap.h
@@ -41,8 +41,8 @@ namespace __push_heap {
 
 struct __fn {
   template <class _Iter, class _Sent, class _Comp, class _Proj>
-  _LIBCPP_HIDE_FROM_ABI constexpr static
-  _Iter __push_heap_fn_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
+  _LIBCPP_HIDE_FROM_ABI constexpr static _Iter
+  __push_heap_fn_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
     auto __last_iter = ranges::next(__first, __last);
 
     auto&& __projected_comp = std::__make_projected(__comp, __proj);
@@ -53,15 +53,15 @@ struct __fn {
 
   template <random_access_iterator _Iter, sentinel_for<_Iter> _Sent, class _Comp = ranges::less, class _Proj = identity>
     requires sortable<_Iter, _Comp, _Proj>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  _Iter operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
     return __push_heap_fn_impl(std::move(__first), std::move(__last), __comp, __proj);
   }
 
   template <random_access_range _Range, class _Comp = ranges::less, class _Proj = identity>
     requires sortable<iterator_t<_Range>, _Comp, _Proj>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_iterator_t<_Range> operator()(_Range&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
     return __push_heap_fn_impl(ranges::begin(__r), ranges::end(__r), __comp, __proj);
   }
 };
@@ -69,7 +69,7 @@ struct __fn {
 } // namespace __push_heap
 
 inline namespace __cpo {
-  inline constexpr auto push_heap = __push_heap::__fn{};
+inline constexpr auto push_heap = __push_heap::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_remove.h b/libcxx/include/__algorithm/ranges_remove.h
index 6dd48e0876130..bf0928df599ce 100644
--- a/libcxx/include/__algorithm/ranges_remove.h
+++ b/libcxx/include/__algorithm/ranges_remove.h
@@ -32,20 +32,19 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 namespace __remove {
 struct __fn {
-
   template <permutable _Iter, sentinel_for<_Iter> _Sent, class _Type, class _Proj = identity>
     requires indirect_binary_predicate<ranges::equal_to, projected<_Iter, _Proj>, const _Type*>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  subrange<_Iter> operator()(_Iter __first, _Sent __last, const _Type& __value, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr subrange<_Iter>
+  operator()(_Iter __first, _Sent __last, const _Type& __value, _Proj __proj = {}) const {
     auto __pred = [&](auto&& __other) { return __value == __other; };
     return ranges::__remove_if_impl(std::move(__first), std::move(__last), __pred, __proj);
   }
 
   template <forward_range _Range, class _Type, class _Proj = identity>
-    requires permutable<iterator_t<_Range>>
-          && indirect_binary_predicate<ranges::equal_to, projected<iterator_t<_Range>, _Proj>, const _Type*>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_subrange_t<_Range> operator()(_Range&& __range, const _Type& __value, _Proj __proj = {}) const {
+    requires permutable<iterator_t<_Range>> &&
+             indirect_binary_predicate<ranges::equal_to, projected<iterator_t<_Range>, _Proj>, const _Type*>
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr borrowed_subrange_t<_Range>
+  operator()(_Range&& __range, const _Type& __value, _Proj __proj = {}) const {
     auto __pred = [&](auto&& __other) { return __value == __other; };
     return ranges::__remove_if_impl(ranges::begin(__range), ranges::end(__range), __pred, __proj);
   }
@@ -53,7 +52,7 @@ struct __fn {
 } // namespace __remove
 
 inline namespace __cpo {
-  inline constexpr auto remove = __remove::__fn{};
+inline constexpr auto remove = __remove::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_remove_copy.h b/libcxx/include/__algorithm/ranges_remove_copy.h
index a9a19c17f6312..457d593957adb 100644
--- a/libcxx/include/__algorithm/ranges_remove_copy.h
+++ b/libcxx/include/__algorithm/ranges_remove_copy.h
@@ -37,35 +37,35 @@ using remove_copy_result = in_out_result<_InIter, _OutIter>;
 
 namespace __remove_copy {
 
-  struct __fn {
-    template <input_iterator _InIter,
-              sentinel_for<_InIter> _Sent,
-              weakly_incrementable _OutIter,
-              class _Type,
-              class _Proj = identity>
-      requires indirectly_copyable<_InIter, _OutIter> &&
-               indirect_binary_predicate<ranges::equal_to, projected<_InIter, _Proj>, const _Type*>
-    _LIBCPP_HIDE_FROM_ABI constexpr remove_copy_result<_InIter, _OutIter>
-    operator()(_InIter __first, _Sent __last, _OutIter __result, const _Type& __value, _Proj __proj = {}) const {
-      auto __pred = [&](auto&& __val) { return __value == __val; };
-      return ranges::__remove_copy_if_impl(std::move(__first), std::move(__last), std::move(__result), __pred, __proj);
-    }
+struct __fn {
+  template <input_iterator _InIter,
+            sentinel_for<_InIter> _Sent,
+            weakly_incrementable _OutIter,
+            class _Type,
+            class _Proj = identity>
+    requires indirectly_copyable<_InIter, _OutIter> &&
+             indirect_binary_predicate<ranges::equal_to, projected<_InIter, _Proj>, const _Type*>
+  _LIBCPP_HIDE_FROM_ABI constexpr remove_copy_result<_InIter, _OutIter>
+  operator()(_InIter __first, _Sent __last, _OutIter __result, const _Type& __value, _Proj __proj = {}) const {
+    auto __pred = [&](auto&& __val) { return __value == __val; };
+    return ranges::__remove_copy_if_impl(std::move(__first), std::move(__last), std::move(__result), __pred, __proj);
+  }
 
-    template <input_range _Range, weakly_incrementable _OutIter, class _Type, class _Proj = identity>
-      requires indirectly_copyable<iterator_t<_Range>, _OutIter> &&
-               indirect_binary_predicate<ranges::equal_to, projected<iterator_t<_Range>, _Proj>, const _Type*>
-    _LIBCPP_HIDE_FROM_ABI constexpr remove_copy_result<borrowed_iterator_t<_Range>, _OutIter>
-    operator()(_Range&& __range, _OutIter __result, const _Type& __value, _Proj __proj = {}) const {
-      auto __pred = [&](auto&& __val) { return __value == __val; };
-      return ranges::__remove_copy_if_impl(
-          ranges::begin(__range), ranges::end(__range), std::move(__result), __pred, __proj);
-    }
-  };
+  template <input_range _Range, weakly_incrementable _OutIter, class _Type, class _Proj = identity>
+    requires indirectly_copyable<iterator_t<_Range>, _OutIter> &&
+             indirect_binary_predicate<ranges::equal_to, projected<iterator_t<_Range>, _Proj>, const _Type*>
+  _LIBCPP_HIDE_FROM_ABI constexpr remove_copy_result<borrowed_iterator_t<_Range>, _OutIter>
+  operator()(_Range&& __range, _OutIter __result, const _Type& __value, _Proj __proj = {}) const {
+    auto __pred = [&](auto&& __val) { return __value == __val; };
+    return ranges::__remove_copy_if_impl(
+        ranges::begin(__range), ranges::end(__range), std::move(__result), __pred, __proj);
+  }
+};
 
 } // namespace __remove_copy
 
 inline namespace __cpo {
-  inline constexpr auto remove_copy = __remove_copy::__fn{};
+inline constexpr auto remove_copy = __remove_copy::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_remove_copy_if.h b/libcxx/include/__algorithm/ranges_remove_copy_if.h
index e6d043e365862..c07b4813d7d0a 100644
--- a/libcxx/include/__algorithm/ranges_remove_copy_if.h
+++ b/libcxx/include/__algorithm/ranges_remove_copy_if.h
@@ -52,34 +52,34 @@ __remove_copy_if_impl(_InIter __first, _Sent __last, _OutIter __result, _Pred& _
 
 namespace __remove_copy_if {
 
-  struct __fn {
-    template <input_iterator _InIter,
-              sentinel_for<_InIter> _Sent,
-              weakly_incrementable _OutIter,
-              class _Proj = identity,
-              indirect_unary_predicate<projected<_InIter, _Proj>> _Pred>
-      requires indirectly_copyable<_InIter, _OutIter>
-    _LIBCPP_HIDE_FROM_ABI constexpr remove_copy_if_result<_InIter, _OutIter>
-    operator()(_InIter __first, _Sent __last, _OutIter __result, _Pred __pred, _Proj __proj = {}) const {
-      return ranges::__remove_copy_if_impl(std::move(__first), std::move(__last), std::move(__result), __pred, __proj);
-    }
+struct __fn {
+  template <input_iterator _InIter,
+            sentinel_for<_InIter> _Sent,
+            weakly_incrementable _OutIter,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<_InIter, _Proj>> _Pred>
+    requires indirectly_copyable<_InIter, _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr remove_copy_if_result<_InIter, _OutIter>
+  operator()(_InIter __first, _Sent __last, _OutIter __result, _Pred __pred, _Proj __proj = {}) const {
+    return ranges::__remove_copy_if_impl(std::move(__first), std::move(__last), std::move(__result), __pred, __proj);
+  }
 
-    template <input_range _Range,
-              weakly_incrementable _OutIter,
-              class _Proj = identity,
-              indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
-      requires indirectly_copyable<iterator_t<_Range>, _OutIter>
-    _LIBCPP_HIDE_FROM_ABI constexpr remove_copy_if_result<borrowed_iterator_t<_Range>, _OutIter>
-    operator()(_Range&& __range, _OutIter __result, _Pred __pred, _Proj __proj = {}) const {
-      return ranges::__remove_copy_if_impl(
-          ranges::begin(__range), ranges::end(__range), std::move(__result), __pred, __proj);
-    }
-  };
+  template <input_range _Range,
+            weakly_incrementable _OutIter,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
+    requires indirectly_copyable<iterator_t<_Range>, _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr remove_copy_if_result<borrowed_iterator_t<_Range>, _OutIter>
+  operator()(_Range&& __range, _OutIter __result, _Pred __pred, _Proj __proj = {}) const {
+    return ranges::__remove_copy_if_impl(
+        ranges::begin(__range), ranges::end(__range), std::move(__result), __pred, __proj);
+  }
+};
 
 } // namespace __remove_copy_if
 
 inline namespace __cpo {
-  inline constexpr auto remove_copy_if = __remove_copy_if::__fn{};
+inline constexpr auto remove_copy_if = __remove_copy_if::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_remove_if.h b/libcxx/include/__algorithm/ranges_remove_if.h
index 7507c21b87d30..4b7aa2d2be78a 100644
--- a/libcxx/include/__algorithm/ranges_remove_if.h
+++ b/libcxx/include/__algorithm/ranges_remove_if.h
@@ -34,8 +34,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 
 template <class _Iter, class _Sent, class _Proj, class _Pred>
-_LIBCPP_HIDE_FROM_ABI constexpr
-subrange<_Iter> __remove_if_impl(_Iter __first, _Sent __last, _Pred& __pred, _Proj& __proj) {
+_LIBCPP_HIDE_FROM_ABI constexpr subrange<_Iter>
+__remove_if_impl(_Iter __first, _Sent __last, _Pred& __pred, _Proj& __proj) {
   auto __new_end = ranges::__find_if_impl(__first, __last, __pred, __proj);
   if (__new_end == __last)
     return {__new_end, __new_end};
@@ -52,12 +52,12 @@ subrange<_Iter> __remove_if_impl(_Iter __first, _Sent __last, _Pred& __pred, _Pr
 
 namespace __remove_if {
 struct __fn {
-
-  template <permutable _Iter, sentinel_for<_Iter> _Sent,
+  template <permutable _Iter,
+            sentinel_for<_Iter> _Sent,
             class _Proj = identity,
             indirect_unary_predicate<projected<_Iter, _Proj>> _Pred>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  subrange<_Iter> operator()(_Iter __first, _Sent __last, _Pred __pred, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr subrange<_Iter>
+  operator()(_Iter __first, _Sent __last, _Pred __pred, _Proj __proj = {}) const {
     return ranges::__remove_if_impl(std::move(__first), std::move(__last), __pred, __proj);
   }
 
@@ -65,16 +65,15 @@ struct __fn {
             class _Proj = identity,
             indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
     requires permutable<iterator_t<_Range>>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_subrange_t<_Range> operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr borrowed_subrange_t<_Range>
+  operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) const {
     return ranges::__remove_if_impl(ranges::begin(__range), ranges::end(__range), __pred, __proj);
   }
-
 };
 } // namespace __remove_if
 
 inline namespace __cpo {
-  inline constexpr auto remove_if = __remove_if::__fn{};
+inline constexpr auto remove_if = __remove_if::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_replace.h b/libcxx/include/__algorithm/ranges_replace.h
index 258e25d0569e9..714fd5c7b089f 100644
--- a/libcxx/include/__algorithm/ranges_replace.h
+++ b/libcxx/include/__algorithm/ranges_replace.h
@@ -31,39 +31,28 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 namespace __replace {
 struct __fn {
-
-  template <input_iterator _Iter, sentinel_for<_Iter> _Sent,
-            class _Type1,
-            class _Type2,
-            class _Proj = identity>
-    requires indirectly_writable<_Iter, const _Type2&>
-          && indirect_binary_predicate<ranges::equal_to, projected<_Iter, _Proj>, const _Type1*>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  _Iter operator()(_Iter __first, _Sent __last,
-                   const _Type1& __old_value,
-                   const _Type2& __new_value,
-                   _Proj __proj = {}) const {
+  template <input_iterator _Iter, sentinel_for<_Iter> _Sent, class _Type1, class _Type2, class _Proj = identity>
+    requires indirectly_writable<_Iter, const _Type2&> &&
+             indirect_binary_predicate<ranges::equal_to, projected<_Iter, _Proj>, const _Type1*>
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter operator()(
+      _Iter __first, _Sent __last, const _Type1& __old_value, const _Type2& __new_value, _Proj __proj = {}) const {
     auto __pred = [&](const auto& __val) { return __val == __old_value; };
     return ranges::__replace_if_impl(std::move(__first), std::move(__last), __pred, __new_value, __proj);
   }
 
-  template <input_range _Range,
-            class _Type1,
-            class _Type2,
-            class _Proj = identity>
-    requires indirectly_writable<iterator_t<_Range>, const _Type2&>
-          && indirect_binary_predicate<ranges::equal_to, projected<iterator_t<_Range>, _Proj>, const _Type1*>
+  template <input_range _Range, class _Type1, class _Type2, class _Proj = identity>
+    requires indirectly_writable<iterator_t<_Range>, const _Type2&> &&
+             indirect_binary_predicate<ranges::equal_to, projected<iterator_t<_Range>, _Proj>, const _Type1*>
   _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
   operator()(_Range&& __range, const _Type1& __old_value, const _Type2& __new_value, _Proj __proj = {}) const {
     auto __pred = [&](auto&& __val) { return __val == __old_value; };
     return ranges::__replace_if_impl(ranges::begin(__range), ranges::end(__range), __pred, __new_value, __proj);
   }
-
 };
 } // namespace __replace
 
 inline namespace __cpo {
-  inline constexpr auto replace = __replace::__fn{};
+inline constexpr auto replace = __replace::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_replace_copy.h b/libcxx/include/__algorithm/ranges_replace_copy.h
index 79eaebb41527d..124ff8f2c559d 100644
--- a/libcxx/include/__algorithm/ranges_replace_copy.h
+++ b/libcxx/include/__algorithm/ranges_replace_copy.h
@@ -37,50 +37,47 @@ using replace_copy_result = in_out_result<_InIter, _OutIter>;
 
 namespace __replace_copy {
 
-  struct __fn {
-    template <input_iterator _InIter,
-              sentinel_for<_InIter> _Sent,
-              class _OldType,
-              class _NewType,
-              output_iterator<const _NewType&> _OutIter,
-              class _Proj = identity>
-      requires indirectly_copyable<_InIter, _OutIter> &&
-               indirect_binary_predicate<ranges::equal_to, projected<_InIter, _Proj>, const _OldType*>
-    _LIBCPP_HIDE_FROM_ABI constexpr replace_copy_result<_InIter, _OutIter>
-    operator()(_InIter __first,
-               _Sent __last,
-               _OutIter __result,
-               const _OldType& __old_value,
-               const _NewType& __new_value,
-               _Proj __proj = {}) const {
-      auto __pred = [&](const auto& __value) { return __value == __old_value; };
-      return ranges::__replace_copy_if_impl(
-          std::move(__first), std::move(__last), std::move(__result), __pred, __new_value, __proj);
-    }
+struct __fn {
+  template <input_iterator _InIter,
+            sentinel_for<_InIter> _Sent,
+            class _OldType,
+            class _NewType,
+            output_iterator<const _NewType&> _OutIter,
+            class _Proj = identity>
+    requires indirectly_copyable<_InIter, _OutIter> &&
+             indirect_binary_predicate<ranges::equal_to, projected<_InIter, _Proj>, const _OldType*>
+  _LIBCPP_HIDE_FROM_ABI constexpr replace_copy_result<_InIter, _OutIter>
+  operator()(_InIter __first,
+             _Sent __last,
+             _OutIter __result,
+             const _OldType& __old_value,
+             const _NewType& __new_value,
+             _Proj __proj = {}) const {
+    auto __pred = [&](const auto& __value) { return __value == __old_value; };
+    return ranges::__replace_copy_if_impl(
+        std::move(__first), std::move(__last), std::move(__result), __pred, __new_value, __proj);
+  }
 
-    template <input_range _Range,
-              class _OldType,
-              class _NewType,
-              output_iterator<const _NewType&> _OutIter,
-              class _Proj = identity>
-      requires indirectly_copyable<iterator_t<_Range>, _OutIter> &&
-               indirect_binary_predicate<ranges::equal_to, projected<iterator_t<_Range>, _Proj>, const _OldType*>
-    _LIBCPP_HIDE_FROM_ABI constexpr replace_copy_result<borrowed_iterator_t<_Range>, _OutIter>
-    operator()(_Range&& __range,
-               _OutIter __result,
-               const _OldType& __old_value,
-               const _NewType& __new_value,
-               _Proj __proj = {}) const {
-      auto __pred = [&](const auto& __value) { return __value == __old_value; };
-      return ranges::__replace_copy_if_impl(
-          ranges::begin(__range), ranges::end(__range), std::move(__result), __pred, __new_value, __proj);
-    }
-  };
+  template <input_range _Range,
+            class _OldType,
+            class _NewType,
+            output_iterator<const _NewType&> _OutIter,
+            class _Proj = identity>
+    requires indirectly_copyable<iterator_t<_Range>, _OutIter> &&
+             indirect_binary_predicate<ranges::equal_to, projected<iterator_t<_Range>, _Proj>, const _OldType*>
+  _LIBCPP_HIDE_FROM_ABI constexpr replace_copy_result<borrowed_iterator_t<_Range>, _OutIter> operator()(
+      _Range&& __range, _OutIter __result, const _OldType& __old_value, const _NewType& __new_value, _Proj __proj = {})
+      const {
+    auto __pred = [&](const auto& __value) { return __value == __old_value; };
+    return ranges::__replace_copy_if_impl(
+        ranges::begin(__range), ranges::end(__range), std::move(__result), __pred, __new_value, __proj);
+  }
+};
 
 } // namespace __replace_copy
 
 inline namespace __cpo {
-  inline constexpr auto replace_copy = __replace_copy::__fn{};
+inline constexpr auto replace_copy = __replace_copy::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_replace_copy_if.h b/libcxx/include/__algorithm/ranges_replace_copy_if.h
index 8ff2eba8fb0a5..10ed1fda6c5c8 100644
--- a/libcxx/include/__algorithm/ranges_replace_copy_if.h
+++ b/libcxx/include/__algorithm/ranges_replace_copy_if.h
@@ -51,38 +51,38 @@ _LIBCPP_HIDE_FROM_ABI constexpr replace_copy_if_result<_InIter, _OutIter> __repl
 
 namespace __replace_copy_if {
 
-  struct __fn {
-    template <input_iterator _InIter,
-              sentinel_for<_InIter> _Sent,
-              class _Type,
-              output_iterator<const _Type&> _OutIter,
-              class _Proj = identity,
-              indirect_unary_predicate<projected<_InIter, _Proj>> _Pred>
-      requires indirectly_copyable<_InIter, _OutIter>
-    _LIBCPP_HIDE_FROM_ABI constexpr replace_copy_if_result<_InIter, _OutIter> operator()(
-        _InIter __first, _Sent __last, _OutIter __result, _Pred __pred, const _Type& __new_value, _Proj __proj = {})
-        const {
-      return ranges::__replace_copy_if_impl(
-          std::move(__first), std::move(__last), std::move(__result), __pred, __new_value, __proj);
-    }
-
-    template <input_range _Range,
-              class _Type,
-              output_iterator<const _Type&> _OutIter,
-              class _Proj = identity,
-              indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
-      requires indirectly_copyable<iterator_t<_Range>, _OutIter>
-    _LIBCPP_HIDE_FROM_ABI constexpr replace_copy_if_result<borrowed_iterator_t<_Range>, _OutIter>
-    operator()(_Range&& __range, _OutIter __result, _Pred __pred, const _Type& __new_value, _Proj __proj = {}) const {
-      return ranges::__replace_copy_if_impl(
-          ranges::begin(__range), ranges::end(__range), std::move(__result), __pred, __new_value, __proj);
-    }
-  };
+struct __fn {
+  template <input_iterator _InIter,
+            sentinel_for<_InIter> _Sent,
+            class _Type,
+            output_iterator<const _Type&> _OutIter,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<_InIter, _Proj>> _Pred>
+    requires indirectly_copyable<_InIter, _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr replace_copy_if_result<_InIter, _OutIter> operator()(
+      _InIter __first, _Sent __last, _OutIter __result, _Pred __pred, const _Type& __new_value, _Proj __proj = {})
+      const {
+    return ranges::__replace_copy_if_impl(
+        std::move(__first), std::move(__last), std::move(__result), __pred, __new_value, __proj);
+  }
+
+  template <input_range _Range,
+            class _Type,
+            output_iterator<const _Type&> _OutIter,
+            class _Proj = identity,
+            indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
+    requires indirectly_copyable<iterator_t<_Range>, _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr replace_copy_if_result<borrowed_iterator_t<_Range>, _OutIter>
+  operator()(_Range&& __range, _OutIter __result, _Pred __pred, const _Type& __new_value, _Proj __proj = {}) const {
+    return ranges::__replace_copy_if_impl(
+        ranges::begin(__range), ranges::end(__range), std::move(__result), __pred, __new_value, __proj);
+  }
+};
 
 } // namespace __replace_copy_if
 
 inline namespace __cpo {
-  inline constexpr auto replace_copy_if = __replace_copy_if::__fn{};
+inline constexpr auto replace_copy_if = __replace_copy_if::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_replace_if.h b/libcxx/include/__algorithm/ranges_replace_if.h
index 2b7dda9e69f3b..519fa32029ac6 100644
--- a/libcxx/include/__algorithm/ranges_replace_if.h
+++ b/libcxx/include/__algorithm/ranges_replace_if.h
@@ -30,8 +30,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 
 template <class _Iter, class _Sent, class _Type, class _Proj, class _Pred>
-_LIBCPP_HIDE_FROM_ABI constexpr
-_Iter __replace_if_impl(_Iter __first, _Sent __last, _Pred& __pred, const _Type& __new_value, _Proj& __proj) {
+_LIBCPP_HIDE_FROM_ABI constexpr _Iter
+__replace_if_impl(_Iter __first, _Sent __last, _Pred& __pred, const _Type& __new_value, _Proj& __proj) {
   for (; __first != __last; ++__first) {
     if (std::invoke(__pred, std::invoke(__proj, *__first)))
       *__first = __new_value;
@@ -41,14 +41,14 @@ _Iter __replace_if_impl(_Iter __first, _Sent __last, _Pred& __pred, const _Type&
 
 namespace __replace_if {
 struct __fn {
-
-  template <input_iterator _Iter, sentinel_for<_Iter> _Sent,
+  template <input_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
             class _Type,
             class _Proj = identity,
             indirect_unary_predicate<projected<_Iter, _Proj>> _Pred>
     requires indirectly_writable<_Iter, const _Type&>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  _Iter operator()(_Iter __first, _Sent __last, _Pred __pred, const _Type& __new_value, _Proj __proj = {}) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, _Sent __last, _Pred __pred, const _Type& __new_value, _Proj __proj = {}) const {
     return ranges::__replace_if_impl(std::move(__first), std::move(__last), __pred, __new_value, __proj);
   }
 
@@ -61,12 +61,11 @@ struct __fn {
   operator()(_Range&& __range, _Pred __pred, const _Type& __new_value, _Proj __proj = {}) const {
     return ranges::__replace_if_impl(ranges::begin(__range), ranges::end(__range), __pred, __new_value, __proj);
   }
-
 };
 } // namespace __replace_if
 
 inline namespace __cpo {
-  inline constexpr auto replace_if = __replace_if::__fn{};
+inline constexpr auto replace_if = __replace_if::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_reverse.h b/libcxx/include/__algorithm/ranges_reverse.h
index 801fccb853f20..9ec865995b4a5 100644
--- a/libcxx/include/__algorithm/ranges_reverse.h
+++ b/libcxx/include/__algorithm/ranges_reverse.h
@@ -29,11 +29,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 namespace __reverse {
 struct __fn {
-
   template <bidirectional_iterator _Iter, sentinel_for<_Iter> _Sent>
     requires permutable<_Iter>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  _Iter operator()(_Iter __first, _Sent __last) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter operator()(_Iter __first, _Sent __last) const {
     if constexpr (random_access_iterator<_Iter>) {
       if (__first == __last)
         return __first;
@@ -63,16 +61,14 @@ struct __fn {
 
   template <bidirectional_range _Range>
     requires permutable<iterator_t<_Range>>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_iterator_t<_Range> operator()(_Range&& __range) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range> operator()(_Range&& __range) const {
     return (*this)(ranges::begin(__range), ranges::end(__range));
   }
-
 };
 } // namespace __reverse
 
 inline namespace __cpo {
-  inline constexpr auto reverse = __reverse::__fn{};
+inline constexpr auto reverse = __reverse::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_reverse_copy.h b/libcxx/include/__algorithm/ranges_reverse_copy.h
index c23039976347b..35b9edba0bfb2 100644
--- a/libcxx/include/__algorithm/ranges_reverse_copy.h
+++ b/libcxx/include/__algorithm/ranges_reverse_copy.h
@@ -36,27 +36,25 @@ using reverse_copy_result = in_out_result<_InIter, _OutIter>;
 
 namespace __reverse_copy {
 struct __fn {
-
   template <bidirectional_iterator _InIter, sentinel_for<_InIter> _Sent, weakly_incrementable _OutIter>
     requires indirectly_copyable<_InIter, _OutIter>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  reverse_copy_result<_InIter, _OutIter> operator()(_InIter __first, _Sent __last, _OutIter __result) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr reverse_copy_result<_InIter, _OutIter>
+  operator()(_InIter __first, _Sent __last, _OutIter __result) const {
     return (*this)(subrange(std::move(__first), std::move(__last)), std::move(__result));
   }
 
   template <bidirectional_range _Range, weakly_incrementable _OutIter>
     requires indirectly_copyable<iterator_t<_Range>, _OutIter>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  reverse_copy_result<borrowed_iterator_t<_Range>, _OutIter> operator()(_Range&& __range, _OutIter __result) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr reverse_copy_result<borrowed_iterator_t<_Range>, _OutIter>
+  operator()(_Range&& __range, _OutIter __result) const {
     auto __ret = ranges::copy(std::__reverse_range(__range), std::move(__result));
     return {ranges::next(ranges::begin(__range), ranges::end(__range)), std::move(__ret.out)};
   }
-
 };
 } // namespace __reverse_copy
 
 inline namespace __cpo {
-  inline constexpr auto reverse_copy = __reverse_copy::__fn{};
+inline constexpr auto reverse_copy = __reverse_copy::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_rotate.h b/libcxx/include/__algorithm/ranges_rotate.h
index 99e6ca67e71a0..ebed9bbd54266 100644
--- a/libcxx/include/__algorithm/ranges_rotate.h
+++ b/libcxx/include/__algorithm/ranges_rotate.h
@@ -33,34 +33,29 @@ namespace ranges {
 namespace __rotate {
 
 struct __fn {
-
   template <class _Iter, class _Sent>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  static subrange<_Iter> __rotate_fn_impl(_Iter __first, _Iter __middle, _Sent __last) {
-    auto __ret = std::__rotate<_RangeAlgPolicy>(
-      std::move(__first), std::move(__middle), std::move(__last));
+  _LIBCPP_HIDE_FROM_ABI constexpr static subrange<_Iter> __rotate_fn_impl(_Iter __first, _Iter __middle, _Sent __last) {
+    auto __ret = std::__rotate<_RangeAlgPolicy>(std::move(__first), std::move(__middle), std::move(__last));
     return {std::move(__ret.first), std::move(__ret.second)};
   }
 
   template <permutable _Iter, sentinel_for<_Iter> _Sent>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  subrange<_Iter> operator()(_Iter __first, _Iter __middle, _Sent __last) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr subrange<_Iter> operator()(_Iter __first, _Iter __middle, _Sent __last) const {
     return __rotate_fn_impl(std::move(__first), std::move(__middle), std::move(__last));
   }
 
   template <forward_range _Range>
-  requires permutable<iterator_t<_Range>>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_subrange_t<_Range> operator()(_Range&& __range, iterator_t<_Range> __middle) const {
+    requires permutable<iterator_t<_Range>>
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_subrange_t<_Range>
+  operator()(_Range&& __range, iterator_t<_Range> __middle) const {
     return __rotate_fn_impl(ranges::begin(__range), std::move(__middle), ranges::end(__range));
   }
-
 };
 
 } // namespace __rotate
 
 inline namespace __cpo {
-  inline constexpr auto rotate = __rotate::__fn{};
+inline constexpr auto rotate = __rotate::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_rotate_copy.h b/libcxx/include/__algorithm/ranges_rotate_copy.h
index 5b9321f90d5da..301672707f72b 100644
--- a/libcxx/include/__algorithm/ranges_rotate_copy.h
+++ b/libcxx/include/__algorithm/ranges_rotate_copy.h
@@ -34,11 +34,9 @@ using rotate_copy_result = in_out_result<_InIter, _OutIter>;
 
 namespace __rotate_copy {
 struct __fn {
-
   template <bidirectional_iterator _InIter, sentinel_for<_InIter> _Sent, weakly_incrementable _OutIter>
     requires indirectly_copyable<_InIter, _OutIter>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  rotate_copy_result<_InIter, _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr rotate_copy_result<_InIter, _OutIter>
   operator()(_InIter __first, _InIter __middle, _Sent __last, _OutIter __result) const {
     auto __res1 = ranges::copy(__middle, __last, std::move(__result));
     auto __res2 = ranges::copy(__first, __middle, std::move(__res1.out));
@@ -47,17 +45,15 @@ struct __fn {
 
   template <bidirectional_range _Range, weakly_incrementable _OutIter>
     requires indirectly_copyable<iterator_t<_Range>, _OutIter>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  rotate_copy_result<borrowed_iterator_t<_Range>, _OutIter>
+  _LIBCPP_HIDE_FROM_ABI constexpr rotate_copy_result<borrowed_iterator_t<_Range>, _OutIter>
   operator()(_Range&& __range, iterator_t<_Range> __middle, _OutIter __result) const {
     return (*this)(ranges::begin(__range), std::move(__middle), ranges::end(__range), std::move(__result));
   }
-
 };
 } // namespace __rotate_copy
 
 inline namespace __cpo {
-  inline constexpr auto rotate_copy = __rotate_copy::__fn{};
+inline constexpr auto rotate_copy = __rotate_copy::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_sample.h b/libcxx/include/__algorithm/ranges_sample.h
index a5ff2d0c8ece1..d347d82205a89 100644
--- a/libcxx/include/__algorithm/ranges_sample.h
+++ b/libcxx/include/__algorithm/ranges_sample.h
@@ -35,35 +35,30 @@ namespace ranges {
 namespace __sample {
 
 struct __fn {
-
   template <input_iterator _Iter, sentinel_for<_Iter> _Sent, weakly_incrementable _OutIter, class _Gen>
-  requires (forward_iterator<_Iter> || random_access_iterator<_OutIter>) &&
-           indirectly_copyable<_Iter, _OutIter> &&
-           uniform_random_bit_generator<remove_reference_t<_Gen>>
-  _LIBCPP_HIDE_FROM_ABI
-  _OutIter operator()(_Iter __first, _Sent __last,
-                      _OutIter __out_first, iter_difference_t<_Iter> __n, _Gen&& __gen) const {
+    requires(forward_iterator<_Iter> || random_access_iterator<_OutIter>) && indirectly_copyable<_Iter, _OutIter> &&
+            uniform_random_bit_generator<remove_reference_t<_Gen>>
+  _LIBCPP_HIDE_FROM_ABI _OutIter
+  operator()(_Iter __first, _Sent __last, _OutIter __out_first, iter_difference_t<_Iter> __n, _Gen&& __gen) const {
     _ClassicGenAdaptor<_Gen> __adapted_gen(__gen);
     return std::__sample<_RangeAlgPolicy>(
         std::move(__first), std::move(__last), std::move(__out_first), __n, __adapted_gen);
   }
 
   template <input_range _Range, weakly_incrementable _OutIter, class _Gen>
-  requires (forward_range<_Range> || random_access_iterator<_OutIter>) &&
-           indirectly_copyable<iterator_t<_Range>, _OutIter> &&
-           uniform_random_bit_generator<remove_reference_t<_Gen>>
-  _LIBCPP_HIDE_FROM_ABI
-  _OutIter operator()(_Range&& __range, _OutIter __out_first, range_difference_t<_Range> __n, _Gen&& __gen) const {
-    return (*this)(ranges::begin(__range), ranges::end(__range),
-                   std::move(__out_first), __n, std::forward<_Gen>(__gen));
+    requires(forward_range<_Range> || random_access_iterator<_OutIter>) &&
+            indirectly_copyable<iterator_t<_Range>, _OutIter> && uniform_random_bit_generator<remove_reference_t<_Gen>>
+  _LIBCPP_HIDE_FROM_ABI _OutIter
+  operator()(_Range&& __range, _OutIter __out_first, range_difference_t<_Range> __n, _Gen&& __gen) const {
+    return (*this)(
+        ranges::begin(__range), ranges::end(__range), std::move(__out_first), __n, std::forward<_Gen>(__gen));
   }
-
 };
 
 } // namespace __sample
 
 inline namespace __cpo {
-  inline constexpr auto sample = __sample::__fn{};
+inline constexpr auto sample = __sample::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_search.h b/libcxx/include/__algorithm/ranges_search.h
index 6d3e3186881a7..ca2326e9ab273 100644
--- a/libcxx/include/__algorithm/ranges_search.h
+++ b/libcxx/include/__algorithm/ranges_search.h
@@ -69,33 +69,33 @@ struct __fn {
     return {__ret.first, __ret.second};
   }
 
-  template <forward_iterator _Iter1, sentinel_for<_Iter1> _Sent1,
-            forward_iterator _Iter2, sentinel_for<_Iter2> _Sent2,
-            class _Pred = ranges::equal_to,
+  template <forward_iterator _Iter1,
+            sentinel_for<_Iter1> _Sent1,
+            forward_iterator _Iter2,
+            sentinel_for<_Iter2> _Sent2,
+            class _Pred  = ranges::equal_to,
             class _Proj1 = identity,
             class _Proj2 = identity>
     requires indirectly_comparable<_Iter1, _Iter2, _Pred, _Proj1, _Proj2>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  subrange<_Iter1> operator()(_Iter1 __first1, _Sent1 __last1,
-                              _Iter2 __first2, _Sent2 __last2,
-                              _Pred __pred = {},
-                              _Proj1 __proj1 = {},
-                              _Proj2 __proj2 = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr subrange<_Iter1> operator()(
+      _Iter1 __first1,
+      _Sent1 __last1,
+      _Iter2 __first2,
+      _Sent2 __last2,
+      _Pred __pred   = {},
+      _Proj1 __proj1 = {},
+      _Proj2 __proj2 = {}) const {
     return __ranges_search_impl(__first1, __last1, __first2, __last2, __pred, __proj1, __proj2);
   }
 
   template <forward_range _Range1,
             forward_range _Range2,
-            class _Pred = ranges::equal_to,
+            class _Pred  = ranges::equal_to,
             class _Proj1 = identity,
             class _Proj2 = identity>
     requires indirectly_comparable<iterator_t<_Range1>, iterator_t<_Range2>, _Pred, _Proj1, _Proj2>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_subrange_t<_Range1> operator()(_Range1&& __range1,
-                                          _Range2&& __range2,
-                                          _Pred __pred = {},
-                                          _Proj1 __proj1 = {},
-                                          _Proj2 __proj2 = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr borrowed_subrange_t<_Range1> operator()(
+      _Range1&& __range1, _Range2&& __range2, _Pred __pred = {}, _Proj1 __proj1 = {}, _Proj2 __proj2 = {}) const {
     auto __first1 = ranges::begin(__range1);
     if constexpr (sized_range<_Range2>) {
       auto __size2 = ranges::size(__range2);
@@ -119,12 +119,11 @@ struct __fn {
         __proj1,
         __proj2);
   }
-
 };
 } // namespace __search
 
 inline namespace __cpo {
-  inline constexpr auto search = __search::__fn{};
+inline constexpr auto search = __search::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_search_n.h b/libcxx/include/__algorithm/ranges_search_n.h
index ed5ec34f06348..4e53f30f71f9d 100644
--- a/libcxx/include/__algorithm/ranges_search_n.h
+++ b/libcxx/include/__algorithm/ranges_search_n.h
@@ -38,7 +38,6 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 namespace __search_n {
 struct __fn {
-
   template <class _Iter1, class _Sent1, class _SizeT, class _Type, class _Pred, class _Proj>
   _LIBCPP_HIDE_FROM_ABI static constexpr subrange<_Iter1> __ranges_search_n_impl(
       _Iter1 __first, _Sent1 __last, _SizeT __count, const _Type& __value, _Pred& __pred, _Proj& __proj) {
@@ -59,36 +58,31 @@ struct __fn {
       }
     }
 
-    auto __ret = std::__search_n_forward_impl<_RangeAlgPolicy>(__first, __last,
-                                                               __count,
-                                                               __value,
-                                                               __pred,
-                                                               __proj);
+    auto __ret = std::__search_n_forward_impl<_RangeAlgPolicy>(__first, __last, __count, __value, __pred, __proj);
     return {std::move(__ret.first), std::move(__ret.second)};
   }
 
-  template <forward_iterator _Iter, sentinel_for<_Iter> _Sent,
+  template <forward_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
             class _Type,
             class _Pred = ranges::equal_to,
             class _Proj = identity>
     requires indirectly_comparable<_Iter, const _Type*, _Pred, _Proj>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  subrange<_Iter> operator()(_Iter __first, _Sent __last,
-                             iter_difference_t<_Iter> __count,
-                             const _Type& __value,
-                             _Pred __pred = {},
-                             _Proj __proj = _Proj{}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr subrange<_Iter>
+  operator()(_Iter __first,
+             _Sent __last,
+             iter_difference_t<_Iter> __count,
+             const _Type& __value,
+             _Pred __pred = {},
+             _Proj __proj = _Proj{}) const {
     return __ranges_search_n_impl(__first, __last, __count, __value, __pred, __proj);
   }
 
   template <forward_range _Range, class _Type, class _Pred = ranges::equal_to, class _Proj = identity>
     requires indirectly_comparable<iterator_t<_Range>, const _Type*, _Pred, _Proj>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_subrange_t<_Range> operator()(_Range&& __range,
-                                         range_difference_t<_Range> __count,
-                                         const _Type& __value,
-                                         _Pred __pred = {},
-                                         _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr borrowed_subrange_t<_Range> operator()(
+      _Range&& __range, range_difference_t<_Range> __count, const _Type& __value, _Pred __pred = {}, _Proj __proj = {})
+      const {
     auto __first = ranges::begin(__range);
     if (__count <= 0)
       return {__first, __first};
@@ -106,7 +100,7 @@ struct __fn {
 } // namespace __search_n
 
 inline namespace __cpo {
-  inline constexpr auto search_n = __search_n::__fn{};
+inline constexpr auto search_n = __search_n::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_set_difference.h b/libcxx/include/__algorithm/ranges_set_difference.h
index 6b9af876b5fe5..a9453ed336f51 100644
--- a/libcxx/include/__algorithm/ranges_set_difference.h
+++ b/libcxx/include/__algorithm/ranges_set_difference.h
@@ -42,15 +42,14 @@ using set_difference_result = in_out_result<_InIter, _OutIter>;
 namespace __set_difference {
 
 struct __fn {
-  template <
-      input_iterator _InIter1,
-      sentinel_for<_InIter1> _Sent1,
-      input_iterator _InIter2,
-      sentinel_for<_InIter2> _Sent2,
-      weakly_incrementable _OutIter,
-      class _Comp  = less,
-      class _Proj1 = identity,
-      class _Proj2 = identity>
+  template <input_iterator _InIter1,
+            sentinel_for<_InIter1> _Sent1,
+            input_iterator _InIter2,
+            sentinel_for<_InIter2> _Sent2,
+            weakly_incrementable _OutIter,
+            class _Comp  = less,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
     requires mergeable<_InIter1, _InIter2, _OutIter, _Comp, _Proj1, _Proj2>
   _LIBCPP_HIDE_FROM_ABI constexpr set_difference_result<_InIter1, _OutIter> operator()(
       _InIter1 __first1,
@@ -66,22 +65,20 @@ struct __fn {
     return {std::move(__ret.first), std::move(__ret.second)};
   }
 
-  template <
-      input_range _Range1,
-      input_range _Range2,
-      weakly_incrementable _OutIter,
-      class _Comp  = less,
-      class _Proj1 = identity,
-      class _Proj2 = identity>
+  template <input_range _Range1,
+            input_range _Range2,
+            weakly_incrementable _OutIter,
+            class _Comp  = less,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
     requires mergeable<iterator_t<_Range1>, iterator_t<_Range2>, _OutIter, _Comp, _Proj1, _Proj2>
   _LIBCPP_HIDE_FROM_ABI constexpr set_difference_result<borrowed_iterator_t<_Range1>, _OutIter>
-    operator()(
-        _Range1&& __range1,
-        _Range2&& __range2,
-        _OutIter __result,
-        _Comp __comp   = {},
-        _Proj1 __proj1 = {},
-        _Proj2 __proj2 = {}) const {
+  operator()(_Range1&& __range1,
+             _Range2&& __range2,
+             _OutIter __result,
+             _Comp __comp   = {},
+             _Proj1 __proj1 = {},
+             _Proj2 __proj2 = {}) const {
     auto __ret = std::__set_difference<_RangeAlgPolicy>(
         ranges::begin(__range1),
         ranges::end(__range1),
@@ -96,7 +93,7 @@ struct __fn {
 } // namespace __set_difference
 
 inline namespace __cpo {
-  inline constexpr auto set_difference = __set_difference::__fn{};
+inline constexpr auto set_difference = __set_difference::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_set_intersection.h b/libcxx/include/__algorithm/ranges_set_intersection.h
index 5848656ce3423..4cdcbb75051a1 100644
--- a/libcxx/include/__algorithm/ranges_set_intersection.h
+++ b/libcxx/include/__algorithm/ranges_set_intersection.h
@@ -40,15 +40,14 @@ using set_intersection_result = in_in_out_result<_InIter1, _InIter2, _OutIter>;
 namespace __set_intersection {
 
 struct __fn {
-  template <
-      input_iterator _InIter1,
-      sentinel_for<_InIter1> _Sent1,
-      input_iterator _InIter2,
-      sentinel_for<_InIter2> _Sent2,
-      weakly_incrementable _OutIter,
-      class _Comp  = less,
-      class _Proj1 = identity,
-      class _Proj2 = identity>
+  template <input_iterator _InIter1,
+            sentinel_for<_InIter1> _Sent1,
+            input_iterator _InIter2,
+            sentinel_for<_InIter2> _Sent2,
+            weakly_incrementable _OutIter,
+            class _Comp  = less,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
     requires mergeable<_InIter1, _InIter2, _OutIter, _Comp, _Proj1, _Proj2>
   _LIBCPP_HIDE_FROM_ABI constexpr set_intersection_result<_InIter1, _InIter2, _OutIter> operator()(
       _InIter1 __first1,
@@ -69,30 +68,22 @@ struct __fn {
     return {std::move(__ret.__in1_), std::move(__ret.__in2_), std::move(__ret.__out_)};
   }
 
-  template <
-      input_range _Range1,
-      input_range _Range2,
-      weakly_incrementable _OutIter,
-      class _Comp  = less,
-      class _Proj1 = identity,
-      class _Proj2 = identity>
-    requires mergeable<
-        iterator_t<_Range1>,
-        iterator_t<_Range2>,
-        _OutIter,
-        _Comp,
-        _Proj1,
-        _Proj2>
-    _LIBCPP_HIDE_FROM_ABI constexpr set_intersection_result<borrowed_iterator_t<_Range1>,
-                                                            borrowed_iterator_t<_Range2>,
-                                                            _OutIter>
-    operator()(
-        _Range1&& __range1,
-        _Range2&& __range2,
-        _OutIter __result,
-        _Comp __comp   = {},
-        _Proj1 __proj1 = {},
-        _Proj2 __proj2 = {}) const {
+  template <input_range _Range1,
+            input_range _Range2,
+            weakly_incrementable _OutIter,
+            class _Comp  = less,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires mergeable<iterator_t<_Range1>, iterator_t<_Range2>, _OutIter, _Comp, _Proj1, _Proj2>
+  _LIBCPP_HIDE_FROM_ABI constexpr set_intersection_result<borrowed_iterator_t<_Range1>,
+                                                          borrowed_iterator_t<_Range2>,
+                                                          _OutIter>
+  operator()(_Range1&& __range1,
+             _Range2&& __range2,
+             _OutIter __result,
+             _Comp __comp   = {},
+             _Proj1 __proj1 = {},
+             _Proj2 __proj2 = {}) const {
     auto __ret = std::__set_intersection<_RangeAlgPolicy>(
         ranges::begin(__range1),
         ranges::end(__range1),
@@ -107,7 +98,7 @@ struct __fn {
 } // namespace __set_intersection
 
 inline namespace __cpo {
-  inline constexpr auto set_intersection = __set_intersection::__fn{};
+inline constexpr auto set_intersection = __set_intersection::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_set_symmetric_difference.h b/libcxx/include/__algorithm/ranges_set_symmetric_difference.h
index f8bcf37954fc9..d8710a1c47b0b 100644
--- a/libcxx/include/__algorithm/ranges_set_symmetric_difference.h
+++ b/libcxx/include/__algorithm/ranges_set_symmetric_difference.h
@@ -40,15 +40,14 @@ using set_symmetric_difference_result = in_in_out_result<_InIter1, _InIter2, _Ou
 namespace __set_symmetric_difference {
 
 struct __fn {
-  template <
-      input_iterator _InIter1,
-      sentinel_for<_InIter1> _Sent1,
-      input_iterator _InIter2,
-      sentinel_for<_InIter2> _Sent2,
-      weakly_incrementable _OutIter,
-      class _Comp  = ranges::less,
-      class _Proj1 = identity,
-      class _Proj2 = identity>
+  template <input_iterator _InIter1,
+            sentinel_for<_InIter1> _Sent1,
+            input_iterator _InIter2,
+            sentinel_for<_InIter2> _Sent2,
+            weakly_incrementable _OutIter,
+            class _Comp  = ranges::less,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
     requires mergeable<_InIter1, _InIter2, _OutIter, _Comp, _Proj1, _Proj2>
   _LIBCPP_HIDE_FROM_ABI constexpr set_symmetric_difference_result<_InIter1, _InIter2, _OutIter> operator()(
       _InIter1 __first1,
@@ -69,30 +68,22 @@ struct __fn {
     return {std::move(__ret.__in1_), std::move(__ret.__in2_), std::move(__ret.__out_)};
   }
 
-  template <
-      input_range _Range1,
-      input_range _Range2,
-      weakly_incrementable _OutIter,
-      class _Comp  = ranges::less,
-      class _Proj1 = identity,
-      class _Proj2 = identity>
-    requires mergeable<
-        iterator_t<_Range1>,
-        iterator_t<_Range2>,
-        _OutIter,
-        _Comp,
-        _Proj1,
-        _Proj2>
+  template <input_range _Range1,
+            input_range _Range2,
+            weakly_incrementable _OutIter,
+            class _Comp  = ranges::less,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires mergeable<iterator_t<_Range1>, iterator_t<_Range2>, _OutIter, _Comp, _Proj1, _Proj2>
   _LIBCPP_HIDE_FROM_ABI constexpr set_symmetric_difference_result<borrowed_iterator_t<_Range1>,
                                                                   borrowed_iterator_t<_Range2>,
                                                                   _OutIter>
-    operator()(
-        _Range1&& __range1,
-        _Range2&& __range2,
-        _OutIter __result,
-        _Comp __comp   = {},
-        _Proj1 __proj1 = {},
-        _Proj2 __proj2 = {}) const {
+  operator()(_Range1&& __range1,
+             _Range2&& __range2,
+             _OutIter __result,
+             _Comp __comp   = {},
+             _Proj1 __proj1 = {},
+             _Proj2 __proj2 = {}) const {
     auto __ret = std::__set_symmetric_difference<_RangeAlgPolicy>(
         ranges::begin(__range1),
         ranges::end(__range1),
@@ -107,7 +98,7 @@ struct __fn {
 } // namespace __set_symmetric_difference
 
 inline namespace __cpo {
-  inline constexpr auto set_symmetric_difference = __set_symmetric_difference::__fn{};
+inline constexpr auto set_symmetric_difference = __set_symmetric_difference::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_set_union.h b/libcxx/include/__algorithm/ranges_set_union.h
index bc669cc83bb73..c627166fffed3 100644
--- a/libcxx/include/__algorithm/ranges_set_union.h
+++ b/libcxx/include/__algorithm/ranges_set_union.h
@@ -43,15 +43,14 @@ using set_union_result = in_in_out_result<_InIter1, _InIter2, _OutIter>;
 namespace __set_union {
 
 struct __fn {
-  template <
-      input_iterator _InIter1,
-      sentinel_for<_InIter1> _Sent1,
-      input_iterator _InIter2,
-      sentinel_for<_InIter2> _Sent2,
-      weakly_incrementable _OutIter,
-      class _Comp  = ranges::less,
-      class _Proj1 = identity,
-      class _Proj2 = identity>
+  template <input_iterator _InIter1,
+            sentinel_for<_InIter1> _Sent1,
+            input_iterator _InIter2,
+            sentinel_for<_InIter2> _Sent2,
+            weakly_incrementable _OutIter,
+            class _Comp  = ranges::less,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
     requires mergeable<_InIter1, _InIter2, _OutIter, _Comp, _Proj1, _Proj2>
   _LIBCPP_HIDE_FROM_ABI constexpr set_union_result<_InIter1, _InIter2, _OutIter> operator()(
       _InIter1 __first1,
@@ -72,30 +71,20 @@ struct __fn {
     return {std::move(__ret.__in1_), std::move(__ret.__in2_), std::move(__ret.__out_)};
   }
 
-  template <
-      input_range _Range1,
-      input_range _Range2,
-      weakly_incrementable _OutIter,
-      class _Comp  = ranges::less,
-      class _Proj1 = identity,
-      class _Proj2 = identity>
-    requires mergeable<
-        iterator_t<_Range1>,
-        iterator_t<_Range2>,
-        _OutIter,
-        _Comp,
-        _Proj1,
-        _Proj2>
-  _LIBCPP_HIDE_FROM_ABI constexpr set_union_result<borrowed_iterator_t<_Range1>,
-                                                   borrowed_iterator_t<_Range2>,
-                                                   _OutIter>
-    operator()(
-        _Range1&& __range1,
-        _Range2&& __range2,
-        _OutIter __result,
-        _Comp __comp   = {},
-        _Proj1 __proj1 = {},
-        _Proj2 __proj2 = {}) const {
+  template <input_range _Range1,
+            input_range _Range2,
+            weakly_incrementable _OutIter,
+            class _Comp  = ranges::less,
+            class _Proj1 = identity,
+            class _Proj2 = identity>
+    requires mergeable<iterator_t<_Range1>, iterator_t<_Range2>, _OutIter, _Comp, _Proj1, _Proj2>
+  _LIBCPP_HIDE_FROM_ABI constexpr set_union_result<borrowed_iterator_t<_Range1>, borrowed_iterator_t<_Range2>, _OutIter>
+  operator()(_Range1&& __range1,
+             _Range2&& __range2,
+             _OutIter __result,
+             _Comp __comp   = {},
+             _Proj1 __proj1 = {},
+             _Proj2 __proj2 = {}) const {
     auto __ret = std::__set_union<_RangeAlgPolicy>(
         ranges::begin(__range1),
         ranges::end(__range1),
@@ -110,7 +99,7 @@ struct __fn {
 } // namespace __set_union
 
 inline namespace __cpo {
-  inline constexpr auto set_union = __set_union::__fn{};
+inline constexpr auto set_union = __set_union::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_shuffle.h b/libcxx/include/__algorithm/ranges_shuffle.h
index 2f45fb00fccc4..fca420058dec0 100644
--- a/libcxx/include/__algorithm/ranges_shuffle.h
+++ b/libcxx/include/__algorithm/ranges_shuffle.h
@@ -39,28 +39,24 @@ namespace ranges {
 namespace __shuffle {
 
 struct __fn {
-
   template <random_access_iterator _Iter, sentinel_for<_Iter> _Sent, class _Gen>
-  requires permutable<_Iter> && uniform_random_bit_generator<remove_reference_t<_Gen>>
-  _LIBCPP_HIDE_FROM_ABI
-  _Iter operator()(_Iter __first, _Sent __last, _Gen&& __gen) const {
+    requires permutable<_Iter> && uniform_random_bit_generator<remove_reference_t<_Gen>>
+  _LIBCPP_HIDE_FROM_ABI _Iter operator()(_Iter __first, _Sent __last, _Gen&& __gen) const {
     _ClassicGenAdaptor<_Gen> __adapted_gen(__gen);
     return std::__shuffle<_RangeAlgPolicy>(std::move(__first), std::move(__last), __adapted_gen);
   }
 
-  template<random_access_range _Range, class _Gen>
-  requires permutable<iterator_t<_Range>> && uniform_random_bit_generator<remove_reference_t<_Gen>>
-  _LIBCPP_HIDE_FROM_ABI
-  borrowed_iterator_t<_Range> operator()(_Range&& __range, _Gen&& __gen) const {
+  template <random_access_range _Range, class _Gen>
+    requires permutable<iterator_t<_Range>> && uniform_random_bit_generator<remove_reference_t<_Gen>>
+  _LIBCPP_HIDE_FROM_ABI borrowed_iterator_t<_Range> operator()(_Range&& __range, _Gen&& __gen) const {
     return (*this)(ranges::begin(__range), ranges::end(__range), std::forward<_Gen>(__gen));
   }
-
 };
 
 } // namespace __shuffle
 
 inline namespace __cpo {
-  inline constexpr auto shuffle = __shuffle::__fn{};
+inline constexpr auto shuffle = __shuffle::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_sort.h b/libcxx/include/__algorithm/ranges_sort.h
index 60305b10abdc7..2ad0e0c233be4 100644
--- a/libcxx/include/__algorithm/ranges_sort.h
+++ b/libcxx/include/__algorithm/ranges_sort.h
@@ -40,8 +40,8 @@ namespace __sort {
 
 struct __fn {
   template <class _Iter, class _Sent, class _Comp, class _Proj>
-  _LIBCPP_HIDE_FROM_ABI constexpr static
-  _Iter __sort_fn_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
+  _LIBCPP_HIDE_FROM_ABI constexpr static _Iter
+  __sort_fn_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
     auto __last_iter = ranges::next(__first, __last);
 
     auto&& __projected_comp = std::__make_projected(__comp, __proj);
@@ -52,15 +52,15 @@ struct __fn {
 
   template <random_access_iterator _Iter, sentinel_for<_Iter> _Sent, class _Comp = ranges::less, class _Proj = identity>
     requires sortable<_Iter, _Comp, _Proj>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  _Iter operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
     return __sort_fn_impl(std::move(__first), std::move(__last), __comp, __proj);
   }
 
   template <random_access_range _Range, class _Comp = ranges::less, class _Proj = identity>
     requires sortable<iterator_t<_Range>, _Comp, _Proj>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_iterator_t<_Range> operator()(_Range&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
     return __sort_fn_impl(ranges::begin(__r), ranges::end(__r), __comp, __proj);
   }
 };
@@ -68,7 +68,7 @@ struct __fn {
 } // namespace __sort
 
 inline namespace __cpo {
-  inline constexpr auto sort = __sort::__fn{};
+inline constexpr auto sort = __sort::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_sort_heap.h b/libcxx/include/__algorithm/ranges_sort_heap.h
index b40eef9761de3..365c7dba61567 100644
--- a/libcxx/include/__algorithm/ranges_sort_heap.h
+++ b/libcxx/include/__algorithm/ranges_sort_heap.h
@@ -41,8 +41,8 @@ namespace __sort_heap {
 
 struct __fn {
   template <class _Iter, class _Sent, class _Comp, class _Proj>
-  _LIBCPP_HIDE_FROM_ABI constexpr static
-  _Iter __sort_heap_fn_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
+  _LIBCPP_HIDE_FROM_ABI constexpr static _Iter
+  __sort_heap_fn_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
     auto __last_iter = ranges::next(__first, __last);
 
     auto&& __projected_comp = std::__make_projected(__comp, __proj);
@@ -53,15 +53,15 @@ struct __fn {
 
   template <random_access_iterator _Iter, sentinel_for<_Iter> _Sent, class _Comp = ranges::less, class _Proj = identity>
     requires sortable<_Iter, _Comp, _Proj>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  _Iter operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
     return __sort_heap_fn_impl(std::move(__first), std::move(__last), __comp, __proj);
   }
 
   template <random_access_range _Range, class _Comp = ranges::less, class _Proj = identity>
     requires sortable<iterator_t<_Range>, _Comp, _Proj>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_iterator_t<_Range> operator()(_Range&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
     return __sort_heap_fn_impl(ranges::begin(__r), ranges::end(__r), __comp, __proj);
   }
 };
@@ -69,7 +69,7 @@ struct __fn {
 } // namespace __sort_heap
 
 inline namespace __cpo {
-  inline constexpr auto sort_heap = __sort_heap::__fn{};
+inline constexpr auto sort_heap = __sort_heap::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_stable_partition.h b/libcxx/include/__algorithm/ranges_stable_partition.h
index 77bef850aaeb8..44937fa589908 100644
--- a/libcxx/include/__algorithm/ranges_stable_partition.h
+++ b/libcxx/include/__algorithm/ranges_stable_partition.h
@@ -42,42 +42,41 @@ namespace ranges {
 namespace __stable_partition {
 
 struct __fn {
-
   template <class _Iter, class _Sent, class _Proj, class _Pred>
-  _LIBCPP_HIDE_FROM_ABI static
-  subrange<__remove_cvref_t<_Iter>> __stable_partition_fn_impl(
-      _Iter&& __first, _Sent&& __last, _Pred&& __pred, _Proj&& __proj) {
+  _LIBCPP_HIDE_FROM_ABI static subrange<__remove_cvref_t<_Iter>>
+  __stable_partition_fn_impl(_Iter&& __first, _Sent&& __last, _Pred&& __pred, _Proj&& __proj) {
     auto __last_iter = ranges::next(__first, __last);
 
     auto&& __projected_pred = std::__make_projected(__pred, __proj);
-    auto __result = std::__stable_partition<_RangeAlgPolicy>(
+    auto __result           = std::__stable_partition<_RangeAlgPolicy>(
         std::move(__first), __last_iter, __projected_pred, __iterator_concept<_Iter>());
 
     return {std::move(__result), std::move(__last_iter)};
   }
 
-  template <bidirectional_iterator _Iter, sentinel_for<_Iter> _Sent, class _Proj = identity,
+  template <bidirectional_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj = identity,
             indirect_unary_predicate<projected<_Iter, _Proj>> _Pred>
-  requires permutable<_Iter>
-  _LIBCPP_HIDE_FROM_ABI
-  subrange<_Iter> operator()(_Iter __first, _Sent __last, _Pred __pred, _Proj __proj = {}) const {
+    requires permutable<_Iter>
+  _LIBCPP_HIDE_FROM_ABI subrange<_Iter> operator()(_Iter __first, _Sent __last, _Pred __pred, _Proj __proj = {}) const {
     return __stable_partition_fn_impl(__first, __last, __pred, __proj);
   }
 
-  template <bidirectional_range _Range, class _Proj = identity,
+  template <bidirectional_range _Range,
+            class _Proj = identity,
             indirect_unary_predicate<projected<iterator_t<_Range>, _Proj>> _Pred>
-  requires permutable<iterator_t<_Range>>
-  _LIBCPP_HIDE_FROM_ABI
-  borrowed_subrange_t<_Range> operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) const {
+    requires permutable<iterator_t<_Range>>
+  _LIBCPP_HIDE_FROM_ABI borrowed_subrange_t<_Range>
+  operator()(_Range&& __range, _Pred __pred, _Proj __proj = {}) const {
     return __stable_partition_fn_impl(ranges::begin(__range), ranges::end(__range), __pred, __proj);
   }
-
 };
 
 } // namespace __stable_partition
 
 inline namespace __cpo {
-  inline constexpr auto stable_partition = __stable_partition::__fn{};
+inline constexpr auto stable_partition = __stable_partition::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_stable_sort.h b/libcxx/include/__algorithm/ranges_stable_sort.h
index b823e3bcb4c77..a4eed3836356d 100644
--- a/libcxx/include/__algorithm/ranges_stable_sort.h
+++ b/libcxx/include/__algorithm/ranges_stable_sort.h
@@ -40,8 +40,7 @@ namespace __stable_sort {
 
 struct __fn {
   template <class _Iter, class _Sent, class _Comp, class _Proj>
-  _LIBCPP_HIDE_FROM_ABI
-  static _Iter __stable_sort_fn_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
+  _LIBCPP_HIDE_FROM_ABI static _Iter __stable_sort_fn_impl(_Iter __first, _Sent __last, _Comp& __comp, _Proj& __proj) {
     auto __last_iter = ranges::next(__first, __last);
 
     auto&& __projected_comp = std::__make_projected(__comp, __proj);
@@ -52,15 +51,14 @@ struct __fn {
 
   template <random_access_iterator _Iter, sentinel_for<_Iter> _Sent, class _Comp = ranges::less, class _Proj = identity>
     requires sortable<_Iter, _Comp, _Proj>
-  _LIBCPP_HIDE_FROM_ABI
-  _Iter operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_HIDE_FROM_ABI _Iter operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
     return __stable_sort_fn_impl(std::move(__first), std::move(__last), __comp, __proj);
   }
 
   template <random_access_range _Range, class _Comp = ranges::less, class _Proj = identity>
     requires sortable<iterator_t<_Range>, _Comp, _Proj>
-  _LIBCPP_HIDE_FROM_ABI
-  borrowed_iterator_t<_Range> operator()(_Range&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_HIDE_FROM_ABI borrowed_iterator_t<_Range>
+  operator()(_Range&& __r, _Comp __comp = {}, _Proj __proj = {}) const {
     return __stable_sort_fn_impl(ranges::begin(__r), ranges::end(__r), __comp, __proj);
   }
 };
@@ -68,7 +66,7 @@ struct __fn {
 } // namespace __stable_sort
 
 inline namespace __cpo {
-  inline constexpr auto stable_sort = __stable_sort::__fn{};
+inline constexpr auto stable_sort = __stable_sort::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_swap_ranges.h b/libcxx/include/__algorithm/ranges_swap_ranges.h
index 34124dbd88de8..1d0ebc0d5221e 100644
--- a/libcxx/include/__algorithm/ranges_swap_ranges.h
+++ b/libcxx/include/__algorithm/ranges_swap_ranges.h
@@ -35,8 +35,7 @@ using swap_ranges_result = in_in_result<_I1, _I2>;
 
 namespace __swap_ranges {
 struct __fn {
-  template <input_iterator _I1, sentinel_for<_I1> _S1,
-            input_iterator _I2, sentinel_for<_I2> _S2>
+  template <input_iterator _I1, sentinel_for<_I1> _S1, input_iterator _I2, sentinel_for<_I2> _S2>
     requires indirectly_swappable<_I1, _I2>
   _LIBCPP_HIDE_FROM_ABI constexpr swap_ranges_result<_I1, _I2>
   operator()(_I1 __first1, _S1 __last1, _I2 __first2, _S2 __last2) const {
@@ -47,17 +46,15 @@ struct __fn {
 
   template <input_range _R1, input_range _R2>
     requires indirectly_swappable<iterator_t<_R1>, iterator_t<_R2>>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  swap_ranges_result<borrowed_iterator_t<_R1>, borrowed_iterator_t<_R2>>
+  _LIBCPP_HIDE_FROM_ABI constexpr swap_ranges_result<borrowed_iterator_t<_R1>, borrowed_iterator_t<_R2>>
   operator()(_R1&& __r1, _R2&& __r2) const {
-    return operator()(ranges::begin(__r1), ranges::end(__r1),
-                      ranges::begin(__r2), ranges::end(__r2));
+    return operator()(ranges::begin(__r1), ranges::end(__r1), ranges::begin(__r2), ranges::end(__r2));
   }
 };
 } // namespace __swap_ranges
 
 inline namespace __cpo {
-  inline constexpr auto swap_ranges = __swap_ranges::__fn{};
+inline constexpr auto swap_ranges = __swap_ranges::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_transform.h b/libcxx/include/__algorithm/ranges_transform.h
index 1bba756fe5bf9..f66a07ac026e5 100644
--- a/libcxx/include/__algorithm/ranges_transform.h
+++ b/libcxx/include/__algorithm/ranges_transform.h
@@ -41,15 +41,9 @@ using binary_transform_result = in_in_out_result<_I1, _I2, _O1>;
 namespace __transform {
 struct __fn {
 private:
-  template <class _InIter, class _Sent,
-            class _OutIter,
-            class _Func,
-            class _Proj>
-  _LIBCPP_HIDE_FROM_ABI static constexpr
-  unary_transform_result<_InIter, _OutIter> __unary(_InIter __first, _Sent __last,
-                                                    _OutIter __result,
-                                                    _Func& __operation,
-                                                    _Proj& __projection) {
+  template <class _InIter, class _Sent, class _OutIter, class _Func, class _Proj>
+  _LIBCPP_HIDE_FROM_ABI static constexpr unary_transform_result<_InIter, _OutIter>
+  __unary(_InIter __first, _Sent __last, _OutIter __result, _Func& __operation, _Proj& __projection) {
     while (__first != __last) {
       *__result = std::invoke(__operation, std::invoke(__projection, *__first));
       ++__first;
@@ -59,76 +53,80 @@ struct __fn {
     return {std::move(__first), std::move(__result)};
   }
 
-  template <class _InIter1, class _Sent1,
-            class _InIter2, class _Sent2,
+  template <class _InIter1,
+            class _Sent1,
+            class _InIter2,
+            class _Sent2,
             class _OutIter,
             class _Func,
             class _Proj1,
             class _Proj2>
   _LIBCPP_HIDE_FROM_ABI static constexpr binary_transform_result<_InIter1, _InIter2, _OutIter>
-  __binary(_InIter1 __first1, _Sent1 __last1,
-           _InIter2 __first2, _Sent2 __last2,
+  __binary(_InIter1 __first1,
+           _Sent1 __last1,
+           _InIter2 __first2,
+           _Sent2 __last2,
            _OutIter __result,
            _Func& __binary_operation,
            _Proj1& __projection1,
            _Proj2& __projection2) {
     while (__first1 != __last1 && __first2 != __last2) {
-      *__result = std::invoke(__binary_operation, std::invoke(__projection1, *__first1),
-                                                  std::invoke(__projection2, *__first2));
+      *__result =
+          std::invoke(__binary_operation, std::invoke(__projection1, *__first1), std::invoke(__projection2, *__first2));
       ++__first1;
       ++__first2;
       ++__result;
     }
     return {std::move(__first1), std::move(__first2), std::move(__result)};
   }
+
 public:
-  template <input_iterator _InIter, sentinel_for<_InIter> _Sent,
+  template <input_iterator _InIter,
+            sentinel_for<_InIter> _Sent,
             weakly_incrementable _OutIter,
             copy_constructible _Func,
             class _Proj = identity>
     requires indirectly_writable<_OutIter, indirect_result_t<_Func&, projected<_InIter, _Proj>>>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  unary_transform_result<_InIter, _OutIter> operator()(_InIter __first, _Sent __last,
-                                                       _OutIter __result,
-                                                       _Func __operation,
-                                                       _Proj __proj = {}) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr unary_transform_result<_InIter, _OutIter>
+  operator()(_InIter __first, _Sent __last, _OutIter __result, _Func __operation, _Proj __proj = {}) const {
     return __unary(std::move(__first), std::move(__last), std::move(__result), __operation, __proj);
   }
 
-  template <input_range _Range,
-            weakly_incrementable _OutIter,
-            copy_constructible _Func,
-            class _Proj = identity>
+  template <input_range _Range, weakly_incrementable _OutIter, copy_constructible _Func, class _Proj = identity>
     requires indirectly_writable<_OutIter, indirect_result_t<_Func, projected<iterator_t<_Range>, _Proj>>>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  unary_transform_result<borrowed_iterator_t<_Range>, _OutIter> operator()(_Range&& __range,
-                                                                           _OutIter __result,
-                                                                           _Func __operation,
-                                                                           _Proj __projection = {}) const {
+  _LIBCPP_HIDE_FROM_ABI constexpr unary_transform_result<borrowed_iterator_t<_Range>, _OutIter>
+  operator()(_Range&& __range, _OutIter __result, _Func __operation, _Proj __projection = {}) const {
     return __unary(ranges::begin(__range), ranges::end(__range), std::move(__result), __operation, __projection);
   }
 
-  template <input_iterator _InIter1, sentinel_for<_InIter1> _Sent1,
-            input_iterator _InIter2, sentinel_for<_InIter2> _Sent2,
+  template <input_iterator _InIter1,
+            sentinel_for<_InIter1> _Sent1,
+            input_iterator _InIter2,
+            sentinel_for<_InIter2> _Sent2,
             weakly_incrementable _OutIter,
             copy_constructible _Func,
             class _Proj1 = identity,
             class _Proj2 = identity>
-    requires indirectly_writable<_OutIter, indirect_result_t<_Func&, projected<_InIter1, _Proj1>,
-                                                                     projected<_InIter2, _Proj2>>>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  binary_transform_result<_InIter1, _InIter2, _OutIter> operator()(_InIter1 __first1, _Sent1 __last1,
-                                                                   _InIter2 __first2, _Sent2 __last2,
-                                                                   _OutIter __result,
-                                                                   _Func __binary_operation,
-                                                                   _Proj1 __projection1 = {},
-                                                                   _Proj2 __projection2 = {}) const {
-    return __binary(std::move(__first1), std::move(__last1),
-                    std::move(__first2), std::move(__last2),
-                    std::move(__result),
-                    __binary_operation,
-                    __projection1,
-                    __projection2);
+    requires indirectly_writable<_OutIter,
+                                 indirect_result_t<_Func&, projected<_InIter1, _Proj1>, projected<_InIter2, _Proj2>>>
+  _LIBCPP_HIDE_FROM_ABI constexpr binary_transform_result<_InIter1, _InIter2, _OutIter> operator()(
+      _InIter1 __first1,
+      _Sent1 __last1,
+      _InIter2 __first2,
+      _Sent2 __last2,
+      _OutIter __result,
+      _Func __binary_operation,
+      _Proj1 __projection1 = {},
+      _Proj2 __projection2 = {}) const {
+    return __binary(
+        std::move(__first1),
+        std::move(__last1),
+        std::move(__first2),
+        std::move(__last2),
+        std::move(__result),
+        __binary_operation,
+        __projection1,
+        __projection2);
   }
 
   template <input_range _Range1,
@@ -137,29 +135,33 @@ struct __fn {
             copy_constructible _Func,
             class _Proj1 = identity,
             class _Proj2 = identity>
-    requires indirectly_writable<_OutIter, indirect_result_t<_Func&, projected<iterator_t<_Range1>, _Proj1>,
-                                                                     projected<iterator_t<_Range2>, _Proj2>>>
-  _LIBCPP_HIDE_FROM_ABI constexpr
-  binary_transform_result<borrowed_iterator_t<_Range1>, borrowed_iterator_t<_Range2>, _OutIter>
+    requires indirectly_writable<
+        _OutIter,
+        indirect_result_t<_Func&, projected<iterator_t<_Range1>, _Proj1>, projected<iterator_t<_Range2>, _Proj2>>>
+  _LIBCPP_HIDE_FROM_ABI constexpr binary_transform_result<borrowed_iterator_t<_Range1>,
+                                                          borrowed_iterator_t<_Range2>,
+                                                          _OutIter>
   operator()(_Range1&& __range1,
              _Range2&& __range2,
              _OutIter __result,
              _Func __binary_operation,
              _Proj1 __projection1 = {},
              _Proj2 __projection2 = {}) const {
-    return __binary(ranges::begin(__range1), ranges::end(__range1),
-                    ranges::begin(__range2), ranges::end(__range2),
-                    std::move(__result),
-                    __binary_operation,
-                    __projection1,
-                    __projection2);
+    return __binary(
+        ranges::begin(__range1),
+        ranges::end(__range1),
+        ranges::begin(__range2),
+        ranges::end(__range2),
+        std::move(__result),
+        __binary_operation,
+        __projection1,
+        __projection2);
   }
-
 };
 } // namespace __transform
 
 inline namespace __cpo {
-  inline constexpr auto transform = __transform::__fn{};
+inline constexpr auto transform = __transform::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_unique.h b/libcxx/include/__algorithm/ranges_unique.h
index 9e05c09fe5ee8..b17e01fc50577 100644
--- a/libcxx/include/__algorithm/ranges_unique.h
+++ b/libcxx/include/__algorithm/ranges_unique.h
@@ -39,36 +39,34 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 namespace __unique {
 
-  struct __fn {
-    template <
-        permutable _Iter,
-        sentinel_for<_Iter> _Sent,
-        class _Proj                                                  = identity,
-        indirect_equivalence_relation<projected<_Iter, _Proj>> _Comp = ranges::equal_to>
-    _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr subrange<_Iter>
-    operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
-      auto __ret = std::__unique<_RangeAlgPolicy>(
-          std::move(__first), std::move(__last), std::__make_projected(__comp, __proj));
-      return {std::move(__ret.first), std::move(__ret.second)};
-    }
+struct __fn {
+  template <permutable _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Proj                                                  = identity,
+            indirect_equivalence_relation<projected<_Iter, _Proj>> _Comp = ranges::equal_to>
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr subrange<_Iter>
+  operator()(_Iter __first, _Sent __last, _Comp __comp = {}, _Proj __proj = {}) const {
+    auto __ret =
+        std::__unique<_RangeAlgPolicy>(std::move(__first), std::move(__last), std::__make_projected(__comp, __proj));
+    return {std::move(__ret.first), std::move(__ret.second)};
+  }
 
-    template <
-        forward_range _Range,
-        class _Proj                                                               = identity,
-        indirect_equivalence_relation<projected<iterator_t<_Range>, _Proj>> _Comp = ranges::equal_to>
-      requires permutable<iterator_t<_Range>>
-    _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr borrowed_subrange_t<_Range>
-    operator()(_Range&& __range, _Comp __comp = {}, _Proj __proj = {}) const {
-      auto __ret = std::__unique<_RangeAlgPolicy>(
-          ranges::begin(__range), ranges::end(__range), std::__make_projected(__comp, __proj));
-      return {std::move(__ret.first), std::move(__ret.second)};
-    }
-  };
+  template <forward_range _Range,
+            class _Proj                                                               = identity,
+            indirect_equivalence_relation<projected<iterator_t<_Range>, _Proj>> _Comp = ranges::equal_to>
+    requires permutable<iterator_t<_Range>>
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr borrowed_subrange_t<_Range>
+  operator()(_Range&& __range, _Comp __comp = {}, _Proj __proj = {}) const {
+    auto __ret = std::__unique<_RangeAlgPolicy>(
+        ranges::begin(__range), ranges::end(__range), std::__make_projected(__comp, __proj));
+    return {std::move(__ret.first), std::move(__ret.second)};
+  }
+};
 
 } // namespace __unique
 
 inline namespace __cpo {
-  inline constexpr auto unique = __unique::__fn{};
+inline constexpr auto unique = __unique::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__algorithm/ranges_unique_copy.h b/libcxx/include/__algorithm/ranges_unique_copy.h
index 2aaa879ea0f06..803ddca37028e 100644
--- a/libcxx/include/__algorithm/ranges_unique_copy.h
+++ b/libcxx/include/__algorithm/ranges_unique_copy.h
@@ -87,9 +87,9 @@ struct __fn {
             class _Proj                                                               = identity,
             indirect_equivalence_relation<projected<iterator_t<_Range>, _Proj>> _Comp = ranges::equal_to>
     requires indirectly_copyable<iterator_t<_Range>, _OutIter> &&
-      (forward_iterator<iterator_t<_Range>> ||
-       (input_iterator<_OutIter> && same_as<range_value_t<_Range>, iter_value_t<_OutIter>>) ||
-       indirectly_copyable_storable<iterator_t<_Range>, _OutIter>)
+             (forward_iterator<iterator_t<_Range>> ||
+              (input_iterator<_OutIter> && same_as<range_value_t<_Range>, iter_value_t<_OutIter>>) ||
+              indirectly_copyable_storable<iterator_t<_Range>, _OutIter>)
   _LIBCPP_HIDE_FROM_ABI constexpr unique_copy_result<borrowed_iterator_t<_Range>, _OutIter>
   operator()(_Range&& __range, _OutIter __result, _Comp __comp = {}, _Proj __proj = {}) const {
     auto __ret = std::__unique_copy<_RangeAlgPolicy>(
diff --git a/libcxx/include/__algorithm/ranges_upper_bound.h b/libcxx/include/__algorithm/ranges_upper_bound.h
index 6e6f18dbc510b..a12a0e39b0849 100644
--- a/libcxx/include/__algorithm/ranges_upper_bound.h
+++ b/libcxx/include/__algorithm/ranges_upper_bound.h
@@ -32,10 +32,13 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 namespace __upper_bound {
 struct __fn {
-  template <forward_iterator _Iter, sentinel_for<_Iter> _Sent, class _Type, class _Proj = identity,
+  template <forward_iterator _Iter,
+            sentinel_for<_Iter> _Sent,
+            class _Type,
+            class _Proj                                                             = identity,
             indirect_strict_weak_order<const _Type*, projected<_Iter, _Proj>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  _Iter operator()(_Iter __first, _Sent __last, const _Type& __value, _Comp __comp = {}, _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr _Iter
+  operator()(_Iter __first, _Sent __last, const _Type& __value, _Comp __comp = {}, _Proj __proj = {}) const {
     auto __comp_lhs_rhs_swapped = [&](const auto& __lhs, const auto& __rhs) {
       return !std::invoke(__comp, __rhs, __lhs);
     };
@@ -43,28 +46,24 @@ struct __fn {
     return std::__lower_bound<_RangeAlgPolicy>(__first, __last, __value, __comp_lhs_rhs_swapped, __proj);
   }
 
-  template <forward_range _Range, class _Type, class _Proj = identity,
+  template <forward_range _Range,
+            class _Type,
+            class _Proj                                                                          = identity,
             indirect_strict_weak_order<const _Type*, projected<iterator_t<_Range>, _Proj>> _Comp = ranges::less>
-  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr
-  borrowed_iterator_t<_Range> operator()(_Range&& __r,
-                                         const _Type& __value,
-                                         _Comp __comp = {},
-                                         _Proj __proj = {}) const {
+  _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI constexpr borrowed_iterator_t<_Range>
+  operator()(_Range&& __r, const _Type& __value, _Comp __comp = {}, _Proj __proj = {}) const {
     auto __comp_lhs_rhs_swapped = [&](const auto& __lhs, const auto& __rhs) {
       return !std::invoke(__comp, __rhs, __lhs);
     };
 
-    return std::__lower_bound<_RangeAlgPolicy>(ranges::begin(__r),
-                                               ranges::end(__r),
-                                               __value,
-                                               __comp_lhs_rhs_swapped,
-                                               __proj);
+    return std::__lower_bound<_RangeAlgPolicy>(
+        ranges::begin(__r), ranges::end(__r), __value, __comp_lhs_rhs_swapped, __proj);
   }
 };
 } // namespace __upper_bound
 
 inline namespace __cpo {
-  inline constexpr auto upper_bound = __upper_bound::__fn{};
+inline constexpr auto upper_bound = __upper_bound::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
diff --git a/libcxx/include/__concepts/arithmetic.h b/libcxx/include/__concepts/arithmetic.h
index 91a0b184b18e3..f41e4c9f2747c 100644
--- a/libcxx/include/__concepts/arithmetic.h
+++ b/libcxx/include/__concepts/arithmetic.h
@@ -26,16 +26,16 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 // [concepts.arithmetic], arithmetic concepts
 
-template<class _Tp>
+template <class _Tp>
 concept integral = is_integral_v<_Tp>;
 
-template<class _Tp>
+template <class _Tp>
 concept signed_integral = integral<_Tp> && is_signed_v<_Tp>;
 
-template<class _Tp>
+template <class _Tp>
 concept unsigned_integral = integral<_Tp> && !signed_integral<_Tp>;
 
-template<class _Tp>
+template <class _Tp>
 concept floating_point = is_floating_point_v<_Tp>;
 
 // Concept helpers for the internal type traits for the fundamental types.
diff --git a/libcxx/include/__concepts/assignable.h b/libcxx/include/__concepts/assignable.h
index 2dabae5766c09..3f7544c934066 100644
--- a/libcxx/include/__concepts/assignable.h
+++ b/libcxx/include/__concepts/assignable.h
@@ -26,13 +26,13 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 // [concept.assignable]
 
-template<class _Lhs, class _Rhs>
+template <class _Lhs, class _Rhs>
 concept assignable_from =
-  is_lvalue_reference_v<_Lhs> &&
-  common_reference_with<__make_const_lvalue_ref<_Lhs>, __make_const_lvalue_ref<_Rhs>> &&
-  requires (_Lhs __lhs, _Rhs&& __rhs) {
-    { __lhs = _VSTD::forward<_Rhs>(__rhs) } -> same_as<_Lhs>;
-  };
+    is_lvalue_reference_v<_Lhs> &&
+    common_reference_with<__make_const_lvalue_ref<_Lhs>, __make_const_lvalue_ref<_Rhs>> &&
+    requires(_Lhs __lhs, _Rhs&& __rhs) {
+      { __lhs = _VSTD::forward<_Rhs>(__rhs) } -> same_as<_Lhs>;
+    };
 
 #endif // _LIBCPP_STD_VER >= 20
 
diff --git a/libcxx/include/__concepts/boolean_testable.h b/libcxx/include/__concepts/boolean_testable.h
index 8efb6e5ff46f5..d290e1cf3d836 100644
--- a/libcxx/include/__concepts/boolean_testable.h
+++ b/libcxx/include/__concepts/boolean_testable.h
@@ -23,10 +23,10 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 // [concepts.booleantestable]
 
-template<class _Tp>
+template <class _Tp>
 concept __boolean_testable_impl = convertible_to<_Tp, bool>;
 
-template<class _Tp>
+template <class _Tp>
 concept __boolean_testable = __boolean_testable_impl<_Tp> && requires(_Tp&& __t) {
   { !_VSTD::forward<_Tp>(__t) } -> __boolean_testable_impl;
 };
diff --git a/libcxx/include/__concepts/class_or_enum.h b/libcxx/include/__concepts/class_or_enum.h
index 04c24bd98ff1e..c1b4a8c258f3a 100644
--- a/libcxx/include/__concepts/class_or_enum.h
+++ b/libcxx/include/__concepts/class_or_enum.h
@@ -25,12 +25,12 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 // Whether a type is a class type or enumeration type according to the Core wording.
 
-template<class _Tp>
+template <class _Tp>
 concept __class_or_enum = is_class_v<_Tp> || is_union_v<_Tp> || is_enum_v<_Tp>;
 
 // Work around Clang bug https://llvm.org/PR52970
 // TODO: remove this workaround once libc++ no longer has to support Clang 13 (it was fixed in Clang 14).
-template<class _Tp>
+template <class _Tp>
 concept __workaround_52970 = is_class_v<__remove_cvref_t<_Tp>> || is_union_v<__remove_cvref_t<_Tp>>;
 
 #endif // _LIBCPP_STD_VER >= 20
diff --git a/libcxx/include/__concepts/common_reference_with.h b/libcxx/include/__concepts/common_reference_with.h
index 6ad0db22431e0..4eb687e071bc5 100644
--- a/libcxx/include/__concepts/common_reference_with.h
+++ b/libcxx/include/__concepts/common_reference_with.h
@@ -24,11 +24,10 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 // [concept.commonref]
 
-template<class _Tp, class _Up>
+template <class _Tp, class _Up>
 concept common_reference_with =
-  same_as<common_reference_t<_Tp, _Up>, common_reference_t<_Up, _Tp>> &&
-  convertible_to<_Tp, common_reference_t<_Tp, _Up>> &&
-  convertible_to<_Up, common_reference_t<_Tp, _Up>>;
+    same_as<common_reference_t<_Tp, _Up>, common_reference_t<_Up, _Tp>> &&
+    convertible_to<_Tp, common_reference_t<_Tp, _Up>> && convertible_to<_Up, common_reference_t<_Tp, _Up>>;
 
 #endif // _LIBCPP_STD_VER >= 20
 
diff --git a/libcxx/include/__concepts/common_with.h b/libcxx/include/__concepts/common_with.h
index e159bcce968d8..85abb05efbc29 100644
--- a/libcxx/include/__concepts/common_with.h
+++ b/libcxx/include/__concepts/common_with.h
@@ -27,21 +27,23 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 // [concept.common]
 
-template<class _Tp, class _Up>
+// clang-format off
+template <class _Tp, class _Up>
 concept common_with =
-  same_as<common_type_t<_Tp, _Up>, common_type_t<_Up, _Tp>> &&
-  requires {
-    static_cast<common_type_t<_Tp, _Up>>(std::declval<_Tp>());
-    static_cast<common_type_t<_Tp, _Up>>(std::declval<_Up>());
-  } &&
-  common_reference_with<
-    add_lvalue_reference_t<const _Tp>,
-    add_lvalue_reference_t<const _Up>> &&
-  common_reference_with<
-    add_lvalue_reference_t<common_type_t<_Tp, _Up>>,
-    common_reference_t<
-      add_lvalue_reference_t<const _Tp>,
-      add_lvalue_reference_t<const _Up>>>;
+    same_as<common_type_t<_Tp, _Up>, common_type_t<_Up, _Tp>> &&
+    requires {
+        static_cast<common_type_t<_Tp, _Up>>(std::declval<_Tp>());
+        static_cast<common_type_t<_Tp, _Up>>(std::declval<_Up>());
+    } &&
+    common_reference_with<
+        add_lvalue_reference_t<const _Tp>,
+        add_lvalue_reference_t<const _Up>> &&
+    common_reference_with<
+        add_lvalue_reference_t<common_type_t<_Tp, _Up>>,
+        common_reference_t<
+            add_lvalue_reference_t<const _Tp>,
+            add_lvalue_reference_t<const _Up>>>;
+// clang-format on
 
 #endif // _LIBCPP_STD_VER >= 20
 
diff --git a/libcxx/include/__concepts/constructible.h b/libcxx/include/__concepts/constructible.h
index 6e3862c51ffcb..835a44429c092 100644
--- a/libcxx/include/__concepts/constructible.h
+++ b/libcxx/include/__concepts/constructible.h
@@ -23,31 +23,30 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 #if _LIBCPP_STD_VER >= 20
 
 // [concept.constructible]
-template<class _Tp, class... _Args>
-concept constructible_from =
-    destructible<_Tp> && is_constructible_v<_Tp, _Args...>;
+template <class _Tp, class... _Args>
+concept constructible_from = destructible<_Tp> && is_constructible_v<_Tp, _Args...>;
 
 // [concept.default.init]
 
-template<class _Tp>
+template <class _Tp>
 concept __default_initializable = requires { ::new _Tp; };
 
-template<class _Tp>
-concept default_initializable = constructible_from<_Tp> &&
-    requires { _Tp{}; } && __default_initializable<_Tp>;
+template <class _Tp>
+concept default_initializable = constructible_from<_Tp> && requires { _Tp{}; } && __default_initializable<_Tp>;
 
 // [concept.moveconstructible]
-template<class _Tp>
-concept move_constructible =
-  constructible_from<_Tp, _Tp> && convertible_to<_Tp, _Tp>;
+template <class _Tp>
+concept move_constructible = constructible_from<_Tp, _Tp> && convertible_to<_Tp, _Tp>;
 
 // [concept.copyconstructible]
-template<class _Tp>
+// clang-format off
+template <class _Tp>
 concept copy_constructible =
-  move_constructible<_Tp> &&
-  constructible_from<_Tp, _Tp&> && convertible_to<_Tp&, _Tp> &&
-  constructible_from<_Tp, const _Tp&> && convertible_to<const _Tp&, _Tp> &&
-  constructible_from<_Tp, const _Tp> && convertible_to<const _Tp, _Tp>;
+    move_constructible<_Tp> &&
+    constructible_from<_Tp, _Tp&> && convertible_to<_Tp&, _Tp> &&
+    constructible_from<_Tp, const _Tp&> && convertible_to<const _Tp&, _Tp> &&
+    constructible_from<_Tp, const _Tp> && convertible_to<const _Tp, _Tp>;
+// clang-format on
 
 #endif // _LIBCPP_STD_VER >= 20
 
diff --git a/libcxx/include/__concepts/convertible_to.h b/libcxx/include/__concepts/convertible_to.h
index 20ee31bc058cd..6d5b6c1268d5d 100644
--- a/libcxx/include/__concepts/convertible_to.h
+++ b/libcxx/include/__concepts/convertible_to.h
@@ -23,12 +23,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 // [concept.convertible]
 
-template<class _From, class _To>
-concept convertible_to =
-  is_convertible_v<_From, _To> &&
-  requires {
-    static_cast<_To>(std::declval<_From>());
-  };
+template <class _From, class _To>
+concept convertible_to = is_convertible_v<_From, _To> && requires { static_cast<_To>(std::declval<_From>()); };
 
 #endif // _LIBCPP_STD_VER >= 20
 
diff --git a/libcxx/include/__concepts/copyable.h b/libcxx/include/__concepts/copyable.h
index 0d6dd50cfbf09..2bf0ad42fc1a8 100644
--- a/libcxx/include/__concepts/copyable.h
+++ b/libcxx/include/__concepts/copyable.h
@@ -24,13 +24,15 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 // [concepts.object]
 
-template<class _Tp>
+// clang-format off
+template <class _Tp>
 concept copyable =
-  copy_constructible<_Tp> &&
-  movable<_Tp> &&
-  assignable_from<_Tp&, _Tp&> &&
-  assignable_from<_Tp&, const _Tp&> &&
-  assignable_from<_Tp&, const _Tp>;
+    copy_constructible<_Tp> &&
+    movable<_Tp> &&
+    assignable_from<_Tp&, _Tp&> &&
+    assignable_from<_Tp&, const _Tp&> &&
+    assignable_from<_Tp&, const _Tp>;
+// clang-format on
 
 #endif // _LIBCPP_STD_VER >= 20
 
diff --git a/libcxx/include/__concepts/derived_from.h b/libcxx/include/__concepts/derived_from.h
index 1cbe458e2fa6f..9875faee81b90 100644
--- a/libcxx/include/__concepts/derived_from.h
+++ b/libcxx/include/__concepts/derived_from.h
@@ -23,10 +23,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 // [concept.derived]
 
-template<class _Dp, class _Bp>
-concept derived_from =
-  is_base_of_v<_Bp, _Dp> &&
-  is_convertible_v<const volatile _Dp*, const volatile _Bp*>;
+template <class _Dp, class _Bp>
+concept derived_from = is_base_of_v<_Bp, _Dp> && is_convertible_v<const volatile _Dp*, const volatile _Bp*>;
 
 #endif // _LIBCPP_STD_VER >= 20
 
diff --git a/libcxx/include/__concepts/destructible.h b/libcxx/include/__concepts/destructible.h
index 8da9c378b76a0..28b4b1bc24ec9 100644
--- a/libcxx/include/__concepts/destructible.h
+++ b/libcxx/include/__concepts/destructible.h
@@ -22,7 +22,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 // [concept.destructible]
 
-template<class _Tp>
+template <class _Tp>
 concept destructible = is_nothrow_destructible_v<_Tp>;
 
 #endif // _LIBCPP_STD_VER >= 20
diff --git a/libcxx/include/__concepts/different_from.h b/libcxx/include/__concepts/different_from.h
index 5ef1467acfd56..fd31f6e25805d 100644
--- a/libcxx/include/__concepts/different_from.h
+++ b/libcxx/include/__concepts/different_from.h
@@ -21,7 +21,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if _LIBCPP_STD_VER >= 20
 
-template<class _Tp, class _Up>
+template <class _Tp, class _Up>
 concept __different_from = !same_as<remove_cvref_t<_Tp>, remove_cvref_t<_Up>>;
 
 #endif // _LIBCPP_STD_VER >= 20
diff --git a/libcxx/include/__concepts/equality_comparable.h b/libcxx/include/__concepts/equality_comparable.h
index f1062884e34be..278fc76409289 100644
--- a/libcxx/include/__concepts/equality_comparable.h
+++ b/libcxx/include/__concepts/equality_comparable.h
@@ -25,27 +25,29 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 // [concept.equalitycomparable]
 
-template<class _Tp, class _Up>
+template <class _Tp, class _Up>
 concept __weakly_equality_comparable_with =
-  requires(__make_const_lvalue_ref<_Tp> __t, __make_const_lvalue_ref<_Up> __u) {
-    { __t == __u } -> __boolean_testable;
-    { __t != __u } -> __boolean_testable;
-    { __u == __t } -> __boolean_testable;
-    { __u != __t } -> __boolean_testable;
-  };
-
-template<class _Tp>
+    requires(__make_const_lvalue_ref<_Tp> __t, __make_const_lvalue_ref<_Up> __u) {
+      { __t == __u } -> __boolean_testable;
+      { __t != __u } -> __boolean_testable;
+      { __u == __t } -> __boolean_testable;
+      { __u != __t } -> __boolean_testable;
+    };
+
+template <class _Tp>
 concept equality_comparable = __weakly_equality_comparable_with<_Tp, _Tp>;
 
-template<class _Tp, class _Up>
+// clang-format off
+template <class _Tp, class _Up>
 concept equality_comparable_with =
-  equality_comparable<_Tp> && equality_comparable<_Up> &&
-  common_reference_with<__make_const_lvalue_ref<_Tp>, __make_const_lvalue_ref<_Up>> &&
-  equality_comparable<
-    common_reference_t<
-      __make_const_lvalue_ref<_Tp>,
-      __make_const_lvalue_ref<_Up>>> &&
-  __weakly_equality_comparable_with<_Tp, _Up>;
+    equality_comparable<_Tp> && equality_comparable<_Up> &&
+    common_reference_with<__make_const_lvalue_ref<_Tp>, __make_const_lvalue_ref<_Up>> &&
+    equality_comparable<
+        common_reference_t<
+            __make_const_lvalue_ref<_Tp>,
+            __make_const_lvalue_ref<_Up>>> &&
+    __weakly_equality_comparable_with<_Tp, _Up>;
+// clang-format on
 
 #endif // _LIBCPP_STD_VER >= 20
 
diff --git a/libcxx/include/__concepts/invocable.h b/libcxx/include/__concepts/invocable.h
index 59eab01f8d37c..e5be514b5c38e 100644
--- a/libcxx/include/__concepts/invocable.h
+++ b/libcxx/include/__concepts/invocable.h
@@ -23,14 +23,14 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 // [concept.invocable]
 
-template<class _Fn, class... _Args>
+template <class _Fn, class... _Args>
 concept invocable = requires(_Fn&& __fn, _Args&&... __args) {
   _VSTD::invoke(_VSTD::forward<_Fn>(__fn), _VSTD::forward<_Args>(__args)...); // not required to be equality preserving
 };
 
 // [concept.regular.invocable]
 
-template<class _Fn, class... _Args>
+template <class _Fn, class... _Args>
 concept regular_invocable = invocable<_Fn, _Args...>;
 
 #endif // _LIBCPP_STD_VER >= 20
diff --git a/libcxx/include/__concepts/movable.h b/libcxx/include/__concepts/movable.h
index f37d49f04eb5e..bc5b9d767c6a5 100644
--- a/libcxx/include/__concepts/movable.h
+++ b/libcxx/include/__concepts/movable.h
@@ -25,12 +25,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 // [concepts.object]
 
-template<class _Tp>
-concept movable =
-  is_object_v<_Tp> &&
-  move_constructible<_Tp> &&
-  assignable_from<_Tp&, _Tp> &&
-  swappable<_Tp>;
+template <class _Tp>
+concept movable = is_object_v<_Tp> && move_constructible<_Tp> && assignable_from<_Tp&, _Tp> && swappable<_Tp>;
 
 #endif // _LIBCPP_STD_VER >= 20
 
diff --git a/libcxx/include/__concepts/predicate.h b/libcxx/include/__concepts/predicate.h
index b09183c5ca8eb..00731efc8fcd9 100644
--- a/libcxx/include/__concepts/predicate.h
+++ b/libcxx/include/__concepts/predicate.h
@@ -24,9 +24,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 // [concept.predicate]
 
-template<class _Fn, class... _Args>
-concept predicate =
-  regular_invocable<_Fn, _Args...> && __boolean_testable<invoke_result_t<_Fn, _Args...>>;
+template <class _Fn, class... _Args>
+concept predicate = regular_invocable<_Fn, _Args...> && __boolean_testable<invoke_result_t<_Fn, _Args...>>;
 
 #endif // _LIBCPP_STD_VER >= 20
 
diff --git a/libcxx/include/__concepts/regular.h b/libcxx/include/__concepts/regular.h
index 93fb7016cf4ed..9f3d8bf30be3e 100644
--- a/libcxx/include/__concepts/regular.h
+++ b/libcxx/include/__concepts/regular.h
@@ -23,7 +23,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 // [concept.object]
 
-template<class _Tp>
+template <class _Tp>
 concept regular = semiregular<_Tp> && equality_comparable<_Tp>;
 
 #endif // _LIBCPP_STD_VER >= 20
diff --git a/libcxx/include/__concepts/relation.h b/libcxx/include/__concepts/relation.h
index 218afef21a382..7545a7db93da7 100644
--- a/libcxx/include/__concepts/relation.h
+++ b/libcxx/include/__concepts/relation.h
@@ -22,19 +22,18 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 // [concept.relation]
 
-template<class _Rp, class _Tp, class _Up>
+template <class _Rp, class _Tp, class _Up>
 concept relation =
-  predicate<_Rp, _Tp, _Tp> && predicate<_Rp, _Up, _Up> &&
-  predicate<_Rp, _Tp, _Up> && predicate<_Rp, _Up, _Tp>;
+    predicate<_Rp, _Tp, _Tp> && predicate<_Rp, _Up, _Up> && predicate<_Rp, _Tp, _Up> && predicate<_Rp, _Up, _Tp>;
 
 // [concept.equiv]
 
-template<class _Rp, class _Tp, class _Up>
+template <class _Rp, class _Tp, class _Up>
 concept equivalence_relation = relation<_Rp, _Tp, _Up>;
 
 // [concept.strictweakorder]
 
-template<class _Rp, class _Tp, class _Up>
+template <class _Rp, class _Tp, class _Up>
 concept strict_weak_order = relation<_Rp, _Tp, _Up>;
 
 #endif // _LIBCPP_STD_VER >= 20
diff --git a/libcxx/include/__concepts/same_as.h b/libcxx/include/__concepts/same_as.h
index b86cadaa1edcc..4241131c70c1f 100644
--- a/libcxx/include/__concepts/same_as.h
+++ b/libcxx/include/__concepts/same_as.h
@@ -22,10 +22,10 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 // [concept.same]
 
-template<class _Tp, class _Up>
+template <class _Tp, class _Up>
 concept __same_as_impl = _IsSame<_Tp, _Up>::value;
 
-template<class _Tp, class _Up>
+template <class _Tp, class _Up>
 concept same_as = __same_as_impl<_Tp, _Up> && __same_as_impl<_Up, _Tp>;
 
 #endif // _LIBCPP_STD_VER >= 20
diff --git a/libcxx/include/__concepts/semiregular.h b/libcxx/include/__concepts/semiregular.h
index ae2f3c6694006..7a159d17dfc10 100644
--- a/libcxx/include/__concepts/semiregular.h
+++ b/libcxx/include/__concepts/semiregular.h
@@ -23,7 +23,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 // [concept.object]
 
-template<class _Tp>
+template <class _Tp>
 concept semiregular = copyable<_Tp> && default_initializable<_Tp>;
 
 #endif // _LIBCPP_STD_VER >= 20
diff --git a/libcxx/include/__concepts/swappable.h b/libcxx/include/__concepts/swappable.h
index 859255dec9aac..c611bbb4eebde 100644
--- a/libcxx/include/__concepts/swappable.h
+++ b/libcxx/include/__concepts/swappable.h
@@ -37,82 +37,79 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 namespace ranges {
 namespace __swap {
 
-  template<class _Tp>
-  void swap(_Tp&, _Tp&) = delete;
+template <class _Tp>
+void swap(_Tp&, _Tp&) = delete;
 
-  template<class _Tp, class _Up>
-  concept __unqualified_swappable_with =
+// clang-format off
+template <class _Tp, class _Up>
+concept __unqualified_swappable_with =
     (__class_or_enum<remove_cvref_t<_Tp>> || __class_or_enum<remove_cvref_t<_Up>>) &&
     requires(_Tp&& __t, _Up&& __u) {
-      swap(_VSTD::forward<_Tp>(__t), _VSTD::forward<_Up>(__u));
+        swap(_VSTD::forward<_Tp>(__t), _VSTD::forward<_Up>(__u));
     };
+// clang-format on
 
-  struct __fn;
+struct __fn;
 
-  template<class _Tp, class _Up, size_t _Size>
-  concept __swappable_arrays =
-    !__unqualified_swappable_with<_Tp(&)[_Size], _Up(&)[_Size]> &&
+// clang-format off
+template <class _Tp, class _Up, size_t _Size>
+concept __swappable_arrays =
+    !__unqualified_swappable_with<_Tp (&)[_Size], _Up (&)[_Size]> &&
     extent_v<_Tp> == extent_v<_Up> &&
-    requires(_Tp(& __t)[_Size], _Up(& __u)[_Size], const __fn& __swap) {
-      __swap(__t[0], __u[0]);
+    requires(_Tp (&__t)[_Size], _Up (&__u)[_Size], const __fn& __swap) {
+        __swap(__t[0], __u[0]);
     };
-
-  template<class _Tp>
-  concept __exchangeable =
-    !__unqualified_swappable_with<_Tp&, _Tp&> &&
-    move_constructible<_Tp> &&
-    assignable_from<_Tp&, _Tp>;
-
-  struct __fn {
-    // 2.1   `S` is `(void)swap(E1, E2)`* if `E1` or `E2` has class or enumeration type and...
-    // *The name `swap` is used here unqualified.
-    template<class _Tp, class _Up>
-      requires __unqualified_swappable_with<_Tp, _Up>
-    _LIBCPP_HIDE_FROM_ABI constexpr void operator()(_Tp&& __t, _Up&& __u) const
-      noexcept(noexcept(swap(_VSTD::forward<_Tp>(__t), _VSTD::forward<_Up>(__u))))
-    {
-      swap(_VSTD::forward<_Tp>(__t), _VSTD::forward<_Up>(__u));
-    }
-
-    // 2.2   Otherwise, if `E1` and `E2` are lvalues of array types with equal extent and...
-    template<class _Tp, class _Up, size_t _Size>
-      requires __swappable_arrays<_Tp, _Up, _Size>
-    _LIBCPP_HIDE_FROM_ABI constexpr void operator()(_Tp(& __t)[_Size], _Up(& __u)[_Size]) const
-      noexcept(noexcept((*this)(*__t, *__u)))
-    {
-      // TODO(cjdb): replace with `ranges::swap_ranges`.
-      for (size_t __i = 0; __i < _Size; ++__i) {
-        (*this)(__t[__i], __u[__i]);
-      }
-    }
-
-    // 2.3   Otherwise, if `E1` and `E2` are lvalues of the same type `T` that models...
-    template<__exchangeable _Tp>
-    _LIBCPP_HIDE_FROM_ABI constexpr void operator()(_Tp& __x, _Tp& __y) const
-      noexcept(is_nothrow_move_constructible_v<_Tp> && is_nothrow_move_assignable_v<_Tp>)
-    {
-      __y = _VSTD::exchange(__x, _VSTD::move(__y));
+// clang-format on
+
+template <class _Tp>
+concept __exchangeable =
+    !__unqualified_swappable_with<_Tp&, _Tp&> && move_constructible<_Tp> && assignable_from<_Tp&, _Tp>;
+
+struct __fn {
+  // 2.1   `S` is `(void)swap(E1, E2)`* if `E1` or `E2` has class or enumeration type and...
+  // *The name `swap` is used here unqualified.
+  template <class _Tp, class _Up>
+    requires __unqualified_swappable_with<_Tp, _Up>
+  _LIBCPP_HIDE_FROM_ABI constexpr void operator()(_Tp&& __t, _Up&& __u) const
+      noexcept(noexcept(swap(_VSTD::forward<_Tp>(__t), _VSTD::forward<_Up>(__u)))) {
+    swap(_VSTD::forward<_Tp>(__t), _VSTD::forward<_Up>(__u));
+  }
+
+  // 2.2   Otherwise, if `E1` and `E2` are lvalues of array types with equal extent and...
+  template <class _Tp, class _Up, size_t _Size>
+    requires __swappable_arrays<_Tp, _Up, _Size>
+  _LIBCPP_HIDE_FROM_ABI constexpr void operator()(_Tp (&__t)[_Size], _Up (&__u)[_Size]) const
+      noexcept(noexcept((*this)(*__t, *__u))) {
+    // TODO(cjdb): replace with `ranges::swap_ranges`.
+    for (size_t __i = 0; __i < _Size; ++__i) {
+      (*this)(__t[__i], __u[__i]);
     }
-  };
+  }
+
+  // 2.3   Otherwise, if `E1` and `E2` are lvalues of the same type `T` that models...
+  template <__exchangeable _Tp>
+  _LIBCPP_HIDE_FROM_ABI constexpr void operator()(_Tp& __x, _Tp& __y) const
+      noexcept(is_nothrow_move_constructible_v<_Tp>&& is_nothrow_move_assignable_v<_Tp>) {
+    __y = _VSTD::exchange(__x, _VSTD::move(__y));
+  }
+};
 } // namespace __swap
 
 inline namespace __cpo {
-  inline constexpr auto swap = __swap::__fn{};
+inline constexpr auto swap = __swap::__fn{};
 } // namespace __cpo
 } // namespace ranges
 
-template<class _Tp>
+template <class _Tp>
 concept swappable = requires(_Tp& __a, _Tp& __b) { ranges::swap(__a, __b); };
 
-template<class _Tp, class _Up>
-concept swappable_with =
-  common_reference_with<_Tp, _Up> &&
-  requires(_Tp&& __t, _Up&& __u) {
-    ranges::swap(_VSTD::forward<_Tp>(__t), _VSTD::forward<_Tp>(__t));
-    ranges::swap(_VSTD::forward<_Up>(__u), _VSTD::forward<_Up>(__u));
-    ranges::swap(_VSTD::forward<_Tp>(__t), _VSTD::forward<_Up>(__u));
-    ranges::swap(_VSTD::forward<_Up>(__u), _VSTD::forward<_Tp>(__t));
-  };
+template <class _Tp, class _Up>
+concept swappable_with = common_reference_with<_Tp, _Up> && requires(_Tp&& __t, _Up&& __u) {
+  ranges::swap(_VSTD::forward<_Tp>(__t), _VSTD::forward<_Tp>(__t));
+  ranges::swap(_VSTD::forward<_Up>(__u), _VSTD::forward<_Up>(__u));
+  ranges::swap(_VSTD::forward<_Tp>(__t), _VSTD::forward<_Up>(__u));
+  ranges::swap(_VSTD::forward<_Up>(__u), _VSTD::forward<_Tp>(__t));
+};
 
 #endif // _LIBCPP_STD_VER >= 20
 
diff --git a/libcxx/include/__concepts/totally_ordered.h b/libcxx/include/__concepts/totally_ordered.h
index 350eff33882b1..186c3b430dd54 100644
--- a/libcxx/include/__concepts/totally_ordered.h
+++ b/libcxx/include/__concepts/totally_ordered.h
@@ -25,31 +25,32 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 // [concept.totallyordered]
 
-template<class _Tp, class _Up>
-concept __partially_ordered_with =
-  requires(__make_const_lvalue_ref<_Tp> __t, __make_const_lvalue_ref<_Up> __u) {
-    { __t <  __u } -> __boolean_testable;
-    { __t >  __u } -> __boolean_testable;
-    { __t <= __u } -> __boolean_testable;
-    { __t >= __u } -> __boolean_testable;
-    { __u <  __t } -> __boolean_testable;
-    { __u >  __t } -> __boolean_testable;
-    { __u <= __t } -> __boolean_testable;
-    { __u >= __t } -> __boolean_testable;
-  };
-
-template<class _Tp>
+template <class _Tp, class _Up>
+concept __partially_ordered_with = requires(__make_const_lvalue_ref<_Tp> __t, __make_const_lvalue_ref<_Up> __u) {
+  { __t < __u } -> __boolean_testable;
+  { __t > __u } -> __boolean_testable;
+  { __t <= __u } -> __boolean_testable;
+  { __t >= __u } -> __boolean_testable;
+  { __u < __t } -> __boolean_testable;
+  { __u > __t } -> __boolean_testable;
+  { __u <= __t } -> __boolean_testable;
+  { __u >= __t } -> __boolean_testable;
+};
+
+template <class _Tp>
 concept totally_ordered = equality_comparable<_Tp> && __partially_ordered_with<_Tp, _Tp>;
 
-template<class _Tp, class _Up>
+// clang-format off
+template <class _Tp, class _Up>
 concept totally_ordered_with =
-  totally_ordered<_Tp> && totally_ordered<_Up> &&
-  equality_comparable_with<_Tp, _Up> &&
-  totally_ordered<
-    common_reference_t<
-      __make_const_lvalue_ref<_Tp>,
-      __make_const_lvalue_ref<_Up>>> &&
-  __partially_ordered_with<_Tp, _Up>;
+    totally_ordered<_Tp> && totally_ordered<_Up> &&
+    equality_comparable_with<_Tp, _Up> &&
+    totally_ordered<
+        common_reference_t<
+            __make_const_lvalue_ref<_Tp>,
+            __make_const_lvalue_ref<_Up>>> &&
+    __partially_ordered_with<_Tp, _Up>;
+// clang-format on
 
 #endif // _LIBCPP_STD_VER >= 20
 
diff --git a/libcxx/include/__type_traits/add_const.h b/libcxx/include/__type_traits/add_const.h
index 44d4058ba8436..9a6f1c10299f7 100644
--- a/libcxx/include/__type_traits/add_const.h
+++ b/libcxx/include/__type_traits/add_const.h
@@ -17,12 +17,14 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS add_const {
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS add_const {
   typedef _LIBCPP_NODEBUG const _Tp type;
 };
 
 #if _LIBCPP_STD_VER >= 14
-template <class _Tp> using add_const_t = typename add_const<_Tp>::type;
+template <class _Tp>
+using add_const_t = typename add_const<_Tp>::type;
 #endif
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/add_cv.h b/libcxx/include/__type_traits/add_cv.h
index ee821b3293cc0..9e23e5ceb7a3b 100644
--- a/libcxx/include/__type_traits/add_cv.h
+++ b/libcxx/include/__type_traits/add_cv.h
@@ -17,12 +17,14 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS add_cv {
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS add_cv {
   typedef _LIBCPP_NODEBUG const volatile _Tp type;
 };
 
 #if _LIBCPP_STD_VER >= 14
-template <class _Tp> using add_cv_t = typename add_cv<_Tp>::type;
+template <class _Tp>
+using add_cv_t = typename add_cv<_Tp>::type;
 #endif
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/add_lvalue_reference.h b/libcxx/include/__type_traits/add_lvalue_reference.h
index 848cc21f60385..a633e39045320 100644
--- a/libcxx/include/__type_traits/add_lvalue_reference.h
+++ b/libcxx/include/__type_traits/add_lvalue_reference.h
@@ -45,7 +45,8 @@ struct add_lvalue_reference {
 };
 
 #if _LIBCPP_STD_VER >= 14
-template <class _Tp> using add_lvalue_reference_t = __add_lvalue_reference_t<_Tp>;
+template <class _Tp>
+using add_lvalue_reference_t = __add_lvalue_reference_t<_Tp>;
 #endif
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/add_pointer.h b/libcxx/include/__type_traits/add_pointer.h
index 1e74dafaf0e8b..358e3cbd23843 100644
--- a/libcxx/include/__type_traits/add_pointer.h
+++ b/libcxx/include/__type_traits/add_pointer.h
@@ -28,13 +28,14 @@ template <class _Tp>
 using __add_pointer_t = __add_pointer(_Tp);
 
 #else
-template <class _Tp,
-          bool = __libcpp_is_referenceable<_Tp>::value || is_void<_Tp>::value>
+template <class _Tp, bool = __libcpp_is_referenceable<_Tp>::value || is_void<_Tp>::value>
 struct __add_pointer_impl {
   typedef _LIBCPP_NODEBUG __libcpp_remove_reference_t<_Tp>* type;
 };
-template <class _Tp> struct __add_pointer_impl<_Tp, false>
-    {typedef _LIBCPP_NODEBUG _Tp type;};
+template <class _Tp>
+struct __add_pointer_impl<_Tp, false> {
+  typedef _LIBCPP_NODEBUG _Tp type;
+};
 
 template <class _Tp>
 using __add_pointer_t = typename __add_pointer_impl<_Tp>::type;
@@ -47,7 +48,8 @@ struct add_pointer {
 };
 
 #if _LIBCPP_STD_VER >= 14
-template <class _Tp> using add_pointer_t = __add_pointer_t<_Tp>;
+template <class _Tp>
+using add_pointer_t = __add_pointer_t<_Tp>;
 #endif
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/add_volatile.h b/libcxx/include/__type_traits/add_volatile.h
index f4fc42f69ebb3..56b7dfaac026e 100644
--- a/libcxx/include/__type_traits/add_volatile.h
+++ b/libcxx/include/__type_traits/add_volatile.h
@@ -17,12 +17,14 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS add_volatile {
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS add_volatile {
   typedef _LIBCPP_NODEBUG volatile _Tp type;
 };
 
 #if _LIBCPP_STD_VER >= 14
-template <class _Tp> using add_volatile_t = typename add_volatile<_Tp>::type;
+template <class _Tp>
+using add_volatile_t = typename add_volatile<_Tp>::type;
 #endif
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/aligned_storage.h b/libcxx/include/__type_traits/aligned_storage.h
index 29938d286244e..46aae12832f86 100644
--- a/libcxx/include/__type_traits/aligned_storage.h
+++ b/libcxx/include/__type_traits/aligned_storage.h
@@ -23,59 +23,63 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp>
-struct __align_type
-{
-    static const size_t value = _LIBCPP_PREFERRED_ALIGNOF(_Tp);
-    typedef _Tp type;
+struct __align_type {
+  static const size_t value = _LIBCPP_PREFERRED_ALIGNOF(_Tp);
+  typedef _Tp type;
 };
 
-struct __struct_double {long double __lx;};
-struct __struct_double4 {double __lx[4];};
-
-typedef
-    __type_list<__align_type<unsigned char>,
-    __type_list<__align_type<unsigned short>,
-    __type_list<__align_type<unsigned int>,
-    __type_list<__align_type<unsigned long>,
-    __type_list<__align_type<unsigned long long>,
-    __type_list<__align_type<double>,
-    __type_list<__align_type<long double>,
-    __type_list<__align_type<__struct_double>,
-    __type_list<__align_type<__struct_double4>,
-    __type_list<__align_type<int*>,
-    __nat
-    > > > > > > > > > > __all_types;
+struct __struct_double {
+  long double __lx;
+};
+struct __struct_double4 {
+  double __lx[4];
+};
+
+// clang-format off
+typedef __type_list<__align_type<unsigned char>,
+        __type_list<__align_type<unsigned short>,
+        __type_list<__align_type<unsigned int>,
+        __type_list<__align_type<unsigned long>,
+        __type_list<__align_type<unsigned long long>,
+        __type_list<__align_type<double>,
+        __type_list<__align_type<long double>,
+        __type_list<__align_type<__struct_double>,
+        __type_list<__align_type<__struct_double4>,
+        __type_list<__align_type<int*>,
+        __nat
+        > > > > > > > > > > __all_types;
+// clang-format on
 
 template <size_t _Align>
 struct _ALIGNAS(_Align) __fallback_overaligned {};
 
-template <class _TL, size_t _Align> struct __find_pod;
+template <class _TL, size_t _Align>
+struct __find_pod;
 
 template <class _Hp, size_t _Align>
-struct __find_pod<__type_list<_Hp, __nat>, _Align>
-{
-    typedef __conditional_t<_Align == _Hp::value, typename _Hp::type, __fallback_overaligned<_Align> > type;
+struct __find_pod<__type_list<_Hp, __nat>, _Align> {
+  typedef __conditional_t<_Align == _Hp::value, typename _Hp::type, __fallback_overaligned<_Align> > type;
 };
 
 template <class _Hp, class _Tp, size_t _Align>
-struct __find_pod<__type_list<_Hp, _Tp>, _Align>
-{
-    typedef __conditional_t<_Align == _Hp::value, typename _Hp::type, typename __find_pod<_Tp, _Align>::type> type;
+struct __find_pod<__type_list<_Hp, _Tp>, _Align> {
+  typedef __conditional_t<_Align == _Hp::value, typename _Hp::type, typename __find_pod<_Tp, _Align>::type> type;
 };
 
-template <class _TL, size_t _Len> struct __find_max_align;
+template <class _TL, size_t _Len>
+struct __find_max_align;
 
 template <class _Hp, size_t _Len>
 struct __find_max_align<__type_list<_Hp, __nat>, _Len> : public integral_constant<size_t, _Hp::value> {};
 
 template <size_t _Len, size_t _A1, size_t _A2>
-struct __select_align
-{
+struct __select_align {
 private:
-    static const size_t __min = _A2 < _A1 ? _A2 : _A1;
-    static const size_t __max = _A1 < _A2 ? _A2 : _A1;
+  static const size_t __min = _A2 < _A1 ? _A2 : _A1;
+  static const size_t __max = _A1 < _A2 ? _A2 : _A1;
+
 public:
-    static const size_t value = _Len < __max ? __min : __max;
+  static const size_t value = _Len < __max ? __min : __max;
 };
 
 template <class _Hp, class _Tp, size_t _Len>
@@ -83,22 +87,20 @@ struct __find_max_align<__type_list<_Hp, _Tp>, _Len>
     : public integral_constant<size_t, __select_align<_Len, _Hp::value, __find_max_align<_Tp, _Len>::value>::value> {};
 
 template <size_t _Len, size_t _Align = __find_max_align<__all_types, _Len>::value>
-struct _LIBCPP_DEPRECATED_IN_CXX23 _LIBCPP_TEMPLATE_VIS aligned_storage
-{
-    typedef typename __find_pod<__all_types, _Align>::type _Aligner;
-    union type
-    {
-        _Aligner __align;
-        unsigned char __data[(_Len + _Align - 1)/_Align * _Align];
-    };
+struct _LIBCPP_DEPRECATED_IN_CXX23 _LIBCPP_TEMPLATE_VIS aligned_storage {
+  typedef typename __find_pod<__all_types, _Align>::type _Aligner;
+  union type {
+    _Aligner __align;
+    unsigned char __data[(_Len + _Align - 1) / _Align * _Align];
+  };
 };
 
 #if _LIBCPP_STD_VER >= 14
 
-  _LIBCPP_SUPPRESS_DEPRECATED_PUSH
+_LIBCPP_SUPPRESS_DEPRECATED_PUSH
 template <size_t _Len, size_t _Align = __find_max_align<__all_types, _Len>::value>
-    using aligned_storage_t _LIBCPP_DEPRECATED_IN_CXX23 = typename aligned_storage<_Len, _Align>::type;
-  _LIBCPP_SUPPRESS_DEPRECATED_POP
+using aligned_storage_t _LIBCPP_DEPRECATED_IN_CXX23 = typename aligned_storage<_Len, _Align>::type;
+_LIBCPP_SUPPRESS_DEPRECATED_POP
 
 #endif
 
diff --git a/libcxx/include/__type_traits/aligned_union.h b/libcxx/include/__type_traits/aligned_union.h
index dd1515a693c11..005ed9a096ea8 100644
--- a/libcxx/include/__type_traits/aligned_union.h
+++ b/libcxx/include/__type_traits/aligned_union.h
@@ -20,30 +20,25 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <size_t _I0, size_t ..._In>
+template <size_t _I0, size_t... _In>
 struct __static_max;
 
 template <size_t _I0>
-struct __static_max<_I0>
-{
-    static const size_t value = _I0;
+struct __static_max<_I0> {
+  static const size_t value = _I0;
 };
 
-template <size_t _I0, size_t _I1, size_t ..._In>
-struct __static_max<_I0, _I1, _In...>
-{
-    static const size_t value = _I0 >= _I1 ? __static_max<_I0, _In...>::value :
-                                             __static_max<_I1, _In...>::value;
+template <size_t _I0, size_t _I1, size_t... _In>
+struct __static_max<_I0, _I1, _In...> {
+  static const size_t value = _I0 >= _I1 ? __static_max<_I0, _In...>::value : __static_max<_I1, _In...>::value;
 };
 
-template <size_t _Len, class _Type0, class ..._Types>
-struct _LIBCPP_DEPRECATED_IN_CXX23 aligned_union
-{
-    static const size_t alignment_value = __static_max<_LIBCPP_PREFERRED_ALIGNOF(_Type0),
-                                                       _LIBCPP_PREFERRED_ALIGNOF(_Types)...>::value;
-    static const size_t __len = __static_max<_Len, sizeof(_Type0),
-                                             sizeof(_Types)...>::value;
-    typedef typename aligned_storage<__len, alignment_value>::type type;
+template <size_t _Len, class _Type0, class... _Types>
+struct _LIBCPP_DEPRECATED_IN_CXX23 aligned_union {
+  static const size_t alignment_value =
+      __static_max<_LIBCPP_PREFERRED_ALIGNOF(_Type0), _LIBCPP_PREFERRED_ALIGNOF(_Types)...>::value;
+  static const size_t __len = __static_max<_Len, sizeof(_Type0), sizeof(_Types)...>::value;
+  typedef typename aligned_storage<__len, alignment_value>::type type;
 };
 
 #if _LIBCPP_STD_VER >= 14
diff --git a/libcxx/include/__type_traits/alignment_of.h b/libcxx/include/__type_traits/alignment_of.h
index d7851aefe7673..f2d069bf2488f 100644
--- a/libcxx/include/__type_traits/alignment_of.h
+++ b/libcxx/include/__type_traits/alignment_of.h
@@ -19,8 +19,8 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS alignment_of
-    : public integral_constant<size_t, _LIBCPP_ALIGNOF(_Tp)> {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS alignment_of : public integral_constant<size_t, _LIBCPP_ALIGNOF(_Tp)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/can_extract_key.h b/libcxx/include/__type_traits/can_extract_key.h
index 454c56bfae9da..b8359d0708810 100644
--- a/libcxx/include/__type_traits/can_extract_key.h
+++ b/libcxx/include/__type_traits/can_extract_key.h
@@ -40,16 +40,13 @@ struct __can_extract_key<_Pair, _Key, pair<_First, _Second> >
 // __can_extract_map_key uses true_type/false_type instead of the tags.
 // It returns true if _Key != _ContainerValueTy (the container is a map not a set)
 // and _ValTy == _Key.
-template <class _ValTy, class _Key, class _ContainerValueTy,
-          class _RawValTy = __remove_const_ref_t<_ValTy> >
-struct __can_extract_map_key
-    : integral_constant<bool, _IsSame<_RawValTy, _Key>::value> {};
+template <class _ValTy, class _Key, class _ContainerValueTy, class _RawValTy = __remove_const_ref_t<_ValTy> >
+struct __can_extract_map_key : integral_constant<bool, _IsSame<_RawValTy, _Key>::value> {};
 
 // This specialization returns __extract_key_fail_tag for non-map containers
 // because _Key == _ContainerValueTy
 template <class _ValTy, class _Key, class _RawValTy>
-struct __can_extract_map_key<_ValTy, _Key, _Key, _RawValTy>
-    : false_type {};
+struct __can_extract_map_key<_ValTy, _Key, _Key, _RawValTy> : false_type {};
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__type_traits/common_reference.h b/libcxx/include/__type_traits/common_reference.h
index 59b3dfbe1233e..c802902eb19fc 100644
--- a/libcxx/include/__type_traits/common_reference.h
+++ b/libcxx/include/__type_traits/common_reference.h
@@ -30,8 +30,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 #if _LIBCPP_STD_VER >= 20
 // Let COND_RES(X, Y) be:
 template <class _Xp, class _Yp>
-using __cond_res =
-    decltype(false ? std::declval<_Xp(&)()>()() : std::declval<_Yp(&)()>()());
+using __cond_res = decltype(false ? std::declval<_Xp (&)()>()() : std::declval<_Yp (&)()>()());
 
 // Let `XREF(A)` denote a unary alias template `T` such that `T<U>` denotes the same type as `U`
 // with the addition of `A`'s cv and reference qualifiers, for a non-reference cv-unqualified type
@@ -39,47 +38,49 @@ using __cond_res =
 // [Note: `XREF(A)` is `__xref<A>::template __apply`]
 template <class _Tp>
 struct __xref {
-  template<class _Up>
+  template <class _Up>
   using __apply = __copy_cvref_t<_Tp, _Up>;
 };
 
 // Given types A and B, let X be remove_reference_t<A>, let Y be remove_reference_t<B>,
 // and let COMMON-REF(A, B) be:
-template<class _Ap, class _Bp, class _Xp = remove_reference_t<_Ap>, class _Yp = remove_reference_t<_Bp>>
+template <class _Ap, class _Bp, class _Xp = remove_reference_t<_Ap>, class _Yp = remove_reference_t<_Bp>>
 struct __common_ref;
 
-template<class _Xp, class _Yp>
+template <class _Xp, class _Yp>
 using __common_ref_t = typename __common_ref<_Xp, _Yp>::__type;
 
-template<class _Xp, class _Yp>
+template <class _Xp, class _Yp>
 using __cv_cond_res = __cond_res<__copy_cv_t<_Xp, _Yp>&, __copy_cv_t<_Yp, _Xp>&>;
 
-
 //    If A and B are both lvalue reference types, COMMON-REF(A, B) is
 //    COND-RES(COPYCV(X, Y)&, COPYCV(Y, X)&) if that type exists and is a reference type.
-template<class _Ap, class _Bp, class _Xp, class _Yp>
-requires requires { typename __cv_cond_res<_Xp, _Yp>; } && is_reference_v<__cv_cond_res<_Xp, _Yp>>
-struct __common_ref<_Ap&, _Bp&, _Xp, _Yp>
-{
-    using __type = __cv_cond_res<_Xp, _Yp>;
+// clang-format off
+template <class _Ap, class _Bp, class _Xp, class _Yp>
+  requires
+    requires { typename __cv_cond_res<_Xp, _Yp>; } &&
+    is_reference_v<__cv_cond_res<_Xp, _Yp>>
+struct __common_ref<_Ap&, _Bp&, _Xp, _Yp> {
+  using __type = __cv_cond_res<_Xp, _Yp>;
 };
+// clang-format on
 
 //    Otherwise, let C be remove_reference_t<COMMON-REF(X&, Y&)>&&. ...
 template <class _Xp, class _Yp>
 using __common_ref_C = remove_reference_t<__common_ref_t<_Xp&, _Yp&>>&&;
 
-
 //    .... If A and B are both rvalue reference types, C is well-formed, and
 //    is_convertible_v<A, C> && is_convertible_v<B, C> is true, then COMMON-REF(A, B) is C.
-template<class _Ap, class _Bp, class _Xp, class _Yp>
-requires
-  requires { typename __common_ref_C<_Xp, _Yp>; } &&
-  is_convertible_v<_Ap&&, __common_ref_C<_Xp, _Yp>> &&
-  is_convertible_v<_Bp&&, __common_ref_C<_Xp, _Yp>>
-struct __common_ref<_Ap&&, _Bp&&, _Xp, _Yp>
-{
-    using __type = __common_ref_C<_Xp, _Yp>;
+// clang-format off
+template <class _Ap, class _Bp, class _Xp, class _Yp>
+  requires
+    requires { typename __common_ref_C<_Xp, _Yp>; } &&
+    is_convertible_v<_Ap&&, __common_ref_C<_Xp, _Yp>> &&
+    is_convertible_v<_Bp&&, __common_ref_C<_Xp, _Yp>>
+struct __common_ref<_Ap&&, _Bp&&, _Xp, _Yp> {
+  using __type = __common_ref_C<_Xp, _Yp>;
 };
+// clang-format on
 
 //    Otherwise, let D be COMMON-REF(const X&, Y&). ...
 template <class _Tp, class _Up>
@@ -87,21 +88,23 @@ using __common_ref_D = __common_ref_t<const _Tp&, _Up&>;
 
 //    ... If A is an rvalue reference and B is an lvalue reference and D is well-formed and
 //    is_convertible_v<A, D> is true, then COMMON-REF(A, B) is D.
-template<class _Ap, class _Bp, class _Xp, class _Yp>
-requires requires { typename __common_ref_D<_Xp, _Yp>; } &&
-         is_convertible_v<_Ap&&, __common_ref_D<_Xp, _Yp>>
-struct __common_ref<_Ap&&, _Bp&, _Xp, _Yp>
-{
-    using __type = __common_ref_D<_Xp, _Yp>;
+// clang-format off
+template <class _Ap, class _Bp, class _Xp, class _Yp>
+  requires
+    requires { typename __common_ref_D<_Xp, _Yp>; } &&
+    is_convertible_v<_Ap&&, __common_ref_D<_Xp, _Yp>>
+struct __common_ref<_Ap&&, _Bp&, _Xp, _Yp> {
+  using __type = __common_ref_D<_Xp, _Yp>;
 };
+// clang-format on
 
 //    Otherwise, if A is an lvalue reference and B is an rvalue reference, then
 //    COMMON-REF(A, B) is COMMON-REF(B, A).
-template<class _Ap, class _Bp, class _Xp, class _Yp>
+template <class _Ap, class _Bp, class _Xp, class _Yp>
 struct __common_ref<_Ap&, _Bp&&, _Xp, _Yp> : __common_ref<_Bp&&, _Ap&> {};
 
 //    Otherwise, COMMON-REF(A, B) is ill-formed.
-template<class _Ap, class _Bp, class _Xp, class _Yp>
+template <class _Ap, class _Bp, class _Xp, class _Yp>
 struct __common_ref {};
 
 // Note C: For the common_reference trait applied to a parameter pack [...]
@@ -113,73 +116,75 @@ template <class... _Types>
 using common_reference_t = typename common_reference<_Types...>::type;
 
 // bullet 1 - sizeof...(T) == 0
-template<>
+template <>
 struct common_reference<> {};
 
 // bullet 2 - sizeof...(T) == 1
 template <class _Tp>
-struct common_reference<_Tp>
-{
-    using type = _Tp;
+struct common_reference<_Tp> {
+  using type = _Tp;
 };
 
 // bullet 3 - sizeof...(T) == 2
-template <class _Tp, class _Up> struct __common_reference_sub_bullet3;
-template <class _Tp, class _Up> struct __common_reference_sub_bullet2 : __common_reference_sub_bullet3<_Tp, _Up> {};
-template <class _Tp, class _Up> struct __common_reference_sub_bullet1 : __common_reference_sub_bullet2<_Tp, _Up> {};
+template <class _Tp, class _Up>
+struct __common_reference_sub_bullet3;
+template <class _Tp, class _Up>
+struct __common_reference_sub_bullet2 : __common_reference_sub_bullet3<_Tp, _Up> {};
+template <class _Tp, class _Up>
+struct __common_reference_sub_bullet1 : __common_reference_sub_bullet2<_Tp, _Up> {};
 
 // sub-bullet 1 - If T1 and T2 are reference types and COMMON-REF(T1, T2) is well-formed, then
 // the member typedef `type` denotes that type.
-template <class _Tp, class _Up> struct common_reference<_Tp, _Up> : __common_reference_sub_bullet1<_Tp, _Up> {};
+template <class _Tp, class _Up>
+struct common_reference<_Tp, _Up> : __common_reference_sub_bullet1<_Tp, _Up> {};
 
 template <class _Tp, class _Up>
-requires is_reference_v<_Tp> && is_reference_v<_Up> && requires { typename __common_ref_t<_Tp, _Up>; }
-struct __common_reference_sub_bullet1<_Tp, _Up>
-{
-    using type = __common_ref_t<_Tp, _Up>;
+  requires is_reference_v<_Tp> && is_reference_v<_Up> && requires { typename __common_ref_t<_Tp, _Up>; }
+struct __common_reference_sub_bullet1<_Tp, _Up> {
+  using type = __common_ref_t<_Tp, _Up>;
 };
 
 // sub-bullet 2 - Otherwise, if basic_common_reference<remove_cvref_t<T1>, remove_cvref_t<T2>, XREF(T1), XREF(T2)>::type
 // is well-formed, then the member typedef `type` denotes that type.
-template <class, class, template <class> class, template <class> class> struct basic_common_reference {};
+template <class, class, template <class> class, template <class> class>
+struct basic_common_reference {};
 
 template <class _Tp, class _Up>
-using __basic_common_reference_t = typename basic_common_reference<
-    remove_cvref_t<_Tp>, remove_cvref_t<_Up>,
-    __xref<_Tp>::template __apply, __xref<_Up>::template __apply>::type;
+using __basic_common_reference_t =
+    typename basic_common_reference<remove_cvref_t<_Tp>,
+                                    remove_cvref_t<_Up>,
+                                    __xref<_Tp>::template __apply,
+                                    __xref<_Up>::template __apply>::type;
 
 template <class _Tp, class _Up>
-requires requires { typename __basic_common_reference_t<_Tp, _Up>; }
-struct __common_reference_sub_bullet2<_Tp, _Up>
-{
-    using type = __basic_common_reference_t<_Tp, _Up>;
+  requires requires { typename __basic_common_reference_t<_Tp, _Up>; }
+struct __common_reference_sub_bullet2<_Tp, _Up> {
+  using type = __basic_common_reference_t<_Tp, _Up>;
 };
 
 // sub-bullet 3 - Otherwise, if COND-RES(T1, T2) is well-formed,
 // then the member typedef `type` denotes that type.
 template <class _Tp, class _Up>
-requires requires { typename __cond_res<_Tp, _Up>; }
-struct __common_reference_sub_bullet3<_Tp, _Up>
-{
-    using type = __cond_res<_Tp, _Up>;
+  requires requires { typename __cond_res<_Tp, _Up>; }
+struct __common_reference_sub_bullet3<_Tp, _Up> {
+  using type = __cond_res<_Tp, _Up>;
 };
 
-
 // sub-bullet 4 & 5 - Otherwise, if common_type_t<T1, T2> is well-formed,
 //                    then the member typedef `type` denotes that type.
 //                  - Otherwise, there shall be no member `type`.
-template <class _Tp, class _Up> struct __common_reference_sub_bullet3 : common_type<_Tp, _Up> {};
+template <class _Tp, class _Up>
+struct __common_reference_sub_bullet3 : common_type<_Tp, _Up> {};
 
 // bullet 4 - If there is such a type `C`, the member typedef type shall denote the same type, if
 //            any, as `common_reference_t<C, Rest...>`.
 template <class _Tp, class _Up, class _Vp, class... _Rest>
-requires requires { typename common_reference_t<_Tp, _Up>; }
-struct common_reference<_Tp, _Up, _Vp, _Rest...>
-    : common_reference<common_reference_t<_Tp, _Up>, _Vp, _Rest...>
-{};
+  requires requires { typename common_reference_t<_Tp, _Up>; }
+struct common_reference<_Tp, _Up, _Vp, _Rest...> : common_reference<common_reference_t<_Tp, _Up>, _Vp, _Rest...> {};
 
 // bullet 5 - Otherwise, there shall be no member `type`.
-template <class...> struct common_reference {};
+template <class...>
+struct common_reference {};
 
 #endif // _LIBCPP_STD_VER >= 20
 
diff --git a/libcxx/include/__type_traits/common_type.h b/libcxx/include/__type_traits/common_type.h
index d6f73f64588c1..7f86fcaaace44 100644
--- a/libcxx/include/__type_traits/common_type.h
+++ b/libcxx/include/__type_traits/common_type.h
@@ -33,9 +33,8 @@ struct __common_type3 {};
 
 // sub-bullet 4 - "if COND_RES(CREF(D1), CREF(D2)) denotes a type..."
 template <class _Tp, class _Up>
-struct __common_type3<_Tp, _Up, void_t<__cond_type<const _Tp&, const _Up&>>>
-{
-    using type = remove_cvref_t<__cond_type<const _Tp&, const _Up&>>;
+struct __common_type3<_Tp, _Up, void_t<__cond_type<const _Tp&, const _Up&>>> {
+  using type = remove_cvref_t<__cond_type<const _Tp&, const _Up&>>;
 };
 
 template <class _Tp, class _Up, class = void>
@@ -47,11 +46,8 @@ struct __common_type2_imp {};
 
 // sub-bullet 3 - "if decay_t<decltype(false ? declval<D1>() : declval<D2>())> ..."
 template <class _Tp, class _Up>
-struct __common_type2_imp<_Tp, _Up, __void_t<decltype(true ? std::declval<_Tp>() : std::declval<_Up>())> >
-{
-  typedef _LIBCPP_NODEBUG __decay_t<decltype(
-                         true ? std::declval<_Tp>() : std::declval<_Up>()
-                         )> type;
+struct __common_type2_imp<_Tp, _Up, __void_t<decltype(true ? std::declval<_Tp>() : std::declval<_Up>())> > {
+  typedef _LIBCPP_NODEBUG __decay_t<decltype(true ? std::declval<_Tp>() : std::declval<_Up>())> type;
 };
 
 template <class, class = void>
@@ -63,19 +59,13 @@ template <class... _Tp>
 struct _LIBCPP_TEMPLATE_VIS common_type;
 
 template <class _Tp, class _Up>
-struct __common_type_impl<
-    __common_types<_Tp, _Up>, __void_t<typename common_type<_Tp, _Up>::type> >
-{
+struct __common_type_impl< __common_types<_Tp, _Up>, __void_t<typename common_type<_Tp, _Up>::type> > {
   typedef typename common_type<_Tp, _Up>::type type;
 };
 
 template <class _Tp, class _Up, class _Vp, class... _Rest>
-struct __common_type_impl<
-    __common_types<_Tp, _Up, _Vp, _Rest...>,
-    __void_t<typename common_type<_Tp, _Up>::type> >
-    : __common_type_impl<__common_types<typename common_type<_Tp, _Up>::type,
-                                        _Vp, _Rest...> > {
-};
+struct __common_type_impl<__common_types<_Tp, _Up, _Vp, _Rest...>, __void_t<typename common_type<_Tp, _Up>::type> >
+    : __common_type_impl<__common_types<typename common_type<_Tp, _Up>::type, _Vp, _Rest...> > {};
 
 // bullet 1 - sizeof...(Tp) == 0
 
@@ -85,31 +75,26 @@ struct _LIBCPP_TEMPLATE_VIS common_type<> {};
 // bullet 2 - sizeof...(Tp) == 1
 
 template <class _Tp>
-struct _LIBCPP_TEMPLATE_VIS common_type<_Tp>
-    : public common_type<_Tp, _Tp> {};
+struct _LIBCPP_TEMPLATE_VIS common_type<_Tp> : public common_type<_Tp, _Tp> {};
 
 // bullet 3 - sizeof...(Tp) == 2
 
 // sub-bullet 1 - "If is_same_v<T1, D1> is false or ..."
 template <class _Tp, class _Up>
 struct _LIBCPP_TEMPLATE_VIS common_type<_Tp, _Up>
-    : conditional<
-        _IsSame<_Tp, __decay_t<_Tp> >::value && _IsSame<_Up, __decay_t<_Up> >::value,
-        __common_type2_imp<_Tp, _Up>,
-        common_type<__decay_t<_Tp>, __decay_t<_Up> >
-    >::type
-{};
+    : conditional<_IsSame<_Tp, __decay_t<_Tp> >::value && _IsSame<_Up, __decay_t<_Up> >::value,
+                  __common_type2_imp<_Tp, _Up>,
+                  common_type<__decay_t<_Tp>, __decay_t<_Up> > >::type {};
 
 // bullet 4 - sizeof...(Tp) > 2
 
 template <class _Tp, class _Up, class _Vp, class... _Rest>
-struct _LIBCPP_TEMPLATE_VIS
-    common_type<_Tp, _Up, _Vp, _Rest...>
-    : __common_type_impl<
-          __common_types<_Tp, _Up, _Vp, _Rest...> > {};
+struct _LIBCPP_TEMPLATE_VIS common_type<_Tp, _Up, _Vp, _Rest...>
+    : __common_type_impl<__common_types<_Tp, _Up, _Vp, _Rest...> > {};
 
 #if _LIBCPP_STD_VER >= 14
-template <class ..._Tp> using common_type_t = typename common_type<_Tp...>::type;
+template <class... _Tp>
+using common_type_t = typename common_type<_Tp...>::type;
 #endif
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/copy_cv.h b/libcxx/include/__type_traits/copy_cv.h
index 8e9bfe0a522f8..3c4ee857f7bc7 100644
--- a/libcxx/include/__type_traits/copy_cv.h
+++ b/libcxx/include/__type_traits/copy_cv.h
@@ -23,27 +23,23 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 // Let COPYCV(FROM, TO) be an alias for type TO with the addition of FROM's
 // top-level cv-qualifiers.
 template <class _From, class _To>
-struct __copy_cv
-{
-    using type = _To;
+struct __copy_cv {
+  using type = _To;
 };
 
 template <class _From, class _To>
-struct __copy_cv<const _From, _To>
-{
-    using type = typename add_const<_To>::type;
+struct __copy_cv<const _From, _To> {
+  using type = typename add_const<_To>::type;
 };
 
 template <class _From, class _To>
-struct __copy_cv<volatile _From, _To>
-{
-    using type = typename add_volatile<_To>::type;
+struct __copy_cv<volatile _From, _To> {
+  using type = typename add_volatile<_To>::type;
 };
 
 template <class _From, class _To>
-struct __copy_cv<const volatile _From, _To>
-{
-    using type = typename add_cv<_To>::type;
+struct __copy_cv<const volatile _From, _To> {
+  using type = typename add_cv<_To>::type;
 };
 
 template <class _From, class _To>
diff --git a/libcxx/include/__type_traits/copy_cvref.h b/libcxx/include/__type_traits/copy_cvref.h
index 8ec59b95a0ada..8bbf8efdf44de 100644
--- a/libcxx/include/__type_traits/copy_cvref.h
+++ b/libcxx/include/__type_traits/copy_cvref.h
@@ -21,21 +21,18 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _From, class _To>
-struct __copy_cvref
-{
-    using type = __copy_cv_t<_From, _To>;
+struct __copy_cvref {
+  using type = __copy_cv_t<_From, _To>;
 };
 
 template <class _From, class _To>
-struct __copy_cvref<_From&, _To>
-{
-    using type = __add_lvalue_reference_t<__copy_cv_t<_From, _To> >;
+struct __copy_cvref<_From&, _To> {
+  using type = __add_lvalue_reference_t<__copy_cv_t<_From, _To> >;
 };
 
 template <class _From, class _To>
-struct __copy_cvref<_From&&, _To>
-{
-    using type = __add_rvalue_reference_t<__copy_cv_t<_From, _To> >;
+struct __copy_cvref<_From&&, _To> {
+  using type = __add_rvalue_reference_t<__copy_cv_t<_From, _To> >;
 };
 
 template <class _From, class _To>
diff --git a/libcxx/include/__type_traits/decay.h b/libcxx/include/__type_traits/decay.h
index a32c140e3c5e2..95dccaa29cbe1 100644
--- a/libcxx/include/__type_traits/decay.h
+++ b/libcxx/include/__type_traits/decay.h
@@ -37,30 +37,24 @@ struct decay {
 #else
 template <class _Up, bool>
 struct __decay {
-    typedef _LIBCPP_NODEBUG __remove_cv_t<_Up> type;
+  typedef _LIBCPP_NODEBUG __remove_cv_t<_Up> type;
 };
 
 template <class _Up>
 struct __decay<_Up, true> {
 public:
-    typedef _LIBCPP_NODEBUG typename conditional
-                     <
-                         is_array<_Up>::value,
-                         __add_pointer_t<__remove_extent_t<_Up> >,
-                         typename conditional
-                         <
-                              is_function<_Up>::value,
-                              typename add_pointer<_Up>::type,
-                              __remove_cv_t<_Up>
-                         >::type
-                     >::type type;
+  typedef _LIBCPP_NODEBUG typename conditional<
+      is_array<_Up>::value,
+      __add_pointer_t<__remove_extent_t<_Up> >,
+      typename conditional<is_function<_Up>::value, typename add_pointer<_Up>::type, __remove_cv_t<_Up> >::type >::type
+      type;
 };
 
 template <class _Tp>
-struct _LIBCPP_TEMPLATE_VIS decay
-{
+struct _LIBCPP_TEMPLATE_VIS decay {
 private:
-    typedef _LIBCPP_NODEBUG __libcpp_remove_reference_t<_Tp> _Up;
+  typedef _LIBCPP_NODEBUG __libcpp_remove_reference_t<_Tp> _Up;
+
 public:
   typedef _LIBCPP_NODEBUG typename __decay<_Up, __libcpp_is_referenceable<_Up>::value>::type type;
 };
@@ -70,7 +64,8 @@ using __decay_t = typename decay<_Tp>::type;
 #endif // __has_builtin(__decay)
 
 #if _LIBCPP_STD_VER >= 14
-template <class _Tp> using decay_t = __decay_t<_Tp>;
+template <class _Tp>
+using decay_t = __decay_t<_Tp>;
 #endif
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/enable_if.h b/libcxx/include/__type_traits/enable_if.h
index 4859981f1c314..77da9622ca28f 100644
--- a/libcxx/include/__type_traits/enable_if.h
+++ b/libcxx/include/__type_traits/enable_if.h
@@ -17,13 +17,19 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <bool, class _Tp = void> struct _LIBCPP_TEMPLATE_VIS enable_if {};
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS enable_if<true, _Tp> {typedef _Tp type;};
+template <bool, class _Tp = void>
+struct _LIBCPP_TEMPLATE_VIS enable_if {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS enable_if<true, _Tp> {
+  typedef _Tp type;
+};
 
-template <bool _Bp, class _Tp = void> using __enable_if_t _LIBCPP_NODEBUG = typename enable_if<_Bp, _Tp>::type;
+template <bool _Bp, class _Tp = void>
+using __enable_if_t _LIBCPP_NODEBUG = typename enable_if<_Bp, _Tp>::type;
 
 #if _LIBCPP_STD_VER >= 14
-template <bool _Bp, class _Tp = void> using enable_if_t = typename enable_if<_Bp, _Tp>::type;
+template <bool _Bp, class _Tp = void>
+using enable_if_t = typename enable_if<_Bp, _Tp>::type;
 #endif
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/extent.h b/libcxx/include/__type_traits/extent.h
index a031f5ed1ef05..bab03fe997eb6 100644
--- a/libcxx/include/__type_traits/extent.h
+++ b/libcxx/include/__type_traits/extent.h
@@ -21,32 +21,31 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if __has_builtin(__array_extent)
 
-template<class _Tp, size_t _Dim = 0>
-struct _LIBCPP_TEMPLATE_VIS extent
-    : integral_constant<size_t, __array_extent(_Tp, _Dim)> { };
+template <class _Tp, size_t _Dim = 0>
+struct _LIBCPP_TEMPLATE_VIS extent : integral_constant<size_t, __array_extent(_Tp, _Dim)> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp, unsigned _Ip = 0>
 inline constexpr size_t extent_v = __array_extent(_Tp, _Ip);
-#endif
+#  endif
 
 #else // __has_builtin(__array_extent)
 
-template <class _Tp, unsigned _Ip = 0> struct _LIBCPP_TEMPLATE_VIS extent
-    : public integral_constant<size_t, 0> {};
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS extent<_Tp[], 0>
-    : public integral_constant<size_t, 0> {};
-template <class _Tp, unsigned _Ip> struct _LIBCPP_TEMPLATE_VIS extent<_Tp[], _Ip>
-    : public integral_constant<size_t, extent<_Tp, _Ip-1>::value> {};
-template <class _Tp, size_t _Np> struct _LIBCPP_TEMPLATE_VIS extent<_Tp[_Np], 0>
-    : public integral_constant<size_t, _Np> {};
-template <class _Tp, size_t _Np, unsigned _Ip> struct _LIBCPP_TEMPLATE_VIS extent<_Tp[_Np], _Ip>
-    : public integral_constant<size_t, extent<_Tp, _Ip-1>::value> {};
-
-#if _LIBCPP_STD_VER >= 17
+template <class _Tp, unsigned _Ip = 0>
+struct _LIBCPP_TEMPLATE_VIS extent : public integral_constant<size_t, 0> {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS extent<_Tp[], 0> : public integral_constant<size_t, 0> {};
+template <class _Tp, unsigned _Ip>
+struct _LIBCPP_TEMPLATE_VIS extent<_Tp[], _Ip> : public integral_constant<size_t, extent<_Tp, _Ip - 1>::value> {};
+template <class _Tp, size_t _Np>
+struct _LIBCPP_TEMPLATE_VIS extent<_Tp[_Np], 0> : public integral_constant<size_t, _Np> {};
+template <class _Tp, size_t _Np, unsigned _Ip>
+struct _LIBCPP_TEMPLATE_VIS extent<_Tp[_Np], _Ip> : public integral_constant<size_t, extent<_Tp, _Ip - 1>::value> {};
+
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp, unsigned _Ip = 0>
 inline constexpr size_t extent_v = extent<_Tp, _Ip>::value;
-#endif
+#  endif
 
 #endif // __has_builtin(__array_extent)
 
diff --git a/libcxx/include/__type_traits/has_unique_object_representation.h b/libcxx/include/__type_traits/has_unique_object_representation.h
index aa45bbb14cbf0..c0ada5618f0e3 100644
--- a/libcxx/include/__type_traits/has_unique_object_representation.h
+++ b/libcxx/include/__type_traits/has_unique_object_representation.h
@@ -22,9 +22,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if _LIBCPP_STD_VER >= 17
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS has_unique_object_representations
-    : public integral_constant<bool,
-       __has_unique_object_representations(remove_cv_t<remove_all_extents_t<_Tp>>)> {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS has_unique_object_representations
+    : public integral_constant<bool, __has_unique_object_representations(remove_cv_t<remove_all_extents_t<_Tp>>)> {};
 
 template <class _Tp>
 inline constexpr bool has_unique_object_representations_v = has_unique_object_representations<_Tp>::value;
diff --git a/libcxx/include/__type_traits/has_virtual_destructor.h b/libcxx/include/__type_traits/has_virtual_destructor.h
index ae3137e9d6e37..4ce96e649e67a 100644
--- a/libcxx/include/__type_traits/has_virtual_destructor.h
+++ b/libcxx/include/__type_traits/has_virtual_destructor.h
@@ -18,8 +18,8 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS has_virtual_destructor
-    : public integral_constant<bool, __has_virtual_destructor(_Tp)> {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS has_virtual_destructor : public integral_constant<bool, __has_virtual_destructor(_Tp)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/integral_constant.h b/libcxx/include/__type_traits/integral_constant.h
index 0951ab5ad837a..a3f4fd34057c9 100644
--- a/libcxx/include/__type_traits/integral_constant.h
+++ b/libcxx/include/__type_traits/integral_constant.h
@@ -18,23 +18,20 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp, _Tp __v>
-struct _LIBCPP_TEMPLATE_VIS integral_constant
-{
-  static _LIBCPP_CONSTEXPR const _Tp      value = __v;
-  typedef _Tp               value_type;
+struct _LIBCPP_TEMPLATE_VIS integral_constant {
+  static _LIBCPP_CONSTEXPR const _Tp value = __v;
+  typedef _Tp value_type;
   typedef integral_constant type;
-  _LIBCPP_INLINE_VISIBILITY
-  _LIBCPP_CONSTEXPR operator value_type() const _NOEXCEPT {return value;}
+  _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR operator value_type() const _NOEXCEPT { return value; }
 #if _LIBCPP_STD_VER >= 14
-  _LIBCPP_INLINE_VISIBILITY
-  constexpr value_type operator ()() const _NOEXCEPT {return value;}
+  _LIBCPP_INLINE_VISIBILITY constexpr value_type operator()() const _NOEXCEPT { return value; }
 #endif
 };
 
 template <class _Tp, _Tp __v>
 _LIBCPP_CONSTEXPR const _Tp integral_constant<_Tp, __v>::value;
 
-typedef integral_constant<bool, true>  true_type;
+typedef integral_constant<bool, true> true_type;
 typedef integral_constant<bool, false> false_type;
 
 template <bool _Val>
diff --git a/libcxx/include/__type_traits/invoke.h b/libcxx/include/__type_traits/invoke.h
index bc1852ca73710..f3c00e7ede7c3 100644
--- a/libcxx/include/__type_traits/invoke.h
+++ b/libcxx/include/__type_traits/invoke.h
@@ -35,224 +35,195 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-struct __any
-{
-    __any(...);
+struct __any {
+  __any(...);
 };
 
 template <class _MP, bool _IsMemberFunctionPtr, bool _IsMemberObjectPtr>
-struct __member_pointer_traits_imp
-{
-};
+struct __member_pointer_traits_imp {};
 
-template <class _Rp, class _Class, class ..._Param>
-struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param...), true, false>
-{
-    typedef _Class _ClassType;
-    typedef _Rp _ReturnType;
-    typedef _Rp (_FnType) (_Param...);
+template <class _Rp, class _Class, class... _Param>
+struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param...), true, false> {
+  typedef _Class _ClassType;
+  typedef _Rp _ReturnType;
+  typedef _Rp(_FnType)(_Param...);
 };
 
-template <class _Rp, class _Class, class ..._Param>
-struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param..., ...), true, false>
-{
-    typedef _Class _ClassType;
-    typedef _Rp _ReturnType;
-    typedef _Rp (_FnType) (_Param..., ...);
+template <class _Rp, class _Class, class... _Param>
+struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param..., ...), true, false> {
+  typedef _Class _ClassType;
+  typedef _Rp _ReturnType;
+  typedef _Rp(_FnType)(_Param..., ...);
 };
 
-template <class _Rp, class _Class, class ..._Param>
-struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param...) const, true, false>
-{
-    typedef _Class const _ClassType;
-    typedef _Rp _ReturnType;
-    typedef _Rp (_FnType) (_Param...);
+template <class _Rp, class _Class, class... _Param>
+struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param...) const, true, false> {
+  typedef _Class const _ClassType;
+  typedef _Rp _ReturnType;
+  typedef _Rp(_FnType)(_Param...);
 };
 
-template <class _Rp, class _Class, class ..._Param>
-struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param..., ...) const, true, false>
-{
-    typedef _Class const _ClassType;
-    typedef _Rp _ReturnType;
-    typedef _Rp (_FnType) (_Param..., ...);
+template <class _Rp, class _Class, class... _Param>
+struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param..., ...) const, true, false> {
+  typedef _Class const _ClassType;
+  typedef _Rp _ReturnType;
+  typedef _Rp(_FnType)(_Param..., ...);
 };
 
-template <class _Rp, class _Class, class ..._Param>
-struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param...) volatile, true, false>
-{
-    typedef _Class volatile _ClassType;
-    typedef _Rp _ReturnType;
-    typedef _Rp (_FnType) (_Param...);
+template <class _Rp, class _Class, class... _Param>
+struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param...) volatile, true, false> {
+  typedef _Class volatile _ClassType;
+  typedef _Rp _ReturnType;
+  typedef _Rp(_FnType)(_Param...);
 };
 
-template <class _Rp, class _Class, class ..._Param>
-struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param..., ...) volatile, true, false>
-{
-    typedef _Class volatile _ClassType;
-    typedef _Rp _ReturnType;
-    typedef _Rp (_FnType) (_Param..., ...);
+template <class _Rp, class _Class, class... _Param>
+struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param..., ...) volatile, true, false> {
+  typedef _Class volatile _ClassType;
+  typedef _Rp _ReturnType;
+  typedef _Rp(_FnType)(_Param..., ...);
 };
 
-template <class _Rp, class _Class, class ..._Param>
-struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param...) const volatile, true, false>
-{
-    typedef _Class const volatile _ClassType;
-    typedef _Rp _ReturnType;
-    typedef _Rp (_FnType) (_Param...);
+template <class _Rp, class _Class, class... _Param>
+struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param...) const volatile, true, false> {
+  typedef _Class const volatile _ClassType;
+  typedef _Rp _ReturnType;
+  typedef _Rp(_FnType)(_Param...);
 };
 
-template <class _Rp, class _Class, class ..._Param>
-struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param..., ...) const volatile, true, false>
-{
-    typedef _Class const volatile _ClassType;
-    typedef _Rp _ReturnType;
-    typedef _Rp (_FnType) (_Param..., ...);
+template <class _Rp, class _Class, class... _Param>
+struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param..., ...) const volatile, true, false> {
+  typedef _Class const volatile _ClassType;
+  typedef _Rp _ReturnType;
+  typedef _Rp(_FnType)(_Param..., ...);
 };
 
-template <class _Rp, class _Class, class ..._Param>
-struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param...) &, true, false>
-{
-    typedef _Class& _ClassType;
-    typedef _Rp _ReturnType;
-    typedef _Rp (_FnType) (_Param...);
+template <class _Rp, class _Class, class... _Param>
+struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param...)&, true, false> {
+  typedef _Class& _ClassType;
+  typedef _Rp _ReturnType;
+  typedef _Rp(_FnType)(_Param...);
 };
 
-template <class _Rp, class _Class, class ..._Param>
-struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param..., ...) &, true, false>
-{
-    typedef _Class& _ClassType;
-    typedef _Rp _ReturnType;
-    typedef _Rp (_FnType) (_Param..., ...);
+template <class _Rp, class _Class, class... _Param>
+struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param..., ...)&, true, false> {
+  typedef _Class& _ClassType;
+  typedef _Rp _ReturnType;
+  typedef _Rp(_FnType)(_Param..., ...);
 };
 
-template <class _Rp, class _Class, class ..._Param>
-struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param...) const&, true, false>
-{
-    typedef _Class const& _ClassType;
-    typedef _Rp _ReturnType;
-    typedef _Rp (_FnType) (_Param...);
+template <class _Rp, class _Class, class... _Param>
+struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param...) const&, true, false> {
+  typedef _Class const& _ClassType;
+  typedef _Rp _ReturnType;
+  typedef _Rp(_FnType)(_Param...);
 };
 
-template <class _Rp, class _Class, class ..._Param>
-struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param..., ...) const&, true, false>
-{
-    typedef _Class const& _ClassType;
-    typedef _Rp _ReturnType;
-    typedef _Rp (_FnType) (_Param..., ...);
+template <class _Rp, class _Class, class... _Param>
+struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param..., ...) const&, true, false> {
+  typedef _Class const& _ClassType;
+  typedef _Rp _ReturnType;
+  typedef _Rp(_FnType)(_Param..., ...);
 };
 
-template <class _Rp, class _Class, class ..._Param>
-struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param...) volatile&, true, false>
-{
-    typedef _Class volatile& _ClassType;
-    typedef _Rp _ReturnType;
-    typedef _Rp (_FnType) (_Param...);
+template <class _Rp, class _Class, class... _Param>
+struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param...) volatile&, true, false> {
+  typedef _Class volatile& _ClassType;
+  typedef _Rp _ReturnType;
+  typedef _Rp(_FnType)(_Param...);
 };
 
-template <class _Rp, class _Class, class ..._Param>
-struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param..., ...) volatile&, true, false>
-{
-    typedef _Class volatile& _ClassType;
-    typedef _Rp _ReturnType;
-    typedef _Rp (_FnType) (_Param..., ...);
+template <class _Rp, class _Class, class... _Param>
+struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param..., ...) volatile&, true, false> {
+  typedef _Class volatile& _ClassType;
+  typedef _Rp _ReturnType;
+  typedef _Rp(_FnType)(_Param..., ...);
 };
 
-template <class _Rp, class _Class, class ..._Param>
-struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param...) const volatile&, true, false>
-{
-    typedef _Class const volatile& _ClassType;
-    typedef _Rp _ReturnType;
-    typedef _Rp (_FnType) (_Param...);
+template <class _Rp, class _Class, class... _Param>
+struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param...) const volatile&, true, false> {
+  typedef _Class const volatile& _ClassType;
+  typedef _Rp _ReturnType;
+  typedef _Rp(_FnType)(_Param...);
 };
 
-template <class _Rp, class _Class, class ..._Param>
-struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param..., ...) const volatile&, true, false>
-{
-    typedef _Class const volatile& _ClassType;
-    typedef _Rp _ReturnType;
-    typedef _Rp (_FnType) (_Param..., ...);
+template <class _Rp, class _Class, class... _Param>
+struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param..., ...) const volatile&, true, false> {
+  typedef _Class const volatile& _ClassType;
+  typedef _Rp _ReturnType;
+  typedef _Rp(_FnType)(_Param..., ...);
 };
 
-template <class _Rp, class _Class, class ..._Param>
-struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param...) &&, true, false>
-{
-    typedef _Class&& _ClassType;
-    typedef _Rp _ReturnType;
-    typedef _Rp (_FnType) (_Param...);
+template <class _Rp, class _Class, class... _Param>
+struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param...)&&, true, false> {
+  typedef _Class&& _ClassType;
+  typedef _Rp _ReturnType;
+  typedef _Rp(_FnType)(_Param...);
 };
 
-template <class _Rp, class _Class, class ..._Param>
-struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param..., ...) &&, true, false>
-{
-    typedef _Class&& _ClassType;
-    typedef _Rp _ReturnType;
-    typedef _Rp (_FnType) (_Param..., ...);
+template <class _Rp, class _Class, class... _Param>
+struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param..., ...)&&, true, false> {
+  typedef _Class&& _ClassType;
+  typedef _Rp _ReturnType;
+  typedef _Rp(_FnType)(_Param..., ...);
 };
 
-template <class _Rp, class _Class, class ..._Param>
-struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param...) const&&, true, false>
-{
-    typedef _Class const&& _ClassType;
-    typedef _Rp _ReturnType;
-    typedef _Rp (_FnType) (_Param...);
+template <class _Rp, class _Class, class... _Param>
+struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param...) const&&, true, false> {
+  typedef _Class const&& _ClassType;
+  typedef _Rp _ReturnType;
+  typedef _Rp(_FnType)(_Param...);
 };
 
-template <class _Rp, class _Class, class ..._Param>
-struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param..., ...) const&&, true, false>
-{
-    typedef _Class const&& _ClassType;
-    typedef _Rp _ReturnType;
-    typedef _Rp (_FnType) (_Param..., ...);
+template <class _Rp, class _Class, class... _Param>
+struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param..., ...) const&&, true, false> {
+  typedef _Class const&& _ClassType;
+  typedef _Rp _ReturnType;
+  typedef _Rp(_FnType)(_Param..., ...);
 };
 
-template <class _Rp, class _Class, class ..._Param>
-struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param...) volatile&&, true, false>
-{
-    typedef _Class volatile&& _ClassType;
-    typedef _Rp _ReturnType;
-    typedef _Rp (_FnType) (_Param...);
+template <class _Rp, class _Class, class... _Param>
+struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param...) volatile&&, true, false> {
+  typedef _Class volatile&& _ClassType;
+  typedef _Rp _ReturnType;
+  typedef _Rp(_FnType)(_Param...);
 };
 
-template <class _Rp, class _Class, class ..._Param>
-struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param..., ...) volatile&&, true, false>
-{
-    typedef _Class volatile&& _ClassType;
-    typedef _Rp _ReturnType;
-    typedef _Rp (_FnType) (_Param..., ...);
+template <class _Rp, class _Class, class... _Param>
+struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param..., ...) volatile&&, true, false> {
+  typedef _Class volatile&& _ClassType;
+  typedef _Rp _ReturnType;
+  typedef _Rp(_FnType)(_Param..., ...);
 };
 
-template <class _Rp, class _Class, class ..._Param>
-struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param...) const volatile&&, true, false>
-{
-    typedef _Class const volatile&& _ClassType;
-    typedef _Rp _ReturnType;
-    typedef _Rp (_FnType) (_Param...);
+template <class _Rp, class _Class, class... _Param>
+struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param...) const volatile&&, true, false> {
+  typedef _Class const volatile&& _ClassType;
+  typedef _Rp _ReturnType;
+  typedef _Rp(_FnType)(_Param...);
 };
 
-template <class _Rp, class _Class, class ..._Param>
-struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param..., ...) const volatile&&, true, false>
-{
-    typedef _Class const volatile&& _ClassType;
-    typedef _Rp _ReturnType;
-    typedef _Rp (_FnType) (_Param..., ...);
+template <class _Rp, class _Class, class... _Param>
+struct __member_pointer_traits_imp<_Rp (_Class::*)(_Param..., ...) const volatile&&, true, false> {
+  typedef _Class const volatile&& _ClassType;
+  typedef _Rp _ReturnType;
+  typedef _Rp(_FnType)(_Param..., ...);
 };
 
 template <class _Rp, class _Class>
-struct __member_pointer_traits_imp<_Rp _Class::*, false, true>
-{
-    typedef _Class _ClassType;
-    typedef _Rp _ReturnType;
+struct __member_pointer_traits_imp<_Rp _Class::*, false, true> {
+  typedef _Class _ClassType;
+  typedef _Rp _ReturnType;
 };
 
 template <class _MP>
 struct __member_pointer_traits
     : public __member_pointer_traits_imp<__remove_cv_t<_MP>,
-                    is_member_function_pointer<_MP>::value,
-                    is_member_object_pointer<_MP>::value>
-{
-//     typedef ... _ClassType;
-//     typedef ... _ReturnType;
-//     typedef ... _FnType;
+                                         is_member_function_pointer<_MP>::value,
+                                         is_member_object_pointer<_MP>::value> {
+  //     typedef ... _ClassType;
+  //     typedef ... _ReturnType;
+  //     typedef ... _FnType;
 };
 
 template <class _DecayedFp>
@@ -263,268 +234,222 @@ struct __member_pointer_class_type<_Ret _ClassType::*> {
   typedef _ClassType type;
 };
 
-template <class _Fp, class _A0,
-         class _DecayFp = __decay_t<_Fp>,
-         class _DecayA0 = __decay_t<_A0>,
-         class _ClassT = typename __member_pointer_class_type<_DecayFp>::type>
-using __enable_if_bullet1 = typename enable_if
-    <
-        is_member_function_pointer<_DecayFp>::value
-        && is_base_of<_ClassT, _DecayA0>::value
-    >::type;
-
-template <class _Fp, class _A0,
-         class _DecayFp = __decay_t<_Fp>,
-         class _DecayA0 = __decay_t<_A0> >
-using __enable_if_bullet2 = typename enable_if
-    <
-        is_member_function_pointer<_DecayFp>::value
-        && __is_reference_wrapper<_DecayA0>::value
-    >::type;
-
-template <class _Fp, class _A0,
-         class _DecayFp = __decay_t<_Fp>,
-         class _DecayA0 = __decay_t<_A0>,
-         class _ClassT = typename __member_pointer_class_type<_DecayFp>::type>
-using __enable_if_bullet3 = typename enable_if
-    <
-        is_member_function_pointer<_DecayFp>::value
-        && !is_base_of<_ClassT, _DecayA0>::value
-        && !__is_reference_wrapper<_DecayA0>::value
-    >::type;
-
-template <class _Fp, class _A0,
-         class _DecayFp = __decay_t<_Fp>,
-         class _DecayA0 = __decay_t<_A0>,
-         class _ClassT = typename __member_pointer_class_type<_DecayFp>::type>
-using __enable_if_bullet4 = typename enable_if
-    <
-        is_member_object_pointer<_DecayFp>::value
-        && is_base_of<_ClassT, _DecayA0>::value
-    >::type;
-
-template <class _Fp, class _A0,
-         class _DecayFp = __decay_t<_Fp>,
-         class _DecayA0 = __decay_t<_A0> >
-using __enable_if_bullet5 = typename enable_if
-    <
-        is_member_object_pointer<_DecayFp>::value
-        && __is_reference_wrapper<_DecayA0>::value
-    >::type;
-
-template <class _Fp, class _A0,
-         class _DecayFp = __decay_t<_Fp>,
-         class _DecayA0 = __decay_t<_A0>,
-         class _ClassT = typename __member_pointer_class_type<_DecayFp>::type>
-using __enable_if_bullet6 = typename enable_if
-    <
-        is_member_object_pointer<_DecayFp>::value
-        && !is_base_of<_ClassT, _DecayA0>::value
-        && !__is_reference_wrapper<_DecayA0>::value
-    >::type;
+template <class _Fp,
+          class _A0,
+          class _DecayFp = __decay_t<_Fp>,
+          class _DecayA0 = __decay_t<_A0>,
+          class _ClassT  = typename __member_pointer_class_type<_DecayFp>::type>
+using __enable_if_bullet1 =
+    typename enable_if<is_member_function_pointer<_DecayFp>::value && is_base_of<_ClassT, _DecayA0>::value >::type;
+
+template <class _Fp, class _A0, class _DecayFp = __decay_t<_Fp>, class _DecayA0 = __decay_t<_A0> >
+using __enable_if_bullet2 =
+    typename enable_if<is_member_function_pointer<_DecayFp>::value && __is_reference_wrapper<_DecayA0>::value >::type;
+
+template <class _Fp,
+          class _A0,
+          class _DecayFp = __decay_t<_Fp>,
+          class _DecayA0 = __decay_t<_A0>,
+          class _ClassT  = typename __member_pointer_class_type<_DecayFp>::type>
+using __enable_if_bullet3 =
+    typename enable_if<is_member_function_pointer<_DecayFp>::value && !is_base_of<_ClassT, _DecayA0>::value &&
+                       !__is_reference_wrapper<_DecayA0>::value >::type;
+
+template <class _Fp,
+          class _A0,
+          class _DecayFp = __decay_t<_Fp>,
+          class _DecayA0 = __decay_t<_A0>,
+          class _ClassT  = typename __member_pointer_class_type<_DecayFp>::type>
+using __enable_if_bullet4 =
+    typename enable_if<is_member_object_pointer<_DecayFp>::value && is_base_of<_ClassT, _DecayA0>::value >::type;
+
+template <class _Fp, class _A0, class _DecayFp = __decay_t<_Fp>, class _DecayA0 = __decay_t<_A0> >
+using __enable_if_bullet5 =
+    typename enable_if<is_member_object_pointer<_DecayFp>::value && __is_reference_wrapper<_DecayA0>::value >::type;
+
+template <class _Fp,
+          class _A0,
+          class _DecayFp = __decay_t<_Fp>,
+          class _DecayA0 = __decay_t<_A0>,
+          class _ClassT  = typename __member_pointer_class_type<_DecayFp>::type>
+using __enable_if_bullet6 =
+    typename enable_if<is_member_object_pointer<_DecayFp>::value && !is_base_of<_ClassT, _DecayA0>::value &&
+                       !__is_reference_wrapper<_DecayA0>::value >::type;
 
 // __invoke forward declarations
 
 // fall back - none of the bullets
 
-template <class ..._Args>
-__nat __invoke(__any, _Args&& ...__args);
+template <class... _Args>
+__nat __invoke(__any, _Args&&... __args);
 
 // bullets 1, 2 and 3
 
-template <class _Fp, class _A0, class ..._Args,
-          class = __enable_if_bullet1<_Fp, _A0> >
-inline _LIBCPP_INLINE_VISIBILITY
-_LIBCPP_CONSTEXPR decltype((std::declval<_A0>().*std::declval<_Fp>())(std::declval<_Args>()...))
-__invoke(_Fp&& __f, _A0&& __a0, _Args&& ...__args)
+// clang-format off
+template <class _Fp, class _A0, class... _Args, class = __enable_if_bullet1<_Fp, _A0> >
+inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
+decltype((std::declval<_A0>().*std::declval<_Fp>())(std::declval<_Args>()...))
+__invoke(_Fp&& __f, _A0&& __a0, _Args&&... __args)
     _NOEXCEPT_(noexcept((static_cast<_A0&&>(__a0).*__f)(static_cast<_Args&&>(__args)...)))
-    { return           (static_cast<_A0&&>(__a0).*__f)(static_cast<_Args&&>(__args)...); }
+               { return (static_cast<_A0&&>(__a0).*__f)(static_cast<_Args&&>(__args)...); }
 
-template <class _Fp, class _A0, class ..._Args,
-          class = __enable_if_bullet2<_Fp, _A0> >
-inline _LIBCPP_INLINE_VISIBILITY
-_LIBCPP_CONSTEXPR decltype((std::declval<_A0>().get().*std::declval<_Fp>())(std::declval<_Args>()...))
-__invoke(_Fp&& __f, _A0&& __a0, _Args&& ...__args)
+template <class _Fp, class _A0, class... _Args, class = __enable_if_bullet2<_Fp, _A0> >
+inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
+decltype((std::declval<_A0>().get().*std::declval<_Fp>())(std::declval<_Args>()...))
+__invoke(_Fp&& __f, _A0&& __a0, _Args&&... __args)
     _NOEXCEPT_(noexcept((__a0.get().*__f)(static_cast<_Args&&>(__args)...)))
-    { return          (__a0.get().*__f)(static_cast<_Args&&>(__args)...); }
+               { return (__a0.get().*__f)(static_cast<_Args&&>(__args)...); }
 
-template <class _Fp, class _A0, class ..._Args,
-          class = __enable_if_bullet3<_Fp, _A0> >
-inline _LIBCPP_INLINE_VISIBILITY
-_LIBCPP_CONSTEXPR decltype(((*std::declval<_A0>()).*std::declval<_Fp>())(std::declval<_Args>()...))
-__invoke(_Fp&& __f, _A0&& __a0, _Args&& ...__args)
+template <class _Fp, class _A0, class... _Args, class = __enable_if_bullet3<_Fp, _A0> >
+inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
+decltype(((*std::declval<_A0>()).*std::declval<_Fp>())(std::declval<_Args>()...))
+__invoke(_Fp&& __f, _A0&& __a0, _Args&&... __args)
     _NOEXCEPT_(noexcept(((*static_cast<_A0&&>(__a0)).*__f)(static_cast<_Args&&>(__args)...)))
-    { return          ((*static_cast<_A0&&>(__a0)).*__f)(static_cast<_Args&&>(__args)...); }
+               { return ((*static_cast<_A0&&>(__a0)).*__f)(static_cast<_Args&&>(__args)...); }
 
 // bullets 4, 5 and 6
 
-template <class _Fp, class _A0,
-          class = __enable_if_bullet4<_Fp, _A0> >
-inline _LIBCPP_INLINE_VISIBILITY
-_LIBCPP_CONSTEXPR decltype(std::declval<_A0>().*std::declval<_Fp>())
+template <class _Fp, class _A0, class = __enable_if_bullet4<_Fp, _A0> >
+inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
+decltype(std::declval<_A0>().*std::declval<_Fp>())
 __invoke(_Fp&& __f, _A0&& __a0)
     _NOEXCEPT_(noexcept(static_cast<_A0&&>(__a0).*__f))
-    { return          static_cast<_A0&&>(__a0).*__f; }
+               { return static_cast<_A0&&>(__a0).*__f; }
 
-template <class _Fp, class _A0,
-          class = __enable_if_bullet5<_Fp, _A0> >
-inline _LIBCPP_INLINE_VISIBILITY
-_LIBCPP_CONSTEXPR decltype(std::declval<_A0>().get().*std::declval<_Fp>())
+template <class _Fp, class _A0, class = __enable_if_bullet5<_Fp, _A0> >
+inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
+decltype(std::declval<_A0>().get().*std::declval<_Fp>())
 __invoke(_Fp&& __f, _A0&& __a0)
     _NOEXCEPT_(noexcept(__a0.get().*__f))
-    { return          __a0.get().*__f; }
+               { return __a0.get().*__f; }
 
-template <class _Fp, class _A0,
-          class = __enable_if_bullet6<_Fp, _A0> >
-inline _LIBCPP_INLINE_VISIBILITY
-_LIBCPP_CONSTEXPR decltype((*std::declval<_A0>()).*std::declval<_Fp>())
+template <class _Fp, class _A0, class = __enable_if_bullet6<_Fp, _A0> >
+inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
+decltype((*std::declval<_A0>()).*std::declval<_Fp>())
 __invoke(_Fp&& __f, _A0&& __a0)
     _NOEXCEPT_(noexcept((*static_cast<_A0&&>(__a0)).*__f))
-    { return          (*static_cast<_A0&&>(__a0)).*__f; }
+               { return (*static_cast<_A0&&>(__a0)).*__f; }
 
 // bullet 7
 
-template <class _Fp, class ..._Args>
-inline _LIBCPP_INLINE_VISIBILITY
-_LIBCPP_CONSTEXPR decltype(std::declval<_Fp>()(std::declval<_Args>()...))
-__invoke(_Fp&& __f, _Args&& ...__args)
+template <class _Fp, class... _Args>
+inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR
+decltype(std::declval<_Fp>()(std::declval<_Args>()...))
+__invoke(_Fp&& __f, _Args&&... __args)
     _NOEXCEPT_(noexcept(static_cast<_Fp&&>(__f)(static_cast<_Args&&>(__args)...)))
-    { return          static_cast<_Fp&&>(__f)(static_cast<_Args&&>(__args)...); }
+               { return static_cast<_Fp&&>(__f)(static_cast<_Args&&>(__args)...); }
+// clang-format on
 
 // __invokable
-template <class _Ret, class _Fp, class ..._Args>
-struct __invokable_r
-{
-  template <class _XFp, class ..._XArgs>
+template <class _Ret, class _Fp, class... _Args>
+struct __invokable_r {
+  template <class _XFp, class... _XArgs>
   static decltype(std::__invoke(std::declval<_XFp>(), std::declval<_XArgs>()...)) __try_call(int);
-  template <class _XFp, class ..._XArgs>
+  template <class _XFp, class... _XArgs>
   static __nat __try_call(...);
 
   // FIXME: Check that _Ret, _Fp, and _Args... are all complete types, cv void,
   // or incomplete array types as required by the standard.
   using _Result = decltype(__try_call<_Fp, _Args...>(0));
 
-  using type = __conditional_t<
-      _IsNotSame<_Result, __nat>::value,
-      __conditional_t<is_void<_Ret>::value, true_type, __is_core_convertible<_Result, _Ret> >,
-      false_type>;
+  using type              = __conditional_t<_IsNotSame<_Result, __nat>::value,
+                               __conditional_t<is_void<_Ret>::value, true_type, __is_core_convertible<_Result, _Ret> >,
+                               false_type>;
   static const bool value = type::value;
 };
-template <class _Fp, class ..._Args>
+template <class _Fp, class... _Args>
 using __invokable = __invokable_r<void, _Fp, _Args...>;
 
-template <bool _IsInvokable, bool _IsCVVoid, class _Ret, class _Fp, class ..._Args>
+template <bool _IsInvokable, bool _IsCVVoid, class _Ret, class _Fp, class... _Args>
 struct __nothrow_invokable_r_imp {
   static const bool value = false;
 };
 
-template <class _Ret, class _Fp, class ..._Args>
-struct __nothrow_invokable_r_imp<true, false, _Ret, _Fp, _Args...>
-{
-    typedef __nothrow_invokable_r_imp _ThisT;
+template <class _Ret, class _Fp, class... _Args>
+struct __nothrow_invokable_r_imp<true, false, _Ret, _Fp, _Args...> {
+  typedef __nothrow_invokable_r_imp _ThisT;
 
-    template <class _Tp>
-    static void __test_noexcept(_Tp) _NOEXCEPT;
+  template <class _Tp>
+  static void __test_noexcept(_Tp) _NOEXCEPT;
 
 #ifdef _LIBCPP_CXX03_LANG
-    static const bool value = false;
+  static const bool value = false;
 #else
-    static const bool value = noexcept(_ThisT::__test_noexcept<_Ret>(
-        _VSTD::__invoke(std::declval<_Fp>(), std::declval<_Args>()...)));
+  static const bool value =
+      noexcept(_ThisT::__test_noexcept<_Ret>(_VSTD::__invoke(std::declval<_Fp>(), std::declval<_Args>()...)));
 #endif
 };
 
-template <class _Ret, class _Fp, class ..._Args>
-struct __nothrow_invokable_r_imp<true, true, _Ret, _Fp, _Args...>
-{
+template <class _Ret, class _Fp, class... _Args>
+struct __nothrow_invokable_r_imp<true, true, _Ret, _Fp, _Args...> {
 #ifdef _LIBCPP_CXX03_LANG
-    static const bool value = false;
+  static const bool value = false;
 #else
-    static const bool value = noexcept(
-        _VSTD::__invoke(std::declval<_Fp>(), std::declval<_Args>()...));
+  static const bool value = noexcept(_VSTD::__invoke(std::declval<_Fp>(), std::declval<_Args>()...));
 #endif
 };
 
-template <class _Ret, class _Fp, class ..._Args>
+template <class _Ret, class _Fp, class... _Args>
 using __nothrow_invokable_r =
-    __nothrow_invokable_r_imp<
-            __invokable_r<_Ret, _Fp, _Args...>::value,
-            is_void<_Ret>::value,
-            _Ret, _Fp, _Args...
-    >;
-
-template <class _Fp, class ..._Args>
-using __nothrow_invokable =
-    __nothrow_invokable_r_imp<
-            __invokable<_Fp, _Args...>::value,
-            true, void, _Fp, _Args...
-    >;
-
-template <class _Fp, class ..._Args>
+    __nothrow_invokable_r_imp<__invokable_r<_Ret, _Fp, _Args...>::value, is_void<_Ret>::value, _Ret, _Fp, _Args...>;
+
+template <class _Fp, class... _Args>
+using __nothrow_invokable = __nothrow_invokable_r_imp<__invokable<_Fp, _Args...>::value, true, void, _Fp, _Args...>;
+
+template <class _Fp, class... _Args>
 struct __invoke_of
-    : public enable_if<
-        __invokable<_Fp, _Args...>::value,
-        typename __invokable_r<void, _Fp, _Args...>::_Result>
-{
-};
+    : public enable_if<__invokable<_Fp, _Args...>::value, typename __invokable_r<void, _Fp, _Args...>::_Result> {};
 
 template <class _Ret, bool = is_void<_Ret>::value>
-struct __invoke_void_return_wrapper
-{
-    template <class ..._Args>
-    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static _Ret __call(_Args&&... __args) {
-        return std::__invoke(std::forward<_Args>(__args)...);
-    }
+struct __invoke_void_return_wrapper {
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static _Ret __call(_Args&&... __args) {
+    return std::__invoke(std::forward<_Args>(__args)...);
+  }
 };
 
 template <class _Ret>
-struct __invoke_void_return_wrapper<_Ret, true>
-{
-    template <class ..._Args>
-    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static void __call(_Args&&... __args) {
-        std::__invoke(std::forward<_Args>(__args)...);
-    }
+struct __invoke_void_return_wrapper<_Ret, true> {
+  template <class... _Args>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static void __call(_Args&&... __args) {
+    std::__invoke(std::forward<_Args>(__args)...);
+  }
 };
 
 #if _LIBCPP_STD_VER >= 17
 
 // is_invocable
 
-template <class _Fn, class ..._Args>
-struct _LIBCPP_TEMPLATE_VIS is_invocable
-    : integral_constant<bool, __invokable<_Fn, _Args...>::value> {};
+template <class _Fn, class... _Args>
+struct _LIBCPP_TEMPLATE_VIS is_invocable : integral_constant<bool, __invokable<_Fn, _Args...>::value> {};
 
-template <class _Ret, class _Fn, class ..._Args>
-struct _LIBCPP_TEMPLATE_VIS is_invocable_r
-    : integral_constant<bool, __invokable_r<_Ret, _Fn, _Args...>::value> {};
+template <class _Ret, class _Fn, class... _Args>
+struct _LIBCPP_TEMPLATE_VIS is_invocable_r : integral_constant<bool, __invokable_r<_Ret, _Fn, _Args...>::value> {};
 
-template <class _Fn, class ..._Args>
+template <class _Fn, class... _Args>
 inline constexpr bool is_invocable_v = is_invocable<_Fn, _Args...>::value;
 
-template <class _Ret, class _Fn, class ..._Args>
+template <class _Ret, class _Fn, class... _Args>
 inline constexpr bool is_invocable_r_v = is_invocable_r<_Ret, _Fn, _Args...>::value;
 
 // is_nothrow_invocable
 
-template <class _Fn, class ..._Args>
-struct _LIBCPP_TEMPLATE_VIS is_nothrow_invocable
-    : integral_constant<bool, __nothrow_invokable<_Fn, _Args...>::value> {};
+template <class _Fn, class... _Args>
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_invocable : integral_constant<bool, __nothrow_invokable<_Fn, _Args...>::value> {
+};
 
-template <class _Ret, class _Fn, class ..._Args>
+template <class _Ret, class _Fn, class... _Args>
 struct _LIBCPP_TEMPLATE_VIS is_nothrow_invocable_r
     : integral_constant<bool, __nothrow_invokable_r<_Ret, _Fn, _Args...>::value> {};
 
-template <class _Fn, class ..._Args>
+template <class _Fn, class... _Args>
 inline constexpr bool is_nothrow_invocable_v = is_nothrow_invocable<_Fn, _Args...>::value;
 
-template <class _Ret, class _Fn, class ..._Args>
+template <class _Ret, class _Fn, class... _Args>
 inline constexpr bool is_nothrow_invocable_r_v = is_nothrow_invocable_r<_Ret, _Fn, _Args...>::value;
 
 template <class _Fn, class... _Args>
-struct _LIBCPP_TEMPLATE_VIS invoke_result
-    : __invoke_of<_Fn, _Args...>
-{
-};
+struct _LIBCPP_TEMPLATE_VIS invoke_result : __invoke_of<_Fn, _Args...> {};
 
 template <class _Fn, class... _Args>
 using invoke_result_t = typename invoke_result<_Fn, _Args...>::type;
diff --git a/libcxx/include/__type_traits/is_abstract.h b/libcxx/include/__type_traits/is_abstract.h
index 5e886bb8d6d93..4aa456be1c48e 100644
--- a/libcxx/include/__type_traits/is_abstract.h
+++ b/libcxx/include/__type_traits/is_abstract.h
@@ -18,8 +18,8 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_abstract
-    : public integral_constant<bool, __is_abstract(_Tp)> {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_abstract : public integral_constant<bool, __is_abstract(_Tp)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/is_aggregate.h b/libcxx/include/__type_traits/is_aggregate.h
index 3b0942048cf34..4e0988071adee 100644
--- a/libcxx/include/__type_traits/is_aggregate.h
+++ b/libcxx/include/__type_traits/is_aggregate.h
@@ -20,8 +20,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if _LIBCPP_STD_VER >= 17
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS
-is_aggregate : public integral_constant<bool, __is_aggregate(_Tp)> {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_aggregate : public integral_constant<bool, __is_aggregate(_Tp)> {};
 
 template <class _Tp>
 inline constexpr bool is_aggregate_v = __is_aggregate(_Tp);
diff --git a/libcxx/include/__type_traits/is_allocator.h b/libcxx/include/__type_traits/is_allocator.h
index ee4154d7e4a43..144ffac4d7ce5 100644
--- a/libcxx/include/__type_traits/is_allocator.h
+++ b/libcxx/include/__type_traits/is_allocator.h
@@ -21,15 +21,13 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template<typename _Alloc, typename = void, typename = void>
+template <typename _Alloc, typename = void, typename = void>
 struct __is_allocator : false_type {};
 
-template<typename _Alloc>
+template <typename _Alloc>
 struct __is_allocator<_Alloc,
-       __void_t<typename _Alloc::value_type>,
-       __void_t<decltype(std::declval<_Alloc&>().allocate(size_t(0)))>
-     >
-   : true_type {};
+                      __void_t<typename _Alloc::value_type>,
+                      __void_t<decltype(std::declval<_Alloc&>().allocate(size_t(0)))> > : true_type {};
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__type_traits/is_always_bitcastable.h b/libcxx/include/__type_traits/is_always_bitcastable.h
index 63304eb492b7c..5bc650b41358a 100644
--- a/libcxx/include/__type_traits/is_always_bitcastable.h
+++ b/libcxx/include/__type_traits/is_always_bitcastable.h
@@ -32,49 +32,52 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 template <class _From, class _To>
 struct __is_always_bitcastable {
   using _UnqualFrom = __remove_cv_t<_From>;
-  using _UnqualTo = __remove_cv_t<_To>;
+  using _UnqualTo   = __remove_cv_t<_To>;
 
+  // clang-format off
   static const bool value =
-    // First, the simple case -- `From` and `To` are the same object type.
-    (is_same<_UnqualFrom, _UnqualTo>::value && is_trivially_copyable<_UnqualFrom>::value) ||
+      // First, the simple case -- `From` and `To` are the same object type.
+      (is_same<_UnqualFrom, _UnqualTo>::value && is_trivially_copyable<_UnqualFrom>::value) ||
 
-    // Beyond the simple case, we say that one type is "always bit-castable" to another if:
-    // - (1) `From` and `To` have the same value representation, and in addition every possible value of `From` has
-    //   a corresponding value in the `To` type (in other words, the set of values of `To` is a superset of the set of
-    //   values of `From`);
-    // - (2) When the corresponding values are not the same value (as, for example, between an unsigned and a signed
-    //   integer, where a large positive value of the unsigned integer corresponds to a negative value in the signed
-    //   integer type), the value of `To` that results from a bitwise copy of `From` is the same what would be produced
-    //   by the built-in assignment (if it were defined for the two types, to which there are minor exceptions, e.g.
-    //   built-in arrays).
-    //
-    // In practice, that means:
-    // - all integral types (except `bool`, see below) -- that is, character types and `int` types, both signed and
-    //   unsigned...
-    // - as well as arrays of such types...
-    // - ...that have the same size.
-    //
-    // Other trivially-copyable types can't be validly bit-cast outside of their own type:
-    // - floating-point types normally have different sizes and thus aren't bit-castable between each other (fails #1);
-    // - integral types and floating-point types use different representations, so for example bit-casting an integral
-    //   `1` to `float` results in a very small less-than-one value, unlike built-in assignment that produces `1.0`
-    //   (fails #2);
-    // - booleans normally use only a single bit of their object representation; bit-casting an integer to a boolean
-    //   will result in a boolean object with an incorrect representation, which is undefined behavior (fails #2).
-    //   Bit-casting from a boolean into an integer, however, is valid;
-    // - enumeration types may have different ranges of possible values (fails #1);
-    // - for pointers, it is not guaranteed that pointers to different types use the same set of values to represent
-    //   addresses, and the conversion results are explicitly unspecified for types with different alignments
-    //   (fails #1);
-    // - for structs and unions it is impossible to determine whether the set of values of one of them is a subset of
-    //   the other (fails #1);
-    // - there is no need to consider `nullptr_t` for practical purposes.
-    (
-      sizeof(_From) == sizeof(_To) &&
-      is_integral<_From>::value &&
-      is_integral<_To>::value &&
-      !is_same<_UnqualTo, bool>::value
-    );
+      // Beyond the simple case, we say that one type is "always bit-castable" to another if:
+      // - (1) `From` and `To` have the same value representation, and in addition every possible value of `From` has
+      //   a corresponding value in the `To` type (in other words, the set of values of `To` is a superset of the set of
+      //   values of `From`);
+      // - (2) When the corresponding values are not the same value (as, for example, between an unsigned and a signed
+      //   integer, where a large positive value of the unsigned integer corresponds to a negative value in the signed
+      //   integer type), the value of `To` that results from a bitwise copy of `From` is the same what would be
+      //   produced by the built-in assignment (if it were defined for the two types, to which there are minor
+      //   exceptions, e.g. built-in arrays).
+      //
+      // In practice, that means:
+      // - all integral types (except `bool`, see below) -- that is, character types and `int` types, both signed and
+      //   unsigned...
+      // - as well as arrays of such types...
+      // - ...that have the same size.
+      //
+      // Other trivially-copyable types can't be validly bit-cast outside of their own type:
+      // - floating-point types normally have different sizes and thus aren't bit-castable between each other (fails
+      // #1);
+      // - integral types and floating-point types use different representations, so for example bit-casting an integral
+      //   `1` to `float` results in a very small less-than-one value, unlike built-in assignment that produces `1.0`
+      //   (fails #2);
+      // - booleans normally use only a single bit of their object representation; bit-casting an integer to a boolean
+      //   will result in a boolean object with an incorrect representation, which is undefined behavior (fails #2).
+      //   Bit-casting from a boolean into an integer, however, is valid;
+      // - enumeration types may have different ranges of possible values (fails #1);
+      // - for pointers, it is not guaranteed that pointers to different types use the same set of values to represent
+      //   addresses, and the conversion results are explicitly unspecified for types with different alignments
+      //   (fails #1);
+      // - for structs and unions it is impossible to determine whether the set of values of one of them is a subset of
+      //   the other (fails #1);
+      // - there is no need to consider `nullptr_t` for practical purposes.
+      (
+        sizeof(_From) == sizeof(_To) &&
+        is_integral<_From>::value &&
+        is_integral<_To>::value &&
+        !is_same<_UnqualTo, bool>::value
+      );
+  // clang-format on
 };
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/is_arithmetic.h b/libcxx/include/__type_traits/is_arithmetic.h
index 03218c8169965..c9713e1840a7b 100644
--- a/libcxx/include/__type_traits/is_arithmetic.h
+++ b/libcxx/include/__type_traits/is_arithmetic.h
@@ -20,9 +20,9 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_arithmetic
-    : public integral_constant<bool, is_integral<_Tp>::value      ||
-                                     is_floating_point<_Tp>::value> {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_arithmetic
+    : public integral_constant<bool, is_integral<_Tp>::value || is_floating_point<_Tp>::value> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/is_array.h b/libcxx/include/__type_traits/is_array.h
index 827517666c373..5b0e0e6f46b81 100644
--- a/libcxx/include/__type_traits/is_array.h
+++ b/libcxx/include/__type_traits/is_array.h
@@ -24,26 +24,26 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 #if __has_builtin(__is_array) && 0
 
 template <class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_array : _BoolConstant<__is_array(_Tp)> { };
+struct _LIBCPP_TEMPLATE_VIS is_array : _BoolConstant<__is_array(_Tp)> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_array_v = __is_array(_Tp);
-#endif
+#  endif
 
 #else
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_array
-    : public false_type {};
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_array<_Tp[]>
-    : public true_type {};
-template <class _Tp, size_t _Np> struct _LIBCPP_TEMPLATE_VIS is_array<_Tp[_Np]>
-    : public true_type {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_array : public false_type {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_array<_Tp[]> : public true_type {};
+template <class _Tp, size_t _Np>
+struct _LIBCPP_TEMPLATE_VIS is_array<_Tp[_Np]> : public true_type {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_array_v = is_array<_Tp>::value;
-#endif
+#  endif
 
 #endif // __has_builtin(__is_array)
 
diff --git a/libcxx/include/__type_traits/is_assignable.h b/libcxx/include/__type_traits/is_assignable.h
index 3ee3e21bf97a1..11134b1c1abaf 100644
--- a/libcxx/include/__type_traits/is_assignable.h
+++ b/libcxx/include/__type_traits/is_assignable.h
@@ -18,8 +18,8 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template<class _Tp, class _Up>
-struct _LIBCPP_TEMPLATE_VIS is_assignable : _BoolConstant<__is_assignable(_Tp, _Up)> { };
+template <class _Tp, class _Up>
+struct _LIBCPP_TEMPLATE_VIS is_assignable : _BoolConstant<__is_assignable(_Tp, _Up)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp, class _Arg>
diff --git a/libcxx/include/__type_traits/is_base_of.h b/libcxx/include/__type_traits/is_base_of.h
index 3338fffe23de1..090abeeb54dcc 100644
--- a/libcxx/include/__type_traits/is_base_of.h
+++ b/libcxx/include/__type_traits/is_base_of.h
@@ -19,8 +19,7 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Bp, class _Dp>
-struct _LIBCPP_TEMPLATE_VIS is_base_of
-    : public integral_constant<bool, __is_base_of(_Bp, _Dp)> {};
+struct _LIBCPP_TEMPLATE_VIS is_base_of : public integral_constant<bool, __is_base_of(_Bp, _Dp)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Bp, class _Dp>
diff --git a/libcxx/include/__type_traits/is_bounded_array.h b/libcxx/include/__type_traits/is_bounded_array.h
index 46da83a5654a7..211403d638d08 100644
--- a/libcxx/include/__type_traits/is_bounded_array.h
+++ b/libcxx/include/__type_traits/is_bounded_array.h
@@ -19,17 +19,20 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class>                 struct _LIBCPP_TEMPLATE_VIS __libcpp_is_bounded_array           : false_type {};
-template <class _Tp, size_t _Np> struct _LIBCPP_TEMPLATE_VIS __libcpp_is_bounded_array<_Tp[_Np]> : true_type {};
+template <class>
+struct _LIBCPP_TEMPLATE_VIS __libcpp_is_bounded_array : false_type {};
+template <class _Tp, size_t _Np>
+struct _LIBCPP_TEMPLATE_VIS __libcpp_is_bounded_array<_Tp[_Np]> : true_type {};
 
 #if _LIBCPP_STD_VER >= 20
 
-template <class>                 struct _LIBCPP_TEMPLATE_VIS is_bounded_array           : false_type {};
-template <class _Tp, size_t _Np> struct _LIBCPP_TEMPLATE_VIS is_bounded_array<_Tp[_Np]> : true_type {};
+template <class>
+struct _LIBCPP_TEMPLATE_VIS is_bounded_array : false_type {};
+template <class _Tp, size_t _Np>
+struct _LIBCPP_TEMPLATE_VIS is_bounded_array<_Tp[_Np]> : true_type {};
 
 template <class _Tp>
-inline constexpr
-bool is_bounded_array_v  = is_bounded_array<_Tp>::value;
+inline constexpr bool is_bounded_array_v = is_bounded_array<_Tp>::value;
 
 #endif
 
diff --git a/libcxx/include/__type_traits/is_callable.h b/libcxx/include/__type_traits/is_callable.h
index 445373c43b9d4..49724fe892ee5 100644
--- a/libcxx/include/__type_traits/is_callable.h
+++ b/libcxx/include/__type_traits/is_callable.h
@@ -19,12 +19,12 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template<class _Func, class... _Args, class = decltype(std::declval<_Func>()(std::declval<_Args>()...))>
+template <class _Func, class... _Args, class = decltype(std::declval<_Func>()(std::declval<_Args>()...))>
 true_type __is_callable_helper(int);
-template<class...>
+template <class...>
 false_type __is_callable_helper(...);
 
-template<class _Func, class... _Args>
+template <class _Func, class... _Args>
 struct __is_callable : decltype(std::__is_callable_helper<_Func, _Args...>(0)) {};
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/is_class.h b/libcxx/include/__type_traits/is_class.h
index 62bf6d9cfa0e8..034f76a7865e3 100644
--- a/libcxx/include/__type_traits/is_class.h
+++ b/libcxx/include/__type_traits/is_class.h
@@ -18,8 +18,8 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_class
-    : public integral_constant<bool, __is_class(_Tp)> {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_class : public integral_constant<bool, __is_class(_Tp)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/is_compound.h b/libcxx/include/__type_traits/is_compound.h
index 73d830c80aff0..cd208ceab2886 100644
--- a/libcxx/include/__type_traits/is_compound.h
+++ b/libcxx/include/__type_traits/is_compound.h
@@ -21,23 +21,23 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if __has_builtin(__is_compound)
 
-template<class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_compound : _BoolConstant<__is_compound(_Tp)> { };
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_compound : _BoolConstant<__is_compound(_Tp)> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_compound_v = __is_compound(_Tp);
-#endif
+#  endif
 
 #else // __has_builtin(__is_compound)
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_compound
-    : public integral_constant<bool, !is_fundamental<_Tp>::value> {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_compound : public integral_constant<bool, !is_fundamental<_Tp>::value> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_compound_v = is_compound<_Tp>::value;
-#endif
+#  endif
 
 #endif // __has_builtin(__is_compound)
 
diff --git a/libcxx/include/__type_traits/is_const.h b/libcxx/include/__type_traits/is_const.h
index fad3602862019..47ef70872b790 100644
--- a/libcxx/include/__type_traits/is_const.h
+++ b/libcxx/include/__type_traits/is_const.h
@@ -21,22 +21,24 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 #if __has_builtin(__is_const)
 
 template <class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_const : _BoolConstant<__is_const(_Tp)> { };
+struct _LIBCPP_TEMPLATE_VIS is_const : _BoolConstant<__is_const(_Tp)> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_const_v = __is_const(_Tp);
-#endif
+#  endif
 
 #else
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_const            : public false_type {};
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_const<_Tp const> : public true_type {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_const : public false_type {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_const<_Tp const> : public true_type {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_const_v = is_const<_Tp>::value;
-#endif
+#  endif
 
 #endif // __has_builtin(__is_const)
 
diff --git a/libcxx/include/__type_traits/is_constant_evaluated.h b/libcxx/include/__type_traits/is_constant_evaluated.h
index d6278f521ab04..d7af462486e13 100644
--- a/libcxx/include/__type_traits/is_constant_evaluated.h
+++ b/libcxx/include/__type_traits/is_constant_evaluated.h
@@ -18,14 +18,14 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if _LIBCPP_STD_VER >= 20
-_LIBCPP_INLINE_VISIBILITY
-inline constexpr bool is_constant_evaluated() noexcept {
+_LIBCPP_INLINE_VISIBILITY inline constexpr bool is_constant_evaluated() noexcept {
   return __builtin_is_constant_evaluated();
 }
 #endif
 
-_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR
-bool __libcpp_is_constant_evaluated() _NOEXCEPT { return __builtin_is_constant_evaluated(); }
+_LIBCPP_HIDE_FROM_ABI inline _LIBCPP_CONSTEXPR bool __libcpp_is_constant_evaluated() _NOEXCEPT {
+  return __builtin_is_constant_evaluated();
+}
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__type_traits/is_constructible.h b/libcxx/include/__type_traits/is_constructible.h
index ebe7bfd373f17..4e62eb061fd59 100644
--- a/libcxx/include/__type_traits/is_constructible.h
+++ b/libcxx/include/__type_traits/is_constructible.h
@@ -18,13 +18,11 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp, class ..._Args>
-struct _LIBCPP_TEMPLATE_VIS is_constructible
-    : public integral_constant<bool, __is_constructible(_Tp, _Args...)>
-{ };
+template <class _Tp, class... _Args>
+struct _LIBCPP_TEMPLATE_VIS is_constructible : public integral_constant<bool, __is_constructible(_Tp, _Args...)> {};
 
 #if _LIBCPP_STD_VER >= 17
-template <class _Tp, class ..._Args>
+template <class _Tp, class... _Args>
 inline constexpr bool is_constructible_v = __is_constructible(_Tp, _Args...);
 #endif
 
diff --git a/libcxx/include/__type_traits/is_convertible.h b/libcxx/include/__type_traits/is_convertible.h
index 749d6fd47af9c..b378a20cedeb6 100644
--- a/libcxx/include/__type_traits/is_convertible.h
+++ b/libcxx/include/__type_traits/is_convertible.h
@@ -31,76 +31,75 @@ struct _LIBCPP_TEMPLATE_VIS is_convertible : public integral_constant<bool, __is
 
 #elif __has_builtin(__is_convertible_to) && !defined(_LIBCPP_USE_IS_CONVERTIBLE_FALLBACK)
 
-template <class _T1, class _T2> struct _LIBCPP_TEMPLATE_VIS is_convertible
-    : public integral_constant<bool, __is_convertible_to(_T1, _T2)> {};
+template <class _T1, class _T2>
+struct _LIBCPP_TEMPLATE_VIS is_convertible : public integral_constant<bool, __is_convertible_to(_T1, _T2)> {};
 
 // TODO: Remove this fallback when GCC < 13 support is no longer required.
 // GCC 13 has the __is_convertible built-in.
-#else  // __has_builtin(__is_convertible_to) && !defined(_LIBCPP_USE_IS_CONVERTIBLE_FALLBACK)
+#else // __has_builtin(__is_convertible_to) && !defined(_LIBCPP_USE_IS_CONVERTIBLE_FALLBACK)
 
-namespace __is_convertible_imp
-{
-template <class _Tp> void  __test_convert(_Tp);
+namespace __is_convertible_imp {
+template <class _Tp>
+void __test_convert(_Tp);
 
 template <class _From, class _To, class = void>
 struct __is_convertible_test : public false_type {};
 
 template <class _From, class _To>
-struct __is_convertible_test<_From, _To,
-    decltype(__is_convertible_imp::__test_convert<_To>(std::declval<_From>()))> : public true_type
-{};
-
-template <class _Tp, bool _IsArray =    is_array<_Tp>::value,
-                     bool _IsFunction = is_function<_Tp>::value,
-                     bool _IsVoid =     is_void<_Tp>::value>
-                     struct __is_array_function_or_void                          {enum {value = 0};};
-template <class _Tp> struct __is_array_function_or_void<_Tp, true, false, false> {enum {value = 1};};
-template <class _Tp> struct __is_array_function_or_void<_Tp, false, true, false> {enum {value = 2};};
-template <class _Tp> struct __is_array_function_or_void<_Tp, false, false, true> {enum {value = 3};};
-}
+struct __is_convertible_test<_From, _To, decltype(__is_convertible_imp::__test_convert<_To>(std::declval<_From>()))>
+    : public true_type {};
 
+// clang-format off
 template <class _Tp,
-    unsigned = __is_convertible_imp::__is_array_function_or_void<__libcpp_remove_reference_t<_Tp> >::value>
-struct __is_convertible_check
-{
-    static const size_t __v = 0;
+          bool _IsArray    = is_array<_Tp>::value,
+          bool _IsFunction = is_function<_Tp>::value,
+          bool _IsVoid     = is_void<_Tp>::value>
+                     struct __is_array_function_or_void                          { enum { value = 0 }; };
+template <class _Tp> struct __is_array_function_or_void<_Tp, true, false, false> { enum { value = 1 }; };
+template <class _Tp> struct __is_array_function_or_void<_Tp, false, true, false> { enum { value = 2 }; };
+template <class _Tp> struct __is_array_function_or_void<_Tp, false, false, true> { enum { value = 3 }; };
+// clang-format on
+} // namespace __is_convertible_imp
+
+template <class _Tp,
+          unsigned = __is_convertible_imp::__is_array_function_or_void<__libcpp_remove_reference_t<_Tp> >::value>
+struct __is_convertible_check {
+  static const size_t __v = 0;
 };
 
 template <class _Tp>
-struct __is_convertible_check<_Tp, 0>
-{
-    static const size_t __v = sizeof(_Tp);
+struct __is_convertible_check<_Tp, 0> {
+  static const size_t __v = sizeof(_Tp);
 };
 
-template <class _T1, class _T2,
-    unsigned _T1_is_array_function_or_void = __is_convertible_imp::__is_array_function_or_void<_T1>::value,
-    unsigned _T2_is_array_function_or_void = __is_convertible_imp::__is_array_function_or_void<_T2>::value>
+template <class _T1,
+          class _T2,
+          unsigned _T1_is_array_function_or_void = __is_convertible_imp::__is_array_function_or_void<_T1>::value,
+          unsigned _T2_is_array_function_or_void = __is_convertible_imp::__is_array_function_or_void<_T2>::value>
 struct __is_convertible
-    : public integral_constant<bool,
-        __is_convertible_imp::__is_convertible_test<_T1, _T2>::value
-    >
-{};
-
-template <class _T1, class _T2> struct __is_convertible<_T1, _T2, 0, 1> : public false_type {};
-template <class _T1, class _T2> struct __is_convertible<_T1, _T2, 1, 1> : public false_type {};
-template <class _T1, class _T2> struct __is_convertible<_T1, _T2, 2, 1> : public false_type {};
-template <class _T1, class _T2> struct __is_convertible<_T1, _T2, 3, 1> : public false_type {};
-
-template <class _T1, class _T2> struct __is_convertible<_T1, _T2, 0, 2> : public false_type {};
-template <class _T1, class _T2> struct __is_convertible<_T1, _T2, 1, 2> : public false_type {};
-template <class _T1, class _T2> struct __is_convertible<_T1, _T2, 2, 2> : public false_type {};
-template <class _T1, class _T2> struct __is_convertible<_T1, _T2, 3, 2> : public false_type {};
-
-template <class _T1, class _T2> struct __is_convertible<_T1, _T2, 0, 3> : public false_type {};
-template <class _T1, class _T2> struct __is_convertible<_T1, _T2, 1, 3> : public false_type {};
-template <class _T1, class _T2> struct __is_convertible<_T1, _T2, 2, 3> : public false_type {};
-template <class _T1, class _T2> struct __is_convertible<_T1, _T2, 3, 3> : public true_type {};
-
-template <class _T1, class _T2> struct _LIBCPP_TEMPLATE_VIS is_convertible
-    : public __is_convertible<_T1, _T2>
-{
-    static const size_t __complete_check1 = __is_convertible_check<_T1>::__v;
-    static const size_t __complete_check2 = __is_convertible_check<_T2>::__v;
+    : public integral_constant<bool, __is_convertible_imp::__is_convertible_test<_T1, _T2>::value >{};
+
+// clang-format off
+template <class _T1, class _T2> struct __is_convertible<_T1, _T2, 0, 1> : public false_type{};
+template <class _T1, class _T2> struct __is_convertible<_T1, _T2, 1, 1> : public false_type{};
+template <class _T1, class _T2> struct __is_convertible<_T1, _T2, 2, 1> : public false_type{};
+template <class _T1, class _T2> struct __is_convertible<_T1, _T2, 3, 1> : public false_type{};
+
+template <class _T1, class _T2> struct __is_convertible<_T1, _T2, 0, 2> : public false_type{};
+template <class _T1, class _T2> struct __is_convertible<_T1, _T2, 1, 2> : public false_type{};
+template <class _T1, class _T2> struct __is_convertible<_T1, _T2, 2, 2> : public false_type{};
+template <class _T1, class _T2> struct __is_convertible<_T1, _T2, 3, 2> : public false_type{};
+
+template <class _T1, class _T2> struct __is_convertible<_T1, _T2, 0, 3> : public false_type{};
+template <class _T1, class _T2> struct __is_convertible<_T1, _T2, 1, 3> : public false_type{};
+template <class _T1, class _T2> struct __is_convertible<_T1, _T2, 2, 3> : public false_type{};
+template <class _T1, class _T2> struct __is_convertible<_T1, _T2, 3, 3> : public true_type{};
+// clang-format on
+
+template <class _T1, class _T2>
+struct _LIBCPP_TEMPLATE_VIS is_convertible : public __is_convertible<_T1, _T2> {
+  static const size_t __complete_check1 = __is_convertible_check<_T1>::__v;
+  static const size_t __complete_check2 = __is_convertible_check<_T2>::__v;
 };
 
 #endif // __has_builtin(__is_convertible_to) && !defined(_LIBCPP_USE_IS_CONVERTIBLE_FALLBACK)
diff --git a/libcxx/include/__type_traits/is_copy_assignable.h b/libcxx/include/__type_traits/is_copy_assignable.h
index 245ae4b28912d..e607ace540437 100644
--- a/libcxx/include/__type_traits/is_copy_assignable.h
+++ b/libcxx/include/__type_traits/is_copy_assignable.h
@@ -24,8 +24,7 @@ template <class _Tp>
 struct _LIBCPP_TEMPLATE_VIS is_copy_assignable
     : public integral_constant<
           bool,
-          __is_assignable(__add_lvalue_reference_t<_Tp>,
-                          __add_lvalue_reference_t<typename add_const<_Tp>::type>)> {};
+          __is_assignable(__add_lvalue_reference_t<_Tp>, __add_lvalue_reference_t<typename add_const<_Tp>::type>)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/is_copy_constructible.h b/libcxx/include/__type_traits/is_copy_constructible.h
index 07e2c93ecd0c8..402f2b89875bb 100644
--- a/libcxx/include/__type_traits/is_copy_constructible.h
+++ b/libcxx/include/__type_traits/is_copy_constructible.h
@@ -22,9 +22,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp>
 struct _LIBCPP_TEMPLATE_VIS is_copy_constructible
-    : public integral_constant<
-          bool,
-          __is_constructible(_Tp, __add_lvalue_reference_t<typename add_const<_Tp>::type>)> {};
+    : public integral_constant<bool, __is_constructible(_Tp, __add_lvalue_reference_t<typename add_const<_Tp>::type>)> {
+};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/is_core_convertible.h b/libcxx/include/__type_traits/is_core_convertible.h
index 1ce4079130524..0de177c7771f4 100644
--- a/libcxx/include/__type_traits/is_core_convertible.h
+++ b/libcxx/include/__type_traits/is_core_convertible.h
@@ -27,9 +27,8 @@ template <class _Tp, class _Up, class = void>
 struct __is_core_convertible : public false_type {};
 
 template <class _Tp, class _Up>
-struct __is_core_convertible<_Tp, _Up, decltype(
-    static_cast<void(*)(_Up)>(0) ( static_cast<_Tp(*)()>(0)() )
-)> : public true_type {};
+struct __is_core_convertible<_Tp, _Up, decltype(static_cast<void (*)(_Up)>(0)(static_cast<_Tp (*)()>(0)()))>
+    : public true_type {};
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__type_traits/is_default_constructible.h b/libcxx/include/__type_traits/is_default_constructible.h
index 7bc21f5724d18..e73e9b98f5dca 100644
--- a/libcxx/include/__type_traits/is_default_constructible.h
+++ b/libcxx/include/__type_traits/is_default_constructible.h
@@ -19,9 +19,7 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_default_constructible
-    : public integral_constant<bool, __is_constructible(_Tp)>
-    {};
+struct _LIBCPP_TEMPLATE_VIS is_default_constructible : public integral_constant<bool, __is_constructible(_Tp)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/is_destructible.h b/libcxx/include/__type_traits/is_destructible.h
index d785b4b47301b..3248b07d36ee6 100644
--- a/libcxx/include/__type_traits/is_destructible.h
+++ b/libcxx/include/__type_traits/is_destructible.h
@@ -24,13 +24,13 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if __has_builtin(__is_destructible)
 
-template<class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_destructible : _BoolConstant<__is_destructible(_Tp)> { };
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_destructible : _BoolConstant<__is_destructible(_Tp)> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_destructible_v = __is_destructible(_Tp);
-#endif
+#  endif
 
 #else // __has_builtin(__is_destructible)
 
@@ -42,19 +42,19 @@ inline constexpr bool is_destructible_v = __is_destructible(_Tp);
 //    where _Up is remove_all_extents<_Tp>::type
 
 template <class>
-struct __is_destructible_apply { typedef int type; };
+struct __is_destructible_apply {
+  typedef int type;
+};
 
 template <typename _Tp>
 struct __is_destructor_wellformed {
-    template <typename _Tp1>
-    static true_type  __test (
-        typename __is_destructible_apply<decltype(std::declval<_Tp1&>().~_Tp1())>::type
-    );
+  template <typename _Tp1>
+  static true_type __test(typename __is_destructible_apply<decltype(std::declval<_Tp1&>().~_Tp1())>::type);
 
-    template <typename _Tp1>
-    static false_type __test (...);
+  template <typename _Tp1>
+  static false_type __test(...);
 
-    static const bool value = decltype(__test<_Tp>(12))::value;
+  static const bool value = decltype(__test<_Tp>(12))::value;
 };
 
 template <class _Tp, bool>
@@ -62,12 +62,10 @@ struct __destructible_imp;
 
 template <class _Tp>
 struct __destructible_imp<_Tp, false>
-   : public integral_constant<bool,
-        __is_destructor_wellformed<__remove_all_extents_t<_Tp> >::value> {};
+    : public integral_constant<bool, __is_destructor_wellformed<__remove_all_extents_t<_Tp> >::value> {};
 
 template <class _Tp>
-struct __destructible_imp<_Tp, true>
-    : public true_type {};
+struct __destructible_imp<_Tp, true> : public true_type {};
 
 template <class _Tp, bool>
 struct __destructible_false;
@@ -79,21 +77,18 @@ template <class _Tp>
 struct __destructible_false<_Tp, true> : public false_type {};
 
 template <class _Tp>
-struct is_destructible
-    : public __destructible_false<_Tp, is_function<_Tp>::value> {};
+struct is_destructible : public __destructible_false<_Tp, is_function<_Tp>::value> {};
 
 template <class _Tp>
-struct is_destructible<_Tp[]>
-    : public false_type {};
+struct is_destructible<_Tp[]> : public false_type {};
 
 template <>
-struct is_destructible<void>
-    : public false_type {};
+struct is_destructible<void> : public false_type {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_destructible_v = is_destructible<_Tp>::value;
-#endif
+#  endif
 
 #endif // __has_builtin(__is_destructible)
 
diff --git a/libcxx/include/__type_traits/is_empty.h b/libcxx/include/__type_traits/is_empty.h
index 46518e19086b1..951d93b5a2f10 100644
--- a/libcxx/include/__type_traits/is_empty.h
+++ b/libcxx/include/__type_traits/is_empty.h
@@ -19,8 +19,7 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_empty
-    : public integral_constant<bool, __is_empty(_Tp)> {};
+struct _LIBCPP_TEMPLATE_VIS is_empty : public integral_constant<bool, __is_empty(_Tp)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/is_enum.h b/libcxx/include/__type_traits/is_enum.h
index 7b7221d40e506..77ca3ea108742 100644
--- a/libcxx/include/__type_traits/is_enum.h
+++ b/libcxx/include/__type_traits/is_enum.h
@@ -18,8 +18,8 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_enum
-    : public integral_constant<bool, __is_enum(_Tp)> {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_enum : public integral_constant<bool, __is_enum(_Tp)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/is_final.h b/libcxx/include/__type_traits/is_final.h
index ad03b213b06e3..499c5e3a1edca 100644
--- a/libcxx/include/__type_traits/is_final.h
+++ b/libcxx/include/__type_traits/is_final.h
@@ -18,12 +18,12 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS
-__libcpp_is_final : public integral_constant<bool, __is_final(_Tp)> {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS __libcpp_is_final : public integral_constant<bool, __is_final(_Tp)> {};
 
 #if _LIBCPP_STD_VER >= 14
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS
-is_final : public integral_constant<bool, __is_final(_Tp)> {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_final : public integral_constant<bool, __is_final(_Tp)> {};
 #endif
 
 #if _LIBCPP_STD_VER >= 17
diff --git a/libcxx/include/__type_traits/is_floating_point.h b/libcxx/include/__type_traits/is_floating_point.h
index 2789943a71076..add34782dfa09 100644
--- a/libcxx/include/__type_traits/is_floating_point.h
+++ b/libcxx/include/__type_traits/is_floating_point.h
@@ -19,13 +19,15 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
+// clang-format off
 template <class _Tp> struct __libcpp_is_floating_point              : public false_type {};
 template <>          struct __libcpp_is_floating_point<float>       : public true_type {};
 template <>          struct __libcpp_is_floating_point<double>      : public true_type {};
 template <>          struct __libcpp_is_floating_point<long double> : public true_type {};
+// clang-format on
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_floating_point
-    : public __libcpp_is_floating_point<__remove_cv_t<_Tp> > {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_floating_point : public __libcpp_is_floating_point<__remove_cv_t<_Tp> > {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/is_fundamental.h b/libcxx/include/__type_traits/is_fundamental.h
index 8a1c8142b27fe..57206e0d9deb1 100644
--- a/libcxx/include/__type_traits/is_fundamental.h
+++ b/libcxx/include/__type_traits/is_fundamental.h
@@ -22,25 +22,24 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if __has_builtin(__is_fundamental)
 
-template<class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_fundamental : _BoolConstant<__is_fundamental(_Tp)> { };
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_fundamental : _BoolConstant<__is_fundamental(_Tp)> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_fundamental_v = __is_fundamental(_Tp);
-#endif
+#  endif
 
 #else // __has_builtin(__is_fundamental)
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_fundamental
-    : public integral_constant<bool, is_void<_Tp>::value        ||
-                                     __is_nullptr_t<_Tp>::value ||
-                                     is_arithmetic<_Tp>::value> {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_fundamental
+    : public integral_constant<bool, is_void<_Tp>::value || __is_nullptr_t<_Tp>::value || is_arithmetic<_Tp>::value> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_fundamental_v = is_fundamental<_Tp>::value;
-#endif
+#  endif
 
 #endif // __has_builtin(__is_fundamental)
 
diff --git a/libcxx/include/__type_traits/is_implicitly_default_constructible.h b/libcxx/include/__type_traits/is_implicitly_default_constructible.h
index b77f94845fde0..576166e52698a 100644
--- a/libcxx/include/__type_traits/is_implicitly_default_constructible.h
+++ b/libcxx/include/__type_traits/is_implicitly_default_constructible.h
@@ -28,19 +28,17 @@ template <class _Tp>
 void __test_implicit_default_constructible(_Tp);
 
 template <class _Tp, class = void, class = typename is_default_constructible<_Tp>::type>
-struct __is_implicitly_default_constructible
-    : false_type
-{ };
+struct __is_implicitly_default_constructible : false_type {};
 
 template <class _Tp>
-struct __is_implicitly_default_constructible<_Tp, decltype(std::__test_implicit_default_constructible<_Tp const&>({})), true_type>
-    : true_type
-{ };
+struct __is_implicitly_default_constructible<_Tp,
+                                             decltype(std::__test_implicit_default_constructible<_Tp const&>({})),
+                                             true_type> : true_type {};
 
 template <class _Tp>
-struct __is_implicitly_default_constructible<_Tp, decltype(std::__test_implicit_default_constructible<_Tp const&>({})), false_type>
-    : false_type
-{ };
+struct __is_implicitly_default_constructible<_Tp,
+                                             decltype(std::__test_implicit_default_constructible<_Tp const&>({})),
+                                             false_type> : false_type {};
 #endif // !C++03
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/is_integral.h b/libcxx/include/__type_traits/is_integral.h
index c4803cb036048..26969885af8df 100644
--- a/libcxx/include/__type_traits/is_integral.h
+++ b/libcxx/include/__type_traits/is_integral.h
@@ -19,6 +19,7 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
+// clang-format off
 template <class _Tp> struct __libcpp_is_integral                     { enum { value = 0 }; };
 template <>          struct __libcpp_is_integral<bool>               { enum { value = 1 }; };
 template <>          struct __libcpp_is_integral<char>               { enum { value = 1 }; };
@@ -44,26 +45,27 @@ template <>          struct __libcpp_is_integral<unsigned long long> { enum { va
 template <>          struct __libcpp_is_integral<__int128_t>         { enum { value = 1 }; };
 template <>          struct __libcpp_is_integral<__uint128_t>        { enum { value = 1 }; };
 #endif
+// clang-format on
 
 #if __has_builtin(__is_integral)
 
 template <class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_integral : _BoolConstant<__is_integral(_Tp)> { };
+struct _LIBCPP_TEMPLATE_VIS is_integral : _BoolConstant<__is_integral(_Tp)> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_integral_v = __is_integral(_Tp);
-#endif
+#  endif
 
 #else
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_integral
-    : public _BoolConstant<__libcpp_is_integral<__remove_cv_t<_Tp> >::value> {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_integral : public _BoolConstant<__libcpp_is_integral<__remove_cv_t<_Tp> >::value> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_integral_v = is_integral<_Tp>::value;
-#endif
+#  endif
 
 #endif // __has_builtin(__is_integral)
 
diff --git a/libcxx/include/__type_traits/is_literal_type.h b/libcxx/include/__type_traits/is_literal_type.h
index 624fa5dbce5a9..81da34572c750 100644
--- a/libcxx/include/__type_traits/is_literal_type.h
+++ b/libcxx/include/__type_traits/is_literal_type.h
@@ -19,15 +19,15 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_TYPE_TRAITS)
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX17 is_literal_type
-    : public integral_constant<bool, __is_literal_type(_Tp)>
-    {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS _LIBCPP_DEPRECATED_IN_CXX17 is_literal_type
+    : public integral_constant<bool, __is_literal_type(_Tp)> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 _LIBCPP_DEPRECATED_IN_CXX17 inline constexpr bool is_literal_type_v = __is_literal_type(_Tp);
-#endif // _LIBCPP_STD_VER >= 17
-#endif // _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_TYPE_TRAITS)
+#  endif // _LIBCPP_STD_VER >= 17
+#endif   // _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_TYPE_TRAITS)
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__type_traits/is_member_function_pointer.h b/libcxx/include/__type_traits/is_member_function_pointer.h
index 72f84bdaff564..11c8a00c99756 100644
--- a/libcxx/include/__type_traits/is_member_function_pointer.h
+++ b/libcxx/include/__type_traits/is_member_function_pointer.h
@@ -21,41 +21,39 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp> struct __libcpp_is_member_pointer {
-  enum {
-    __is_member = false,
-    __is_func = false,
-    __is_obj = false
-  };
+template <class _Tp>
+struct __libcpp_is_member_pointer {
+  enum { __is_member = false, __is_func = false, __is_obj = false };
 };
-template <class _Tp, class _Up> struct __libcpp_is_member_pointer<_Tp _Up::*> {
+template <class _Tp, class _Up>
+struct __libcpp_is_member_pointer<_Tp _Up::*> {
   enum {
     __is_member = true,
-    __is_func = is_function<_Tp>::value,
-    __is_obj = !__is_func,
+    __is_func   = is_function<_Tp>::value,
+    __is_obj    = !__is_func,
   };
 };
 
 #if __has_builtin(__is_member_function_pointer)
 
-template<class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_member_function_pointer
-    : _BoolConstant<__is_member_function_pointer(_Tp)> { };
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_member_function_pointer : _BoolConstant<__is_member_function_pointer(_Tp)> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_member_function_pointer_v = __is_member_function_pointer(_Tp);
-#endif
+#  endif
 
 #else // __has_builtin(__is_member_function_pointer)
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_member_function_pointer
-    : public _BoolConstant< __libcpp_is_member_pointer<__remove_cv_t<_Tp> >::__is_func > {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_member_function_pointer
+    : public _BoolConstant<__libcpp_is_member_pointer<__remove_cv_t<_Tp> >::__is_func> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_member_function_pointer_v = is_member_function_pointer<_Tp>::value;
-#endif
+#  endif
 
 #endif // __has_builtin(__is_member_function_pointer)
 
diff --git a/libcxx/include/__type_traits/is_member_object_pointer.h b/libcxx/include/__type_traits/is_member_object_pointer.h
index bcdc651683282..9141f9da530f2 100644
--- a/libcxx/include/__type_traits/is_member_object_pointer.h
+++ b/libcxx/include/__type_traits/is_member_object_pointer.h
@@ -20,24 +20,24 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if __has_builtin(__is_member_object_pointer)
 
-template<class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_member_object_pointer
-    : _BoolConstant<__is_member_object_pointer(_Tp)> { };
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_member_object_pointer : _BoolConstant<__is_member_object_pointer(_Tp)> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_member_object_pointer_v = __is_member_object_pointer(_Tp);
-#endif
+#  endif
 
 #else // __has_builtin(__is_member_object_pointer)
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_member_object_pointer
-    : public _BoolConstant< __libcpp_is_member_pointer<__remove_cv_t<_Tp> >::__is_obj >  {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_member_object_pointer
+    : public _BoolConstant<__libcpp_is_member_pointer<__remove_cv_t<_Tp> >::__is_obj> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_member_object_pointer_v = is_member_object_pointer<_Tp>::value;
-#endif
+#  endif
 
 #endif // __has_builtin(__is_member_object_pointer)
 
diff --git a/libcxx/include/__type_traits/is_member_pointer.h b/libcxx/include/__type_traits/is_member_pointer.h
index 5339fb0509b46..4ca01d1c2fc89 100644
--- a/libcxx/include/__type_traits/is_member_pointer.h
+++ b/libcxx/include/__type_traits/is_member_pointer.h
@@ -21,23 +21,24 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if __has_builtin(__is_member_pointer)
 
-template<class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_member_pointer : _BoolConstant<__is_member_pointer(_Tp)> { };
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_member_pointer : _BoolConstant<__is_member_pointer(_Tp)> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_member_pointer_v = __is_member_pointer(_Tp);
-#endif
+#  endif
 
 #else // __has_builtin(__is_member_pointer)
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_member_pointer
- : public _BoolConstant< __libcpp_is_member_pointer<__remove_cv_t<_Tp> >::__is_member > {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_member_pointer
+    : public _BoolConstant<__libcpp_is_member_pointer<__remove_cv_t<_Tp> >::__is_member> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_member_pointer_v = is_member_pointer<_Tp>::value;
-#endif
+#  endif
 
 #endif // __has_builtin(__is_member_pointer)
 
diff --git a/libcxx/include/__type_traits/is_move_assignable.h b/libcxx/include/__type_traits/is_move_assignable.h
index 8c4c82084bfa1..867bc00d824ae 100644
--- a/libcxx/include/__type_traits/is_move_assignable.h
+++ b/libcxx/include/__type_traits/is_move_assignable.h
@@ -22,9 +22,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp>
 struct _LIBCPP_TEMPLATE_VIS is_move_assignable
-    : public integral_constant<
-          bool,
-          __is_assignable(__add_lvalue_reference_t<_Tp>, __add_rvalue_reference_t<_Tp>)> {};
+    : public integral_constant<bool, __is_assignable(__add_lvalue_reference_t<_Tp>, __add_rvalue_reference_t<_Tp>)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/is_move_constructible.h b/libcxx/include/__type_traits/is_move_constructible.h
index d908e49863822..40ec4a070816b 100644
--- a/libcxx/include/__type_traits/is_move_constructible.h
+++ b/libcxx/include/__type_traits/is_move_constructible.h
@@ -21,8 +21,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp>
 struct _LIBCPP_TEMPLATE_VIS is_move_constructible
-    : public integral_constant<bool, __is_constructible(_Tp, __add_rvalue_reference_t<_Tp>)>
-    {};
+    : public integral_constant<bool, __is_constructible(_Tp, __add_rvalue_reference_t<_Tp>)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/is_nothrow_assignable.h b/libcxx/include/__type_traits/is_nothrow_assignable.h
index 4d17b9fc9b7c7..cc57493313859 100644
--- a/libcxx/include/__type_traits/is_nothrow_assignable.h
+++ b/libcxx/include/__type_traits/is_nothrow_assignable.h
@@ -19,8 +19,8 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp, class _Arg>
-struct _LIBCPP_TEMPLATE_VIS is_nothrow_assignable
-    : public integral_constant<bool, __is_nothrow_assignable(_Tp, _Arg)> {};
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_assignable : public integral_constant<bool, __is_nothrow_assignable(_Tp, _Arg)> {
+};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp, class _Arg>
diff --git a/libcxx/include/__type_traits/is_nothrow_constructible.h b/libcxx/include/__type_traits/is_nothrow_constructible.h
index a967bc8791223..d4686d89fd96e 100644
--- a/libcxx/include/__type_traits/is_nothrow_constructible.h
+++ b/libcxx/include/__type_traits/is_nothrow_constructible.h
@@ -24,52 +24,44 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if __has_builtin(__is_nothrow_constructible)
 
-template <
-    class _Tp, class... _Args>
+template < class _Tp, class... _Args>
 struct _LIBCPP_TEMPLATE_VIS is_nothrow_constructible
     : public integral_constant<bool, __is_nothrow_constructible(_Tp, _Args...)> {};
 #else
 
-template <bool, bool, class _Tp, class... _Args> struct __libcpp_is_nothrow_constructible;
+template <bool, bool, class _Tp, class... _Args>
+struct __libcpp_is_nothrow_constructible;
 
 template <class _Tp, class... _Args>
-struct __libcpp_is_nothrow_constructible</*is constructible*/true, /*is reference*/false, _Tp, _Args...>
-    : public integral_constant<bool, noexcept(_Tp(std::declval<_Args>()...))>
-{
-};
+struct __libcpp_is_nothrow_constructible</*is constructible*/ true, /*is reference*/ false, _Tp, _Args...>
+    : public integral_constant<bool, noexcept(_Tp(std::declval<_Args>()...))> {};
 
 template <class _Tp>
-void __implicit_conversion_to(_Tp) noexcept { }
+void __implicit_conversion_to(_Tp) noexcept {}
 
 template <class _Tp, class _Arg>
-struct __libcpp_is_nothrow_constructible</*is constructible*/true, /*is reference*/true, _Tp, _Arg>
-    : public integral_constant<bool, noexcept(_VSTD::__implicit_conversion_to<_Tp>(std::declval<_Arg>()))>
-{
-};
+struct __libcpp_is_nothrow_constructible</*is constructible*/ true, /*is reference*/ true, _Tp, _Arg>
+    : public integral_constant<bool, noexcept(_VSTD::__implicit_conversion_to<_Tp>(std::declval<_Arg>()))> {};
 
 template <class _Tp, bool _IsReference, class... _Args>
-struct __libcpp_is_nothrow_constructible</*is constructible*/false, _IsReference, _Tp, _Args...>
-    : public false_type
-{
+struct __libcpp_is_nothrow_constructible</*is constructible*/ false, _IsReference, _Tp, _Args...> : public false_type {
 };
 
 template <class _Tp, class... _Args>
 struct _LIBCPP_TEMPLATE_VIS is_nothrow_constructible
-    : __libcpp_is_nothrow_constructible<is_constructible<_Tp, _Args...>::value, is_reference<_Tp>::value, _Tp, _Args...>
-{
-};
+    : __libcpp_is_nothrow_constructible<is_constructible<_Tp, _Args...>::value,
+                                        is_reference<_Tp>::value,
+                                        _Tp,
+                                        _Args...> {};
 
 template <class _Tp, size_t _Ns>
 struct _LIBCPP_TEMPLATE_VIS is_nothrow_constructible<_Tp[_Ns]>
-    : __libcpp_is_nothrow_constructible<is_constructible<_Tp>::value, is_reference<_Tp>::value, _Tp>
-{
-};
+    : __libcpp_is_nothrow_constructible<is_constructible<_Tp>::value, is_reference<_Tp>::value, _Tp> {};
 
 #endif // __has_builtin(__is_nothrow_constructible)
 
-
 #if _LIBCPP_STD_VER >= 17
-template <class _Tp, class ..._Args>
+template <class _Tp, class... _Args>
 inline constexpr bool is_nothrow_constructible_v = is_nothrow_constructible<_Tp, _Args...>::value;
 #endif
 
diff --git a/libcxx/include/__type_traits/is_nothrow_convertible.h b/libcxx/include/__type_traits/is_nothrow_convertible.h
index 64f75e1fd68db..a8f8b4d063a10 100644
--- a/libcxx/include/__type_traits/is_nothrow_convertible.h
+++ b/libcxx/include/__type_traits/is_nothrow_convertible.h
@@ -29,19 +29,16 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 template <typename _Tp>
 void __test_noexcept(_Tp) noexcept;
 
-template<typename _Fm, typename _To>
-bool_constant<noexcept(_VSTD::__test_noexcept<_To>(std::declval<_Fm>()))>
-__is_nothrow_convertible_test();
+template <typename _Fm, typename _To>
+bool_constant<noexcept(_VSTD::__test_noexcept<_To>(std::declval<_Fm>()))> __is_nothrow_convertible_test();
 
 template <typename _Fm, typename _To>
-struct __is_nothrow_convertible_helper: decltype(__is_nothrow_convertible_test<_Fm, _To>())
-{ };
+struct __is_nothrow_convertible_helper : decltype(__is_nothrow_convertible_test<_Fm, _To>()) {};
 
 template <typename _Fm, typename _To>
-struct is_nothrow_convertible : _Or<
-    _And<is_void<_To>, is_void<_Fm>>,
-    _Lazy<_And, is_convertible<_Fm, _To>, __is_nothrow_convertible_helper<_Fm, _To>>
->::type { };
+struct is_nothrow_convertible
+    : _Or<_And<is_void<_To>, is_void<_Fm>>,
+          _Lazy<_And, is_convertible<_Fm, _To>, __is_nothrow_convertible_helper<_Fm, _To> > >::type {};
 
 template <typename _Fm, typename _To>
 inline constexpr bool is_nothrow_convertible_v = is_nothrow_convertible<_Fm, _To>::value;
diff --git a/libcxx/include/__type_traits/is_nothrow_copy_assignable.h b/libcxx/include/__type_traits/is_nothrow_copy_assignable.h
index d11ec1c01fba6..a97e962b308c8 100644
--- a/libcxx/include/__type_traits/is_nothrow_copy_assignable.h
+++ b/libcxx/include/__type_traits/is_nothrow_copy_assignable.h
@@ -22,11 +22,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp>
 struct _LIBCPP_TEMPLATE_VIS is_nothrow_copy_assignable
-    : public integral_constant<
-          bool,
-          __is_nothrow_assignable(
-              __add_lvalue_reference_t<_Tp>,
-              __add_lvalue_reference_t<typename add_const<_Tp>::type>)> {};
+    : public integral_constant<bool,
+                               __is_nothrow_assignable(__add_lvalue_reference_t<_Tp>,
+                                                       __add_lvalue_reference_t<typename add_const<_Tp>::type>)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/is_nothrow_copy_constructible.h b/libcxx/include/__type_traits/is_nothrow_copy_constructible.h
index de4e272277765..dd7f1d5ef6276 100644
--- a/libcxx/include/__type_traits/is_nothrow_copy_constructible.h
+++ b/libcxx/include/__type_traits/is_nothrow_copy_constructible.h
@@ -24,9 +24,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 // TODO: remove this implementation once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=106611 is fixed
 #ifdef _LIBCPP_COMPILER_GCC
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_nothrow_copy_constructible
-    : public is_nothrow_constructible<_Tp,
-                  __add_lvalue_reference_t<typename add_const<_Tp>::type> > {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_copy_constructible
+    : public is_nothrow_constructible<_Tp, __add_lvalue_reference_t<typename add_const<_Tp>::type> > {};
 
 #else // _LIBCPP_COMPILER_GCC
 
diff --git a/libcxx/include/__type_traits/is_nothrow_default_constructible.h b/libcxx/include/__type_traits/is_nothrow_default_constructible.h
index 446808f44c756..58f31f21b0267 100644
--- a/libcxx/include/__type_traits/is_nothrow_default_constructible.h
+++ b/libcxx/include/__type_traits/is_nothrow_default_constructible.h
@@ -18,9 +18,9 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_nothrow_default_constructible
-    : public integral_constant<bool, __is_nothrow_constructible(_Tp)>
-    {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_default_constructible
+    : public integral_constant<bool, __is_nothrow_constructible(_Tp)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/is_nothrow_destructible.h b/libcxx/include/__type_traits/is_nothrow_destructible.h
index e842465a3ca4b..370ba27ecd1f1 100644
--- a/libcxx/include/__type_traits/is_nothrow_destructible.h
+++ b/libcxx/include/__type_traits/is_nothrow_destructible.h
@@ -26,56 +26,41 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if !defined(_LIBCPP_CXX03_LANG)
 
-template <bool, class _Tp> struct __libcpp_is_nothrow_destructible;
+template <bool, class _Tp>
+struct __libcpp_is_nothrow_destructible;
 
 template <class _Tp>
-struct __libcpp_is_nothrow_destructible<false, _Tp>
-    : public false_type
-{
-};
+struct __libcpp_is_nothrow_destructible<false, _Tp> : public false_type {};
 
 template <class _Tp>
 struct __libcpp_is_nothrow_destructible<true, _Tp>
-    : public integral_constant<bool, noexcept(std::declval<_Tp>().~_Tp()) >
-{
-};
+    : public integral_constant<bool, noexcept(std::declval<_Tp>().~_Tp()) > {};
 
 template <class _Tp>
 struct _LIBCPP_TEMPLATE_VIS is_nothrow_destructible
-    : public __libcpp_is_nothrow_destructible<is_destructible<_Tp>::value, _Tp>
-{
-};
+    : public __libcpp_is_nothrow_destructible<is_destructible<_Tp>::value, _Tp> {};
 
 template <class _Tp, size_t _Ns>
-struct _LIBCPP_TEMPLATE_VIS is_nothrow_destructible<_Tp[_Ns]>
-    : public is_nothrow_destructible<_Tp>
-{
-};
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_destructible<_Tp[_Ns]> : public is_nothrow_destructible<_Tp> {};
 
 template <class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_nothrow_destructible<_Tp&>
-    : public true_type
-{
-};
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_destructible<_Tp&> : public true_type {};
 
 template <class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_nothrow_destructible<_Tp&&>
-    : public true_type
-{
-};
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_destructible<_Tp&&> : public true_type {};
 
 #else
 
-template <class _Tp> struct __libcpp_nothrow_destructor
-    : public integral_constant<bool, is_scalar<_Tp>::value ||
-                                     is_reference<_Tp>::value> {};
+template <class _Tp>
+struct __libcpp_nothrow_destructor : public integral_constant<bool, is_scalar<_Tp>::value || is_reference<_Tp>::value> {
+};
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_nothrow_destructible
-    : public __libcpp_nothrow_destructor<__remove_all_extents_t<_Tp> > {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_destructible : public __libcpp_nothrow_destructor<__remove_all_extents_t<_Tp> > {
+};
 
 template <class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_nothrow_destructible<_Tp[]>
-    : public false_type {};
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_destructible<_Tp[]> : public false_type {};
 
 #endif
 
diff --git a/libcxx/include/__type_traits/is_nothrow_move_assignable.h b/libcxx/include/__type_traits/is_nothrow_move_assignable.h
index 7649a3fcd4a51..aa87e369cd5d0 100644
--- a/libcxx/include/__type_traits/is_nothrow_move_assignable.h
+++ b/libcxx/include/__type_traits/is_nothrow_move_assignable.h
@@ -22,9 +22,8 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp>
 struct _LIBCPP_TEMPLATE_VIS is_nothrow_move_assignable
-    : public integral_constant<
-          bool,
-          __is_nothrow_assignable(__add_lvalue_reference_t<_Tp>, __add_rvalue_reference_t<_Tp>)> {
+    : public integral_constant<bool,
+                               __is_nothrow_assignable(__add_lvalue_reference_t<_Tp>, __add_rvalue_reference_t<_Tp>)> {
 };
 
 #if _LIBCPP_STD_VER >= 17
diff --git a/libcxx/include/__type_traits/is_nothrow_move_constructible.h b/libcxx/include/__type_traits/is_nothrow_move_constructible.h
index 21a22bbb8ce5c..dab5a019b33bf 100644
--- a/libcxx/include/__type_traits/is_nothrow_move_constructible.h
+++ b/libcxx/include/__type_traits/is_nothrow_move_constructible.h
@@ -23,15 +23,15 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 // TODO: remove this implementation once https://gcc.gnu.org/bugzilla/show_bug.cgi?id=106611 is fixed
 #ifndef _LIBCPP_COMPILER_GCC
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_nothrow_move_constructible
-    : public integral_constant<bool, __is_nothrow_constructible(_Tp, __add_rvalue_reference_t<_Tp>)>
-    {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_move_constructible
+    : public integral_constant<bool, __is_nothrow_constructible(_Tp, __add_rvalue_reference_t<_Tp>)> {};
 
 #else // _LIBCPP_COMPILER_GCC
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_nothrow_move_constructible
-    : public is_nothrow_constructible<_Tp, __add_rvalue_reference_t<_Tp> >
-    {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_nothrow_move_constructible
+    : public is_nothrow_constructible<_Tp, __add_rvalue_reference_t<_Tp> > {};
 
 #endif // _LIBCPP_COMPILER_GCC
 
diff --git a/libcxx/include/__type_traits/is_null_pointer.h b/libcxx/include/__type_traits/is_null_pointer.h
index 3a94fce1bc8aa..c666f5f24759c 100644
--- a/libcxx/include/__type_traits/is_null_pointer.h
+++ b/libcxx/include/__type_traits/is_null_pointer.h
@@ -20,20 +20,22 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp> struct __is_nullptr_t_impl       : public false_type {};
-template <>          struct __is_nullptr_t_impl<nullptr_t> : public true_type {};
+template <class _Tp>
+struct __is_nullptr_t_impl : public false_type {};
+template <>
+struct __is_nullptr_t_impl<nullptr_t> : public true_type {};
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS __is_nullptr_t
-    : public __is_nullptr_t_impl<__remove_cv_t<_Tp> > {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS __is_nullptr_t : public __is_nullptr_t_impl<__remove_cv_t<_Tp> > {};
 
 #if _LIBCPP_STD_VER >= 14
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_null_pointer
-    : public __is_nullptr_t_impl<__remove_cv_t<_Tp> > {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_null_pointer : public __is_nullptr_t_impl<__remove_cv_t<_Tp> > {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_null_pointer_v = is_null_pointer<_Tp>::value;
-#endif
+#  endif
 #endif // _LIBCPP_STD_VER >= 14
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/is_object.h b/libcxx/include/__type_traits/is_object.h
index 4d038c3e67a09..5ac1d854a6958 100644
--- a/libcxx/include/__type_traits/is_object.h
+++ b/libcxx/include/__type_traits/is_object.h
@@ -24,26 +24,26 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if __has_builtin(__is_object)
 
-template<class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_object : _BoolConstant<__is_object(_Tp)> { };
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_object : _BoolConstant<__is_object(_Tp)> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_object_v = __is_object(_Tp);
-#endif
+#  endif
 
 #else // __has_builtin(__is_object)
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_object
-    : public integral_constant<bool, is_scalar<_Tp>::value ||
-                                     is_array<_Tp>::value  ||
-                                     is_union<_Tp>::value  ||
-                                     is_class<_Tp>::value  > {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_object
+    : public integral_constant<bool,
+                               is_scalar<_Tp>::value || is_array<_Tp>::value || is_union<_Tp>::value ||
+                                   is_class<_Tp>::value > {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_object_v = is_object<_Tp>::value;
-#endif
+#  endif
 
 #endif // __has_builtin(__is_object)
 
diff --git a/libcxx/include/__type_traits/is_pod.h b/libcxx/include/__type_traits/is_pod.h
index 851cbd165a636..5888fbf457d8b 100644
--- a/libcxx/include/__type_traits/is_pod.h
+++ b/libcxx/include/__type_traits/is_pod.h
@@ -18,8 +18,8 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_pod
-    : public integral_constant<bool, __is_pod(_Tp)> {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_pod : public integral_constant<bool, __is_pod(_Tp)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/is_pointer.h b/libcxx/include/__type_traits/is_pointer.h
index 51c628f452e00..38eb7996c6864 100644
--- a/libcxx/include/__type_traits/is_pointer.h
+++ b/libcxx/include/__type_traits/is_pointer.h
@@ -21,34 +21,42 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if __has_builtin(__is_pointer)
 
-template<class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_pointer : _BoolConstant<__is_pointer(_Tp)> { };
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_pointer : _BoolConstant<__is_pointer(_Tp)> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_pointer_v = __is_pointer(_Tp);
-#endif
+#  endif
 
 #else // __has_builtin(__is_pointer)
 
-template <class _Tp> struct __libcpp_is_pointer       : public false_type {};
-template <class _Tp> struct __libcpp_is_pointer<_Tp*> : public true_type {};
+template <class _Tp>
+struct __libcpp_is_pointer : public false_type {};
+template <class _Tp>
+struct __libcpp_is_pointer<_Tp*> : public true_type {};
 
-template <class _Tp> struct __libcpp_remove_objc_qualifiers { typedef _Tp type; };
-#if defined(_LIBCPP_HAS_OBJC_ARC)
+template <class _Tp>
+struct __libcpp_remove_objc_qualifiers {
+  typedef _Tp type;
+};
+#  if defined(_LIBCPP_HAS_OBJC_ARC)
+// clang-format off
 template <class _Tp> struct __libcpp_remove_objc_qualifiers<_Tp __strong> { typedef _Tp type; };
 template <class _Tp> struct __libcpp_remove_objc_qualifiers<_Tp __weak> { typedef _Tp type; };
 template <class _Tp> struct __libcpp_remove_objc_qualifiers<_Tp __autoreleasing> { typedef _Tp type; };
 template <class _Tp> struct __libcpp_remove_objc_qualifiers<_Tp __unsafe_unretained> { typedef _Tp type; };
-#endif
+// clang-format on
+#  endif
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_pointer
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_pointer
     : public __libcpp_is_pointer<typename __libcpp_remove_objc_qualifiers<__remove_cv_t<_Tp> >::type> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_pointer_v = is_pointer<_Tp>::value;
-#endif
+#  endif
 
 #endif // __has_builtin(__is_pointer)
 
diff --git a/libcxx/include/__type_traits/is_polymorphic.h b/libcxx/include/__type_traits/is_polymorphic.h
index 21b3cdcdc75d7..d122e1c87775b 100644
--- a/libcxx/include/__type_traits/is_polymorphic.h
+++ b/libcxx/include/__type_traits/is_polymorphic.h
@@ -19,8 +19,7 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_polymorphic
-    : public integral_constant<bool, __is_polymorphic(_Tp)> {};
+struct _LIBCPP_TEMPLATE_VIS is_polymorphic : public integral_constant<bool, __is_polymorphic(_Tp)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/is_primary_template.h b/libcxx/include/__type_traits/is_primary_template.h
index a9bebcf7e64c0..f308dfadc8ec8 100644
--- a/libcxx/include/__type_traits/is_primary_template.h
+++ b/libcxx/include/__type_traits/is_primary_template.h
@@ -21,13 +21,10 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp>
-using __test_for_primary_template = __enable_if_t<
-    _IsSame<_Tp, typename _Tp::__primary_template>::value
-  >;
+using __test_for_primary_template = __enable_if_t<_IsSame<_Tp, typename _Tp::__primary_template>::value>;
+
 template <class _Tp>
-using __is_primary_template = _IsValidExpansion<
-    __test_for_primary_template, _Tp
-  >;
+using __is_primary_template = _IsValidExpansion<__test_for_primary_template, _Tp>;
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__type_traits/is_reference.h b/libcxx/include/__type_traits/is_reference.h
index 3334c60cc58c4..42853d4790395 100644
--- a/libcxx/include/__type_traits/is_reference.h
+++ b/libcxx/include/__type_traits/is_reference.h
@@ -18,41 +18,46 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-#if __has_builtin(__is_lvalue_reference) && \
-    __has_builtin(__is_rvalue_reference) && \
-    __has_builtin(__is_reference)
+#if __has_builtin(__is_lvalue_reference) && __has_builtin(__is_rvalue_reference) && __has_builtin(__is_reference)
 
-template<class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_lvalue_reference : _BoolConstant<__is_lvalue_reference(_Tp)> { };
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_lvalue_reference : _BoolConstant<__is_lvalue_reference(_Tp)> {};
 
-template<class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_rvalue_reference : _BoolConstant<__is_rvalue_reference(_Tp)> { };
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_rvalue_reference : _BoolConstant<__is_rvalue_reference(_Tp)> {};
 
-template<class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_reference : _BoolConstant<__is_reference(_Tp)> { };
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_reference : _BoolConstant<__is_reference(_Tp)> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_reference_v = __is_reference(_Tp);
 template <class _Tp>
 inline constexpr bool is_lvalue_reference_v = __is_lvalue_reference(_Tp);
 template <class _Tp>
 inline constexpr bool is_rvalue_reference_v = __is_rvalue_reference(_Tp);
-#endif
+#  endif
 
 #else // __has_builtin(__is_lvalue_reference) && etc...
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_lvalue_reference       : public false_type {};
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_lvalue_reference<_Tp&> : public true_type {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_lvalue_reference : public false_type {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_lvalue_reference<_Tp&> : public true_type {};
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_rvalue_reference        : public false_type {};
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_rvalue_reference<_Tp&&> : public true_type {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_rvalue_reference : public false_type {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_rvalue_reference<_Tp&&> : public true_type {};
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_reference        : public false_type {};
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_reference<_Tp&>  : public true_type {};
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_reference<_Tp&&> : public true_type {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_reference : public false_type {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_reference<_Tp&> : public true_type {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_reference<_Tp&&> : public true_type {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_reference_v = is_reference<_Tp>::value;
 
@@ -61,7 +66,7 @@ inline constexpr bool is_lvalue_reference_v = is_lvalue_reference<_Tp>::value;
 
 template <class _Tp>
 inline constexpr bool is_rvalue_reference_v = is_rvalue_reference<_Tp>::value;
-#endif
+#  endif
 
 #endif // __has_builtin(__is_lvalue_reference) && etc...
 
diff --git a/libcxx/include/__type_traits/is_reference_wrapper.h b/libcxx/include/__type_traits/is_reference_wrapper.h
index d187eec1e1320..b638e7046b71c 100644
--- a/libcxx/include/__type_traits/is_reference_wrapper.h
+++ b/libcxx/include/__type_traits/is_reference_wrapper.h
@@ -19,12 +19,15 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp> class _LIBCPP_TEMPLATE_VIS reference_wrapper;
-
-template <class _Tp> struct __is_reference_wrapper_impl : public false_type {};
-template <class _Tp> struct __is_reference_wrapper_impl<reference_wrapper<_Tp> > : public true_type {};
-template <class _Tp> struct __is_reference_wrapper
-    : public __is_reference_wrapper_impl<__remove_cv_t<_Tp> > {};
+template <class _Tp>
+class _LIBCPP_TEMPLATE_VIS reference_wrapper;
+
+template <class _Tp>
+struct __is_reference_wrapper_impl : public false_type {};
+template <class _Tp>
+struct __is_reference_wrapper_impl<reference_wrapper<_Tp> > : public true_type {};
+template <class _Tp>
+struct __is_reference_wrapper : public __is_reference_wrapper_impl<__remove_cv_t<_Tp> > {};
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__type_traits/is_same.h b/libcxx/include/__type_traits/is_same.h
index c598f0d407842..9561b7b5d6da3 100644
--- a/libcxx/include/__type_traits/is_same.h
+++ b/libcxx/include/__type_traits/is_same.h
@@ -19,7 +19,7 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp, class _Up>
-struct _LIBCPP_TEMPLATE_VIS is_same : _BoolConstant<__is_same(_Tp, _Up)> { };
+struct _LIBCPP_TEMPLATE_VIS is_same : _BoolConstant<__is_same(_Tp, _Up)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp, class _Up>
diff --git a/libcxx/include/__type_traits/is_scalar.h b/libcxx/include/__type_traits/is_scalar.h
index 9892ae089d7b6..2bfedc3693659 100644
--- a/libcxx/include/__type_traits/is_scalar.h
+++ b/libcxx/include/__type_traits/is_scalar.h
@@ -18,41 +18,45 @@
 #include <__type_traits/is_pointer.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
-#  pragma GCC system_header
+#pragma GCC system_header
 #endif
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if __has_builtin(__is_scalar)
 
-template<class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_scalar : _BoolConstant<__is_scalar(_Tp)> { };
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_scalar : _BoolConstant<__is_scalar(_Tp)> {};
 
 #if _LIBCPP_STD_VER >= 17
-template <class _Tp>
-inline constexpr bool is_scalar_v = __is_scalar(_Tp);
+template <class _Tp> inline constexpr bool is_scalar_v = __is_scalar(_Tp);
 #endif
 
 #else // __has_builtin(__is_scalar)
 
 template <class _Tp> struct __is_block : false_type {};
 #if defined(_LIBCPP_HAS_EXTENSION_BLOCKS)
-template <class _Rp, class ..._Args> struct __is_block<_Rp (^)(_Args...)> : true_type {};
+template <class _Rp, class... _Args>
+struct __is_block<_Rp (^)(_Args...)> : true_type {};
 #endif
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_scalar
-    : public integral_constant<bool, is_arithmetic<_Tp>::value     ||
-                                     is_member_pointer<_Tp>::value ||
-                                     is_pointer<_Tp>::value        ||
-                                     __is_nullptr_t<_Tp>::value    ||
-                                     __is_block<_Tp>::value        ||
-                                     is_enum<_Tp>::value           > {};
+// clang-format off
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_scalar
+    : public integral_constant<
+          bool, is_arithmetic<_Tp>::value ||
+                is_member_pointer<_Tp>::value ||
+                is_pointer<_Tp>::value ||
+                __is_nullptr_t<_Tp>::value ||
+                __is_block<_Tp>::value ||
+                is_enum<_Tp>::value> {};
+// clang-format on
 
-template <> struct _LIBCPP_TEMPLATE_VIS is_scalar<nullptr_t> : public true_type {};
+template <>
+struct _LIBCPP_TEMPLATE_VIS is_scalar<nullptr_t> : public true_type {};
 
 #if _LIBCPP_STD_VER >= 17
-template <class _Tp>
-inline constexpr bool is_scalar_v = is_scalar<_Tp>::value;
+template <class _Tp> inline constexpr bool is_scalar_v = is_scalar<_Tp>::value;
 #endif
 
 #endif // __has_builtin(__is_scalar)
diff --git a/libcxx/include/__type_traits/is_scoped_enum.h b/libcxx/include/__type_traits/is_scoped_enum.h
index 3f0fd589a8aa7..43b3a6b66b1f5 100644
--- a/libcxx/include/__type_traits/is_scoped_enum.h
+++ b/libcxx/include/__type_traits/is_scoped_enum.h
@@ -26,12 +26,10 @@ template <class _Tp, bool = is_enum_v<_Tp> >
 struct __is_scoped_enum_helper : false_type {};
 
 template <class _Tp>
-struct __is_scoped_enum_helper<_Tp, true>
-    : public bool_constant<!is_convertible_v<_Tp, underlying_type_t<_Tp> > > {};
+struct __is_scoped_enum_helper<_Tp, true> : public bool_constant<!is_convertible_v<_Tp, underlying_type_t<_Tp> > > {};
 
 template <class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_scoped_enum
-    : public __is_scoped_enum_helper<_Tp> {};
+struct _LIBCPP_TEMPLATE_VIS is_scoped_enum : public __is_scoped_enum_helper<_Tp> {};
 
 template <class _Tp>
 inline constexpr bool is_scoped_enum_v = is_scoped_enum<_Tp>::value;
diff --git a/libcxx/include/__type_traits/is_signed.h b/libcxx/include/__type_traits/is_signed.h
index ebd3873d5db26..fd6f93e182362 100644
--- a/libcxx/include/__type_traits/is_signed.h
+++ b/libcxx/include/__type_traits/is_signed.h
@@ -22,13 +22,13 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if __has_builtin(__is_signed)
 
-template<class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_signed : _BoolConstant<__is_signed(_Tp)> { };
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_signed : _BoolConstant<__is_signed(_Tp)> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_signed_v = __is_signed(_Tp);
-#endif
+#  endif
 
 #else // __has_builtin(__is_signed)
 
@@ -36,19 +36,21 @@ template <class _Tp, bool = is_integral<_Tp>::value>
 struct __libcpp_is_signed_impl : public _BoolConstant<(_Tp(-1) < _Tp(0))> {};
 
 template <class _Tp>
-struct __libcpp_is_signed_impl<_Tp, false> : public true_type {};  // floating point
+struct __libcpp_is_signed_impl<_Tp, false> : public true_type {}; // floating point
 
 template <class _Tp, bool = is_arithmetic<_Tp>::value>
 struct __libcpp_is_signed : public __libcpp_is_signed_impl<_Tp> {};
 
-template <class _Tp> struct __libcpp_is_signed<_Tp, false> : public false_type {};
+template <class _Tp>
+struct __libcpp_is_signed<_Tp, false> : public false_type {};
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_signed : public __libcpp_is_signed<_Tp> {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_signed : public __libcpp_is_signed<_Tp> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_signed_v = is_signed<_Tp>::value;
-#endif
+#  endif
 
 #endif // __has_builtin(__is_signed)
 
diff --git a/libcxx/include/__type_traits/is_signed_integer.h b/libcxx/include/__type_traits/is_signed_integer.h
index 95aa11bfa1b58..467548d0aaafb 100644
--- a/libcxx/include/__type_traits/is_signed_integer.h
+++ b/libcxx/include/__type_traits/is_signed_integer.h
@@ -18,15 +18,17 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp> struct __libcpp_is_signed_integer : public false_type {};
-template <> struct __libcpp_is_signed_integer<signed char>      : public true_type {};
-template <> struct __libcpp_is_signed_integer<signed short>     : public true_type {};
-template <> struct __libcpp_is_signed_integer<signed int>       : public true_type {};
-template <> struct __libcpp_is_signed_integer<signed long>      : public true_type {};
-template <> struct __libcpp_is_signed_integer<signed long long> : public true_type {};
+// clang-format off
+template <class _Tp> struct __libcpp_is_signed_integer                   : public false_type {};
+template <>          struct __libcpp_is_signed_integer<signed char>      : public true_type {};
+template <>          struct __libcpp_is_signed_integer<signed short>     : public true_type {};
+template <>          struct __libcpp_is_signed_integer<signed int>       : public true_type {};
+template <>          struct __libcpp_is_signed_integer<signed long>      : public true_type {};
+template <>          struct __libcpp_is_signed_integer<signed long long> : public true_type {};
 #ifndef _LIBCPP_HAS_NO_INT128
-template <> struct __libcpp_is_signed_integer<__int128_t>       : public true_type {};
+template <>          struct __libcpp_is_signed_integer<__int128_t>       : public true_type {};
 #endif
+// clang-format on
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__type_traits/is_standard_layout.h b/libcxx/include/__type_traits/is_standard_layout.h
index aae533b453ef5..76484f3e2a301 100644
--- a/libcxx/include/__type_traits/is_standard_layout.h
+++ b/libcxx/include/__type_traits/is_standard_layout.h
@@ -18,9 +18,8 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_standard_layout
-    : public integral_constant<bool, __is_standard_layout(_Tp)>
-    {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_standard_layout : public integral_constant<bool, __is_standard_layout(_Tp)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/is_swappable.h b/libcxx/include/__type_traits/is_swappable.h
index 5a78f470874db..07c56619856d3 100644
--- a/libcxx/include/__type_traits/is_swappable.h
+++ b/libcxx/include/__type_traits/is_swappable.h
@@ -30,9 +30,10 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp> struct __is_swappable;
-template <class _Tp> struct __is_nothrow_swappable;
-
+template <class _Tp>
+struct __is_swappable;
+template <class _Tp>
+struct __is_nothrow_swappable;
 
 #ifndef _LIBCPP_CXX03_LANG
 template <class _Tp>
@@ -43,49 +44,40 @@ using __swap_result_t = void;
 #endif
 
 template <class _Tp>
-inline _LIBCPP_INLINE_VISIBILITY
-_LIBCPP_CONSTEXPR_SINCE_CXX20 __swap_result_t<_Tp>
-swap(_Tp& __x, _Tp& __y) _NOEXCEPT_(is_nothrow_move_constructible<_Tp>::value &&
-                                    is_nothrow_move_assignable<_Tp>::value);
+inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20 __swap_result_t<_Tp> swap(_Tp& __x, _Tp& __y)
+    _NOEXCEPT_(is_nothrow_move_constructible<_Tp>::value&& is_nothrow_move_assignable<_Tp>::value);
 
-template<class _Tp, size_t _Np>
+template <class _Tp, size_t _Np>
 inline _LIBCPP_INLINE_VISIBILITY _LIBCPP_CONSTEXPR_SINCE_CXX20
-typename enable_if<
-    __is_swappable<_Tp>::value
->::type
-swap(_Tp (&__a)[_Np], _Tp (&__b)[_Np]) _NOEXCEPT_(__is_nothrow_swappable<_Tp>::value);
+    typename enable_if<__is_swappable<_Tp>::value>::type swap(_Tp (&__a)[_Np], _Tp (&__b)[_Np])
+        _NOEXCEPT_(__is_nothrow_swappable<_Tp>::value);
 
-namespace __detail
-{
+namespace __detail {
 // ALL generic swap overloads MUST already have a declaration available at this point.
 
-template <class _Tp, class _Up = _Tp,
-          bool _NotVoid = !is_void<_Tp>::value && !is_void<_Up>::value>
-struct __swappable_with
-{
-    template <class _LHS, class _RHS>
-    static decltype(swap(std::declval<_LHS>(), std::declval<_RHS>()))
-    __test_swap(int);
-    template <class, class>
-    static __nat __test_swap(long);
-
-    // Extra parens are needed for the C++03 definition of decltype.
-    typedef decltype((__test_swap<_Tp, _Up>(0))) __swap1;
-    typedef decltype((__test_swap<_Up, _Tp>(0))) __swap2;
-
-    static const bool value = _IsNotSame<__swap1, __nat>::value
-                           && _IsNotSame<__swap2, __nat>::value;
+template <class _Tp, class _Up = _Tp, bool _NotVoid = !is_void<_Tp>::value && !is_void<_Up>::value>
+struct __swappable_with {
+  template <class _LHS, class _RHS>
+  static decltype(swap(std::declval<_LHS>(), std::declval<_RHS>())) __test_swap(int);
+  template <class, class>
+  static __nat __test_swap(long);
+
+  // Extra parens are needed for the C++03 definition of decltype.
+  typedef decltype((__test_swap<_Tp, _Up>(0))) __swap1;
+  typedef decltype((__test_swap<_Up, _Tp>(0))) __swap2;
+
+  static const bool value = _IsNotSame<__swap1, __nat>::value && _IsNotSame<__swap2, __nat>::value;
 };
 
 template <class _Tp, class _Up>
-struct __swappable_with<_Tp, _Up,  false> : false_type {};
+struct __swappable_with<_Tp, _Up, false> : false_type {};
 
 template <class _Tp, class _Up = _Tp, bool _Swappable = __swappable_with<_Tp, _Up>::value>
 struct __nothrow_swappable_with {
   static const bool value =
 #ifndef _LIBCPP_HAS_NO_NOEXCEPT
-      noexcept(swap(std::declval<_Tp>(), std::declval<_Up>()))
-  &&  noexcept(swap(std::declval<_Up>(), std::declval<_Tp>()));
+      noexcept(swap(std::declval<_Tp>(), std::declval<_Up>()))&& noexcept(
+          swap(std::declval<_Up>(), std::declval<_Tp>()));
 #else
       false;
 #endif
@@ -97,54 +89,32 @@ struct __nothrow_swappable_with<_Tp, _Up, false> : false_type {};
 } // namespace __detail
 
 template <class _Tp>
-struct __is_swappable
-    : public integral_constant<bool, __detail::__swappable_with<_Tp&>::value>
-{
-};
+struct __is_swappable : public integral_constant<bool, __detail::__swappable_with<_Tp&>::value> {};
 
 template <class _Tp>
-struct __is_nothrow_swappable
-    : public integral_constant<bool, __detail::__nothrow_swappable_with<_Tp&>::value>
-{
-};
+struct __is_nothrow_swappable : public integral_constant<bool, __detail::__nothrow_swappable_with<_Tp&>::value> {};
 
 #if _LIBCPP_STD_VER >= 17
 
 template <class _Tp, class _Up>
 struct _LIBCPP_TEMPLATE_VIS is_swappable_with
-    : public integral_constant<bool, __detail::__swappable_with<_Tp, _Up>::value>
-{
-};
+    : public integral_constant<bool, __detail::__swappable_with<_Tp, _Up>::value> {};
 
 template <class _Tp>
 struct _LIBCPP_TEMPLATE_VIS is_swappable
-    : public __conditional_t<
-        __libcpp_is_referenceable<_Tp>::value,
-        is_swappable_with<
-            __add_lvalue_reference_t<_Tp>,
-            __add_lvalue_reference_t<_Tp> >,
-        false_type
-    >
-{
-};
+    : public __conditional_t<__libcpp_is_referenceable<_Tp>::value,
+                             is_swappable_with<__add_lvalue_reference_t<_Tp>, __add_lvalue_reference_t<_Tp> >,
+                             false_type> {};
 
 template <class _Tp, class _Up>
 struct _LIBCPP_TEMPLATE_VIS is_nothrow_swappable_with
-    : public integral_constant<bool, __detail::__nothrow_swappable_with<_Tp, _Up>::value>
-{
-};
+    : public integral_constant<bool, __detail::__nothrow_swappable_with<_Tp, _Up>::value> {};
 
 template <class _Tp>
 struct _LIBCPP_TEMPLATE_VIS is_nothrow_swappable
-    : public __conditional_t<
-        __libcpp_is_referenceable<_Tp>::value,
-        is_nothrow_swappable_with<
-            __add_lvalue_reference_t<_Tp>,
-            __add_lvalue_reference_t<_Tp> >,
-        false_type
-    >
-{
-};
+    : public __conditional_t<__libcpp_is_referenceable<_Tp>::value,
+                             is_nothrow_swappable_with<__add_lvalue_reference_t<_Tp>, __add_lvalue_reference_t<_Tp> >,
+                             false_type> {};
 
 template <class _Tp, class _Up>
 inline constexpr bool is_swappable_with_v = is_swappable_with<_Tp, _Up>::value;
diff --git a/libcxx/include/__type_traits/is_trivial.h b/libcxx/include/__type_traits/is_trivial.h
index f43904891b21f..0007c7446d5e5 100644
--- a/libcxx/include/__type_traits/is_trivial.h
+++ b/libcxx/include/__type_traits/is_trivial.h
@@ -18,9 +18,8 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_trivial
-    : public integral_constant<bool, __is_trivial(_Tp)>
-    {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_trivial : public integral_constant<bool, __is_trivial(_Tp)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/is_trivially_assignable.h b/libcxx/include/__type_traits/is_trivially_assignable.h
index c16d17732d6b1..19169d13d6d23 100644
--- a/libcxx/include/__type_traits/is_trivially_assignable.h
+++ b/libcxx/include/__type_traits/is_trivially_assignable.h
@@ -19,9 +19,7 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp, class _Arg>
-struct is_trivially_assignable
-    : integral_constant<bool, __is_trivially_assignable(_Tp, _Arg)>
-{ };
+struct is_trivially_assignable : integral_constant<bool, __is_trivially_assignable(_Tp, _Arg)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp, class _Arg>
diff --git a/libcxx/include/__type_traits/is_trivially_constructible.h b/libcxx/include/__type_traits/is_trivially_constructible.h
index a71a49bae255d..4faaf9323cd5b 100644
--- a/libcxx/include/__type_traits/is_trivially_constructible.h
+++ b/libcxx/include/__type_traits/is_trivially_constructible.h
@@ -20,9 +20,7 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp, class... _Args>
 struct _LIBCPP_TEMPLATE_VIS is_trivially_constructible
-    : integral_constant<bool, __is_trivially_constructible(_Tp, _Args...)>
-{
-};
+    : integral_constant<bool, __is_trivially_constructible(_Tp, _Args...)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp, class... _Args>
diff --git a/libcxx/include/__type_traits/is_trivially_copy_assignable.h b/libcxx/include/__type_traits/is_trivially_copy_assignable.h
index b38ab7ef4dac8..b6b3c119746f5 100644
--- a/libcxx/include/__type_traits/is_trivially_copy_assignable.h
+++ b/libcxx/include/__type_traits/is_trivially_copy_assignable.h
@@ -22,11 +22,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp>
 struct _LIBCPP_TEMPLATE_VIS is_trivially_copy_assignable
-    : public integral_constant<
-          bool,
-          __is_trivially_assignable(
-              __add_lvalue_reference_t<_Tp>,
-              __add_lvalue_reference_t<typename add_const<_Tp>::type>)> {};
+    : public integral_constant<bool,
+                               __is_trivially_assignable(__add_lvalue_reference_t<_Tp>,
+                                                         __add_lvalue_reference_t<typename add_const<_Tp>::type>)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/is_trivially_copy_constructible.h b/libcxx/include/__type_traits/is_trivially_copy_constructible.h
index c6c96a051d5bd..8e71fd1fbf8b3 100644
--- a/libcxx/include/__type_traits/is_trivially_copy_constructible.h
+++ b/libcxx/include/__type_traits/is_trivially_copy_constructible.h
@@ -19,9 +19,9 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_trivially_copy_constructible
-    : public integral_constant<bool, __is_trivially_constructible(_Tp, __add_lvalue_reference_t<const _Tp>)>
-    {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_trivially_copy_constructible
+    : public integral_constant<bool, __is_trivially_constructible(_Tp, __add_lvalue_reference_t<const _Tp>)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/is_trivially_copyable.h b/libcxx/include/__type_traits/is_trivially_copyable.h
index a725a0beb4ffa..e92af126ee94d 100644
--- a/libcxx/include/__type_traits/is_trivially_copyable.h
+++ b/libcxx/include/__type_traits/is_trivially_copyable.h
@@ -19,9 +19,8 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_trivially_copyable
-    : public integral_constant<bool, __is_trivially_copyable(_Tp)>
-    {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_trivially_copyable : public integral_constant<bool, __is_trivially_copyable(_Tp)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/is_trivially_default_constructible.h b/libcxx/include/__type_traits/is_trivially_default_constructible.h
index 2b64a69a7ff95..c3b6152a9fcad 100644
--- a/libcxx/include/__type_traits/is_trivially_default_constructible.h
+++ b/libcxx/include/__type_traits/is_trivially_default_constructible.h
@@ -18,9 +18,9 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_trivially_default_constructible
-    : public integral_constant<bool, __is_trivially_constructible(_Tp)>
-    {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_trivially_default_constructible
+    : public integral_constant<bool, __is_trivially_constructible(_Tp)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/is_trivially_destructible.h b/libcxx/include/__type_traits/is_trivially_destructible.h
index e718af015ea72..5f9652f2a5011 100644
--- a/libcxx/include/__type_traits/is_trivially_destructible.h
+++ b/libcxx/include/__type_traits/is_trivially_destructible.h
@@ -21,17 +21,19 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if __has_builtin(__is_trivially_destructible)
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_trivially_destructible
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_trivially_destructible
     : public integral_constant<bool, __is_trivially_destructible(_Tp)> {};
 
 #elif __has_builtin(__has_trivial_destructor)
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_trivially_destructible
-    : public integral_constant<bool, is_destructible<_Tp>::value && __has_trivial_destructor(_Tp)> {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_trivially_destructible
+    : public integral_constant<bool, is_destructible<_Tp>::value&& __has_trivial_destructor(_Tp)> {};
 
 #else
 
-#error is_trivially_destructible is not implemented
+#  error is_trivially_destructible is not implemented
 
 #endif // __has_builtin(__is_trivially_destructible)
 
diff --git a/libcxx/include/__type_traits/is_unbounded_array.h b/libcxx/include/__type_traits/is_unbounded_array.h
index 16bc62d17f34a..d58bb09e10428 100644
--- a/libcxx/include/__type_traits/is_unbounded_array.h
+++ b/libcxx/include/__type_traits/is_unbounded_array.h
@@ -18,17 +18,20 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class>     struct _LIBCPP_TEMPLATE_VIS __libcpp_is_unbounded_array        : false_type {};
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS __libcpp_is_unbounded_array<_Tp[]> : true_type {};
+template <class>
+struct _LIBCPP_TEMPLATE_VIS __libcpp_is_unbounded_array : false_type {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS __libcpp_is_unbounded_array<_Tp[]> : true_type {};
 
 #if _LIBCPP_STD_VER >= 20
 
-template <class>     struct _LIBCPP_TEMPLATE_VIS is_unbounded_array        : false_type {};
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_unbounded_array<_Tp[]> : true_type {};
+template <class>
+struct _LIBCPP_TEMPLATE_VIS is_unbounded_array : false_type {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_unbounded_array<_Tp[]> : true_type {};
 
 template <class _Tp>
-inline constexpr
-bool is_unbounded_array_v  = is_unbounded_array<_Tp>::value;
+inline constexpr bool is_unbounded_array_v = is_unbounded_array<_Tp>::value;
 
 #endif
 
diff --git a/libcxx/include/__type_traits/is_union.h b/libcxx/include/__type_traits/is_union.h
index 6d2f47a116bdb..1f009d993545b 100644
--- a/libcxx/include/__type_traits/is_union.h
+++ b/libcxx/include/__type_traits/is_union.h
@@ -18,8 +18,8 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_union
-    : public integral_constant<bool, __is_union(_Tp)> {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_union : public integral_constant<bool, __is_union(_Tp)> {};
 
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
diff --git a/libcxx/include/__type_traits/is_unsigned.h b/libcxx/include/__type_traits/is_unsigned.h
index d955106d5333b..48c5751ed70d8 100644
--- a/libcxx/include/__type_traits/is_unsigned.h
+++ b/libcxx/include/__type_traits/is_unsigned.h
@@ -22,13 +22,13 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if __has_builtin(__is_unsigned)
 
-template<class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_unsigned : _BoolConstant<__is_unsigned(_Tp)> { };
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_unsigned : _BoolConstant<__is_unsigned(_Tp)> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_unsigned_v = __is_unsigned(_Tp);
-#endif
+#  endif
 
 #else // __has_builtin(__is_unsigned)
 
@@ -36,19 +36,21 @@ template <class _Tp, bool = is_integral<_Tp>::value>
 struct __libcpp_is_unsigned_impl : public _BoolConstant<(_Tp(0) < _Tp(-1))> {};
 
 template <class _Tp>
-struct __libcpp_is_unsigned_impl<_Tp, false> : public false_type {};  // floating point
+struct __libcpp_is_unsigned_impl<_Tp, false> : public false_type {}; // floating point
 
 template <class _Tp, bool = is_arithmetic<_Tp>::value>
 struct __libcpp_is_unsigned : public __libcpp_is_unsigned_impl<_Tp> {};
 
-template <class _Tp> struct __libcpp_is_unsigned<_Tp, false> : public false_type {};
+template <class _Tp>
+struct __libcpp_is_unsigned<_Tp, false> : public false_type {};
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_unsigned : public __libcpp_is_unsigned<_Tp> {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_unsigned : public __libcpp_is_unsigned<_Tp> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_unsigned_v = is_unsigned<_Tp>::value;
-#endif
+#  endif
 
 #endif // __has_builtin(__is_unsigned)
 
diff --git a/libcxx/include/__type_traits/is_unsigned_integer.h b/libcxx/include/__type_traits/is_unsigned_integer.h
index 54b29acd9ea38..265894b32d4fc 100644
--- a/libcxx/include/__type_traits/is_unsigned_integer.h
+++ b/libcxx/include/__type_traits/is_unsigned_integer.h
@@ -18,15 +18,17 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp> struct __libcpp_is_unsigned_integer : public false_type {};
-template <> struct __libcpp_is_unsigned_integer<unsigned char>      : public true_type {};
-template <> struct __libcpp_is_unsigned_integer<unsigned short>     : public true_type {};
-template <> struct __libcpp_is_unsigned_integer<unsigned int>       : public true_type {};
-template <> struct __libcpp_is_unsigned_integer<unsigned long>      : public true_type {};
-template <> struct __libcpp_is_unsigned_integer<unsigned long long> : public true_type {};
+// clang-format off
+template <class _Tp> struct __libcpp_is_unsigned_integer                     : public false_type {};
+template <>          struct __libcpp_is_unsigned_integer<unsigned char>      : public true_type {};
+template <>          struct __libcpp_is_unsigned_integer<unsigned short>     : public true_type {};
+template <>          struct __libcpp_is_unsigned_integer<unsigned int>       : public true_type {};
+template <>          struct __libcpp_is_unsigned_integer<unsigned long>      : public true_type {};
+template <>          struct __libcpp_is_unsigned_integer<unsigned long long> : public true_type {};
 #ifndef _LIBCPP_HAS_NO_INT128
-template <> struct __libcpp_is_unsigned_integer<__uint128_t>        : public true_type {};
+template <>          struct __libcpp_is_unsigned_integer<__uint128_t>        : public true_type {};
 #endif
+// clang-format on
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__type_traits/is_valid_expansion.h b/libcxx/include/__type_traits/is_valid_expansion.h
index db7d7e5f1d057..346bc98b110ff 100644
--- a/libcxx/include/__type_traits/is_valid_expansion.h
+++ b/libcxx/include/__type_traits/is_valid_expansion.h
@@ -18,12 +18,12 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <template <class...> class _Templ, class ..._Args, class = _Templ<_Args...> >
+template <template <class...> class _Templ, class... _Args, class = _Templ<_Args...> >
 true_type __sfinae_test_impl(int);
-template <template <class...> class, class ...>
+template <template <class...> class, class...>
 false_type __sfinae_test_impl(...);
 
-template <template <class ...> class _Templ, class ..._Args>
+template <template <class...> class _Templ, class... _Args>
 using _IsValidExpansion _LIBCPP_NODEBUG = decltype(std::__sfinae_test_impl<_Templ, _Args...>(0));
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/is_void.h b/libcxx/include/__type_traits/is_void.h
index 7e8b8b64178eb..4c27060530c8e 100644
--- a/libcxx/include/__type_traits/is_void.h
+++ b/libcxx/include/__type_traits/is_void.h
@@ -23,22 +23,22 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 #if __has_builtin(__is_void)
 
 template <class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_void : _BoolConstant<__is_void(_Tp)> { };
+struct _LIBCPP_TEMPLATE_VIS is_void : _BoolConstant<__is_void(_Tp)> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_void_v = __is_void(_Tp);
-#endif
+#  endif
 
 #else
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_void
-    : public is_same<__remove_cv_t<_Tp>, void> {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_void : public is_same<__remove_cv_t<_Tp>, void> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_void_v = is_void<_Tp>::value;
-#endif
+#  endif
 
 #endif // __has_builtin(__is_void)
 
diff --git a/libcxx/include/__type_traits/is_volatile.h b/libcxx/include/__type_traits/is_volatile.h
index c020220e03070..87960a819c8fc 100644
--- a/libcxx/include/__type_traits/is_volatile.h
+++ b/libcxx/include/__type_traits/is_volatile.h
@@ -21,22 +21,24 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 #if __has_builtin(__is_volatile)
 
 template <class _Tp>
-struct _LIBCPP_TEMPLATE_VIS is_volatile : _BoolConstant<__is_volatile(_Tp)> { };
+struct _LIBCPP_TEMPLATE_VIS is_volatile : _BoolConstant<__is_volatile(_Tp)> {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_volatile_v = __is_volatile(_Tp);
-#endif
+#  endif
 
 #else
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_volatile               : public false_type {};
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS is_volatile<_Tp volatile> : public true_type {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_volatile : public false_type {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS is_volatile<_Tp volatile> : public true_type {};
 
-#if _LIBCPP_STD_VER >= 17
+#  if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 inline constexpr bool is_volatile_v = is_volatile<_Tp>::value;
-#endif
+#  endif
 
 #endif // __has_builtin(__is_volatile)
 
diff --git a/libcxx/include/__type_traits/lazy.h b/libcxx/include/__type_traits/lazy.h
index 6874b06f3c5af..80826f1d64f60 100644
--- a/libcxx/include/__type_traits/lazy.h
+++ b/libcxx/include/__type_traits/lazy.h
@@ -17,7 +17,7 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <template <class...> class _Func, class ..._Args>
+template <template <class...> class _Func, class... _Args>
 struct _Lazy : _Func<_Args...> {};
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/make_32_64_or_128_bit.h b/libcxx/include/__type_traits/make_32_64_or_128_bit.h
index 30595712b67a3..f7f2e81735daf 100644
--- a/libcxx/include/__type_traits/make_32_64_or_128_bit.h
+++ b/libcxx/include/__type_traits/make_32_64_or_128_bit.h
@@ -28,20 +28,21 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 /// The restriction is the same as the integral version of to_char.
 template <class _Tp>
 #if _LIBCPP_STD_VER >= 20
-  requires (is_signed_v<_Tp> || is_unsigned_v<_Tp> || is_same_v<_Tp, char>)
+  requires(is_signed_v<_Tp> || is_unsigned_v<_Tp> || is_same_v<_Tp, char>)
 #endif
+// clang-format off
 using __make_32_64_or_128_bit_t =
-  __copy_unsigned_t<_Tp,
-    __conditional_t<sizeof(_Tp) <= sizeof(int32_t),    int32_t,
-    __conditional_t<sizeof(_Tp) <= sizeof(int64_t),    int64_t,
+    __copy_unsigned_t<_Tp,
+        __conditional_t<sizeof(_Tp) <= sizeof(int32_t),    int32_t,
+        __conditional_t<sizeof(_Tp) <= sizeof(int64_t),    int64_t,
 #ifndef _LIBCPP_HAS_NO_INT128
-    __conditional_t<sizeof(_Tp) <= sizeof(__int128_t), __int128_t,
-    /* else */                                         void>
+        __conditional_t<sizeof(_Tp) <= sizeof(__int128_t), __int128_t,
+        /* else */                                         void>
 #else
-    /* else */                                         void
+        /* else */                                         void
 #endif
-    > >
-  >;
+    > > >;
+// clang-format on
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__type_traits/make_const_lvalue_ref.h b/libcxx/include/__type_traits/make_const_lvalue_ref.h
index deea40d5e61a1..469d4cb31ef7d 100644
--- a/libcxx/include/__type_traits/make_const_lvalue_ref.h
+++ b/libcxx/include/__type_traits/make_const_lvalue_ref.h
@@ -18,7 +18,7 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template<class _Tp>
+template <class _Tp>
 using __make_const_lvalue_ref = const __libcpp_remove_reference_t<_Tp>&;
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/make_signed.h b/libcxx/include/__type_traits/make_signed.h
index a8142a43cb515..1a8a35f3859d2 100644
--- a/libcxx/include/__type_traits/make_signed.h
+++ b/libcxx/include/__type_traits/make_signed.h
@@ -29,30 +29,31 @@ template <class _Tp>
 using __make_signed_t = __make_signed(_Tp);
 
 #else
-typedef
-    __type_list<signed char,
-    __type_list<signed short,
-    __type_list<signed int,
-    __type_list<signed long,
-    __type_list<signed long long,
+// clang-format off
+typedef __type_list<signed char,
+        __type_list<signed short,
+        __type_list<signed int,
+        __type_list<signed long,
+        __type_list<signed long long,
 #  ifndef _LIBCPP_HAS_NO_INT128
-    __type_list<__int128_t,
+        __type_list<__int128_t,
 #  endif
-    __nat
+        __nat
 #  ifndef _LIBCPP_HAS_NO_INT128
-    >
+        >
 #  endif
-    > > > > > __signed_types;
+        > > > > > __signed_types;
+// clang-format on
 
 template <class _Tp, bool = is_integral<_Tp>::value || is_enum<_Tp>::value>
-struct __make_signed {};
+struct __make_signed{};
 
 template <class _Tp>
-struct __make_signed<_Tp, true>
-{
-    typedef typename __find_first<__signed_types, sizeof(_Tp)>::type type;
+struct __make_signed<_Tp, true> {
+  typedef typename __find_first<__signed_types, sizeof(_Tp)>::type type;
 };
 
+// clang-format off
 template <> struct __make_signed<bool,               true> {};
 template <> struct __make_signed<  signed short,     true> {typedef short     type;};
 template <> struct __make_signed<unsigned short,     true> {typedef short     type;};
@@ -66,6 +67,7 @@ template <> struct __make_signed<unsigned long long, true> {typedef long long ty
 template <> struct __make_signed<__int128_t,         true> {typedef __int128_t type;};
 template <> struct __make_signed<__uint128_t,        true> {typedef __int128_t type;};
 #  endif
+// clang-format on
 
 template <class _Tp>
 using __make_signed_t = __apply_cv_t<_Tp, typename __make_signed<__remove_cv_t<_Tp> >::type>;
@@ -78,7 +80,8 @@ struct make_signed {
 };
 
 #if _LIBCPP_STD_VER >= 14
-template <class _Tp> using make_signed_t = __make_signed_t<_Tp>;
+template <class _Tp>
+using make_signed_t = __make_signed_t<_Tp>;
 #endif
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/make_unsigned.h b/libcxx/include/__type_traits/make_unsigned.h
index 0ee4347a721aa..98967371e7738 100644
--- a/libcxx/include/__type_traits/make_unsigned.h
+++ b/libcxx/include/__type_traits/make_unsigned.h
@@ -31,30 +31,31 @@ template <class _Tp>
 using __make_unsigned_t = __make_unsigned(_Tp);
 
 #else
-typedef
-    __type_list<unsigned char,
-    __type_list<unsigned short,
-    __type_list<unsigned int,
-    __type_list<unsigned long,
-    __type_list<unsigned long long,
+// clang-format off
+typedef __type_list<unsigned char,
+        __type_list<unsigned short,
+        __type_list<unsigned int,
+        __type_list<unsigned long,
+        __type_list<unsigned long long,
 #  ifndef _LIBCPP_HAS_NO_INT128
-    __type_list<__uint128_t,
+        __type_list<__uint128_t,
 #  endif
-    __nat
+        __nat
 #  ifndef _LIBCPP_HAS_NO_INT128
-    >
+        >
 #  endif
-    > > > > > __unsigned_types;
+        > > > > > __unsigned_types;
+// clang-format on
 
 template <class _Tp, bool = is_integral<_Tp>::value || is_enum<_Tp>::value>
-struct __make_unsigned {};
+struct __make_unsigned{};
 
 template <class _Tp>
-struct __make_unsigned<_Tp, true>
-{
-    typedef typename __find_first<__unsigned_types, sizeof(_Tp)>::type type;
+struct __make_unsigned<_Tp, true> {
+  typedef typename __find_first<__unsigned_types, sizeof(_Tp)>::type type;
 };
 
+// clang-format off
 template <> struct __make_unsigned<bool,               true> {};
 template <> struct __make_unsigned<  signed short,     true> {typedef unsigned short     type;};
 template <> struct __make_unsigned<unsigned short,     true> {typedef unsigned short     type;};
@@ -68,6 +69,7 @@ template <> struct __make_unsigned<unsigned long long, true> {typedef unsigned l
 template <> struct __make_unsigned<__int128_t,         true> {typedef __uint128_t        type;};
 template <> struct __make_unsigned<__uint128_t,        true> {typedef __uint128_t        type;};
 #  endif
+// clang-format on
 
 template <class _Tp>
 using __make_unsigned_t = __apply_cv_t<_Tp, typename __make_unsigned<__remove_cv_t<_Tp> >::type>;
@@ -80,14 +82,14 @@ struct make_unsigned {
 };
 
 #if _LIBCPP_STD_VER >= 14
-template <class _Tp> using make_unsigned_t = __make_unsigned_t<_Tp>;
+template <class _Tp>
+using make_unsigned_t = __make_unsigned_t<_Tp>;
 #endif
 
 #ifndef _LIBCPP_CXX03_LANG
 template <class _Tp>
-_LIBCPP_HIDE_FROM_ABI constexpr
-__make_unsigned_t<_Tp> __to_unsigned_like(_Tp __x) noexcept {
-    return static_cast<__make_unsigned_t<_Tp> >(__x);
+_LIBCPP_HIDE_FROM_ABI constexpr __make_unsigned_t<_Tp> __to_unsigned_like(_Tp __x) noexcept {
+  return static_cast<__make_unsigned_t<_Tp> >(__x);
 }
 #endif
 
diff --git a/libcxx/include/__type_traits/maybe_const.h b/libcxx/include/__type_traits/maybe_const.h
index 8403f6aba7468..25fba58fb7730 100644
--- a/libcxx/include/__type_traits/maybe_const.h
+++ b/libcxx/include/__type_traits/maybe_const.h
@@ -18,7 +18,7 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template<bool _Const, class _Tp>
+template <bool _Const, class _Tp>
 using __maybe_const = __conditional_t<_Const, const _Tp, _Tp>;
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/nat.h b/libcxx/include/__type_traits/nat.h
index 5216ef5204202..9f39b806814e2 100644
--- a/libcxx/include/__type_traits/nat.h
+++ b/libcxx/include/__type_traits/nat.h
@@ -17,13 +17,12 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-struct __nat
-{
+struct __nat {
 #ifndef _LIBCPP_CXX03_LANG
-    __nat() = delete;
-    __nat(const __nat&) = delete;
-    __nat& operator=(const __nat&) = delete;
-    ~__nat() = delete;
+  __nat()                        = delete;
+  __nat(const __nat&)            = delete;
+  __nat& operator=(const __nat&) = delete;
+  ~__nat()                       = delete;
 #endif
 };
 
diff --git a/libcxx/include/__type_traits/negation.h b/libcxx/include/__type_traits/negation.h
index 3a70ef1408cd6..a72e62d3f96e0 100644
--- a/libcxx/include/__type_traits/negation.h
+++ b/libcxx/include/__type_traits/negation.h
@@ -24,7 +24,7 @@ struct _Not : _BoolConstant<!_Pred::value> {};
 #if _LIBCPP_STD_VER >= 17
 template <class _Tp>
 struct negation : _Not<_Tp> {};
-template<class _Tp>
+template <class _Tp>
 inline constexpr bool negation_v = !_Tp::value;
 #endif // _LIBCPP_STD_VER >= 17
 
diff --git a/libcxx/include/__type_traits/noexcept_move_assign_container.h b/libcxx/include/__type_traits/noexcept_move_assign_container.h
index 65384d05ae1b9..1d6c07e8cf96d 100644
--- a/libcxx/include/__type_traits/noexcept_move_assign_container.h
+++ b/libcxx/include/__type_traits/noexcept_move_assign_container.h
@@ -20,15 +20,17 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <typename _Alloc, typename _Traits=allocator_traits<_Alloc> >
-struct __noexcept_move_assign_container : public integral_constant<bool,
-    _Traits::propagate_on_container_move_assignment::value
+template <typename _Alloc, typename _Traits = allocator_traits<_Alloc> >
+struct __noexcept_move_assign_container
+    : public integral_constant<bool,
+                               _Traits::propagate_on_container_move_assignment::value
 #if _LIBCPP_STD_VER >= 17
-        || _Traits::is_always_equal::value
+                                   || _Traits::is_always_equal::value
 #else
-        && is_nothrow_move_assignable<_Alloc>::value
+                                   && is_nothrow_move_assignable<_Alloc>::value
 #endif
-    > {};
+                               > {
+};
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__type_traits/promote.h b/libcxx/include/__type_traits/promote.h
index e01e7781c5b3d..018bd69df2604 100644
--- a/libcxx/include/__type_traits/promote.h
+++ b/libcxx/include/__type_traits/promote.h
@@ -21,73 +21,69 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp>
-struct __numeric_type
-{
-   static void __test(...);
-   static float __test(float);
-   static double __test(char);
-   static double __test(int);
-   static double __test(unsigned);
-   static double __test(long);
-   static double __test(unsigned long);
-   static double __test(long long);
-   static double __test(unsigned long long);
+struct __numeric_type {
+  static void __test(...);
+  static float __test(float);
+  static double __test(char);
+  static double __test(int);
+  static double __test(unsigned);
+  static double __test(long);
+  static double __test(unsigned long);
+  static double __test(long long);
+  static double __test(unsigned long long);
 #ifndef _LIBCPP_HAS_NO_INT128
-   static double __test(__int128_t);
-   static double __test(__uint128_t);
+  static double __test(__int128_t);
+  static double __test(__uint128_t);
 #endif
-   static double __test(double);
-   static long double __test(long double);
+  static double __test(double);
+  static long double __test(long double);
 
-   typedef decltype(__test(std::declval<_Tp>())) type;
-   static const bool value = _IsNotSame<type, void>::value;
+  typedef decltype(__test(std::declval<_Tp>())) type;
+  static const bool value = _IsNotSame<type, void>::value;
 };
 
 template <>
-struct __numeric_type<void>
-{
-   static const bool value = true;
+struct __numeric_type<void> {
+  static const bool value = true;
 };
 
-template <class _A1, class _A2 = void, class _A3 = void,
-          bool = __numeric_type<_A1>::value &&
-                 __numeric_type<_A2>::value &&
-                 __numeric_type<_A3>::value>
-class __promote_imp
-{
+template <class _A1,
+          class _A2 = void,
+          class _A3 = void,
+          bool      = __numeric_type<_A1>::value&& __numeric_type<_A2>::value&& __numeric_type<_A3>::value>
+class __promote_imp {
 public:
-    static const bool value = false;
+  static const bool value = false;
 };
 
 template <class _A1, class _A2, class _A3>
-class __promote_imp<_A1, _A2, _A3, true>
-{
+class __promote_imp<_A1, _A2, _A3, true> {
 private:
-    typedef typename __promote_imp<_A1>::type __type1;
-    typedef typename __promote_imp<_A2>::type __type2;
-    typedef typename __promote_imp<_A3>::type __type3;
+  typedef typename __promote_imp<_A1>::type __type1;
+  typedef typename __promote_imp<_A2>::type __type2;
+  typedef typename __promote_imp<_A3>::type __type3;
+
 public:
-    typedef decltype(__type1() + __type2() + __type3()) type;
-    static const bool value = true;
+  typedef decltype(__type1() + __type2() + __type3()) type;
+  static const bool value = true;
 };
 
 template <class _A1, class _A2>
-class __promote_imp<_A1, _A2, void, true>
-{
+class __promote_imp<_A1, _A2, void, true> {
 private:
-    typedef typename __promote_imp<_A1>::type __type1;
-    typedef typename __promote_imp<_A2>::type __type2;
+  typedef typename __promote_imp<_A1>::type __type1;
+  typedef typename __promote_imp<_A2>::type __type2;
+
 public:
-    typedef decltype(__type1() + __type2()) type;
-    static const bool value = true;
+  typedef decltype(__type1() + __type2()) type;
+  static const bool value = true;
 };
 
 template <class _A1>
-class __promote_imp<_A1, void, void, true>
-{
+class __promote_imp<_A1, void, void, true> {
 public:
-    typedef typename __numeric_type<_A1>::type type;
-    static const bool value = true;
+  typedef typename __numeric_type<_A1>::type type;
+  static const bool value = true;
 };
 
 template <class _A1, class _A2 = void, class _A3 = void>
diff --git a/libcxx/include/__type_traits/rank.h b/libcxx/include/__type_traits/rank.h
index 5f3fab89d95d0..7f6fad1c54024 100644
--- a/libcxx/include/__type_traits/rank.h
+++ b/libcxx/include/__type_traits/rank.h
@@ -27,12 +27,12 @@ struct rank : integral_constant<size_t, __array_rank(_Tp)> {};
 
 #else
 
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS rank
-    : public integral_constant<size_t, 0> {};
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS rank<_Tp[]>
-    : public integral_constant<size_t, rank<_Tp>::value + 1> {};
-template <class _Tp, size_t _Np> struct _LIBCPP_TEMPLATE_VIS rank<_Tp[_Np]>
-    : public integral_constant<size_t, rank<_Tp>::value + 1> {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS rank : public integral_constant<size_t, 0> {};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS rank<_Tp[]> : public integral_constant<size_t, rank<_Tp>::value + 1> {};
+template <class _Tp, size_t _Np>
+struct _LIBCPP_TEMPLATE_VIS rank<_Tp[_Np]> : public integral_constant<size_t, rank<_Tp>::value + 1> {};
 
 #endif // __has_builtin(__array_rank)
 
diff --git a/libcxx/include/__type_traits/remove_all_extents.h b/libcxx/include/__type_traits/remove_all_extents.h
index f7e32251c0cf9..d5373b51f5221 100644
--- a/libcxx/include/__type_traits/remove_all_extents.h
+++ b/libcxx/include/__type_traits/remove_all_extents.h
@@ -27,19 +27,26 @@ struct remove_all_extents {
 template <class _Tp>
 using __remove_all_extents_t = __remove_all_extents(_Tp);
 #else
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS remove_all_extents
-    {typedef _Tp type;};
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS remove_all_extents<_Tp[]>
-    {typedef typename remove_all_extents<_Tp>::type type;};
-template <class _Tp, size_t _Np> struct _LIBCPP_TEMPLATE_VIS remove_all_extents<_Tp[_Np]>
-    {typedef typename remove_all_extents<_Tp>::type type;};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS remove_all_extents {
+  typedef _Tp type;
+};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS remove_all_extents<_Tp[]> {
+  typedef typename remove_all_extents<_Tp>::type type;
+};
+template <class _Tp, size_t _Np>
+struct _LIBCPP_TEMPLATE_VIS remove_all_extents<_Tp[_Np]> {
+  typedef typename remove_all_extents<_Tp>::type type;
+};
 
 template <class _Tp>
 using __remove_all_extents_t = typename remove_all_extents<_Tp>::type;
 #endif // __has_builtin(__remove_all_extents)
 
 #if _LIBCPP_STD_VER >= 14
-template <class _Tp> using remove_all_extents_t = __remove_all_extents_t<_Tp>;
+template <class _Tp>
+using remove_all_extents_t = __remove_all_extents_t<_Tp>;
 #endif
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/remove_const.h b/libcxx/include/__type_traits/remove_const.h
index 37f0609e4dbeb..a3f0648c47850 100644
--- a/libcxx/include/__type_traits/remove_const.h
+++ b/libcxx/include/__type_traits/remove_const.h
@@ -26,15 +26,22 @@ struct remove_const {
 template <class _Tp>
 using __remove_const_t = __remove_const(_Tp);
 #else
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS remove_const            {typedef _Tp type;};
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS remove_const<const _Tp> {typedef _Tp type;};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS remove_const {
+  typedef _Tp type;
+};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS remove_const<const _Tp> {
+  typedef _Tp type;
+};
 
 template <class _Tp>
 using __remove_const_t = typename remove_const<_Tp>::type;
 #endif // __has_builtin(__remove_const)
 
 #if _LIBCPP_STD_VER >= 14
-template <class _Tp> using remove_const_t = __remove_const_t<_Tp>;
+template <class _Tp>
+using remove_const_t = __remove_const_t<_Tp>;
 #endif
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/remove_cv.h b/libcxx/include/__type_traits/remove_cv.h
index 7eb894074a058..8fe8fb0e49596 100644
--- a/libcxx/include/__type_traits/remove_cv.h
+++ b/libcxx/include/__type_traits/remove_cv.h
@@ -28,15 +28,18 @@ struct remove_cv {
 template <class _Tp>
 using __remove_cv_t = __remove_cv(_Tp);
 #else
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS remove_cv
-{typedef __remove_volatile_t<__remove_const_t<_Tp> > type;};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS remove_cv {
+  typedef __remove_volatile_t<__remove_const_t<_Tp> > type;
+};
 
 template <class _Tp>
 using __remove_cv_t = __remove_volatile_t<__remove_const_t<_Tp> >;
 #endif // __has_builtin(__remove_cv)
 
 #if _LIBCPP_STD_VER >= 14
-template <class _Tp> using remove_cv_t = __remove_cv_t<_Tp>;
+template <class _Tp>
+using remove_cv_t = __remove_cv_t<_Tp>;
 #endif
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/remove_cvref.h b/libcxx/include/__type_traits/remove_cvref.h
index f1111fd9429bf..4dc950ac31ad8 100644
--- a/libcxx/include/__type_traits/remove_cvref.h
+++ b/libcxx/include/__type_traits/remove_cvref.h
@@ -34,10 +34,11 @@ struct __is_same_uncvref : _IsSame<__remove_cvref_t<_Tp>, __remove_cvref_t<_Up>
 #if _LIBCPP_STD_VER >= 20
 template <class _Tp>
 struct remove_cvref {
-    using type _LIBCPP_NODEBUG = __remove_cvref_t<_Tp>;
+  using type _LIBCPP_NODEBUG = __remove_cvref_t<_Tp>;
 };
 
-template <class _Tp> using remove_cvref_t = __remove_cvref_t<_Tp>;
+template <class _Tp>
+using remove_cvref_t = __remove_cvref_t<_Tp>;
 #endif
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/remove_extent.h b/libcxx/include/__type_traits/remove_extent.h
index 65dfcdc00aa8d..fe37b5c7266c6 100644
--- a/libcxx/include/__type_traits/remove_extent.h
+++ b/libcxx/include/__type_traits/remove_extent.h
@@ -27,19 +27,26 @@ struct remove_extent {
 template <class _Tp>
 using __remove_extent_t = __remove_extent(_Tp);
 #else
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS remove_extent
-    {typedef _Tp type;};
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS remove_extent<_Tp[]>
-    {typedef _Tp type;};
-template <class _Tp, size_t _Np> struct _LIBCPP_TEMPLATE_VIS remove_extent<_Tp[_Np]>
-    {typedef _Tp type;};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS remove_extent {
+  typedef _Tp type;
+};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS remove_extent<_Tp[]> {
+  typedef _Tp type;
+};
+template <class _Tp, size_t _Np>
+struct _LIBCPP_TEMPLATE_VIS remove_extent<_Tp[_Np]> {
+  typedef _Tp type;
+};
 
 template <class _Tp>
 using __remove_extent_t = typename remove_extent<_Tp>::type;
 #endif // __has_builtin(__remove_extent)
 
 #if _LIBCPP_STD_VER >= 14
-template <class _Tp> using remove_extent_t = __remove_extent_t<_Tp>;
+template <class _Tp>
+using remove_extent_t = __remove_extent_t<_Tp>;
 #endif
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/remove_pointer.h b/libcxx/include/__type_traits/remove_pointer.h
index 8b3a9c7c039f4..54390a1939f7d 100644
--- a/libcxx/include/__type_traits/remove_pointer.h
+++ b/libcxx/include/__type_traits/remove_pointer.h
@@ -26,18 +26,21 @@ struct remove_pointer {
 template <class _Tp>
 using __remove_pointer_t = __remove_pointer(_Tp);
 #else
+// clang-format off
 template <class _Tp> struct _LIBCPP_TEMPLATE_VIS remove_pointer                      {typedef _LIBCPP_NODEBUG _Tp type;};
 template <class _Tp> struct _LIBCPP_TEMPLATE_VIS remove_pointer<_Tp*>                {typedef _LIBCPP_NODEBUG _Tp type;};
 template <class _Tp> struct _LIBCPP_TEMPLATE_VIS remove_pointer<_Tp* const>          {typedef _LIBCPP_NODEBUG _Tp type;};
 template <class _Tp> struct _LIBCPP_TEMPLATE_VIS remove_pointer<_Tp* volatile>       {typedef _LIBCPP_NODEBUG _Tp type;};
 template <class _Tp> struct _LIBCPP_TEMPLATE_VIS remove_pointer<_Tp* const volatile> {typedef _LIBCPP_NODEBUG _Tp type;};
+// clang-format on
 
 template <class _Tp>
 using __remove_pointer_t = typename remove_pointer<_Tp>::type;
 #endif // !defined(_LIBCPP_WORKAROUND_OBJCXX_COMPILER_INTRINSICS) && __has_builtin(__remove_pointer)
 
 #if _LIBCPP_STD_VER >= 14
-template <class _Tp> using remove_pointer_t = __remove_pointer_t<_Tp>;
+template <class _Tp>
+using remove_pointer_t = __remove_pointer_t<_Tp>;
 #endif
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/remove_reference.h b/libcxx/include/__type_traits/remove_reference.h
index 1d510c248abf9..fd66417bd84fe 100644
--- a/libcxx/include/__type_traits/remove_reference.h
+++ b/libcxx/include/__type_traits/remove_reference.h
@@ -27,16 +27,19 @@ struct remove_reference {
 template <class _Tp>
 using __libcpp_remove_reference_t = __remove_reference_t(_Tp);
 #else
+// clang-format off
 template <class _Tp> struct _LIBCPP_TEMPLATE_VIS remove_reference        {typedef _LIBCPP_NODEBUG _Tp type;};
 template <class _Tp> struct _LIBCPP_TEMPLATE_VIS remove_reference<_Tp&>  {typedef _LIBCPP_NODEBUG _Tp type;};
 template <class _Tp> struct _LIBCPP_TEMPLATE_VIS remove_reference<_Tp&&> {typedef _LIBCPP_NODEBUG _Tp type;};
+// clang-format on
 
 template <class _Tp>
 using __libcpp_remove_reference_t = typename remove_reference<_Tp>::type;
 #endif // __has_builtin(__remove_reference_t)
 
 #if _LIBCPP_STD_VER >= 14
-template <class _Tp> using remove_reference_t = __libcpp_remove_reference_t<_Tp>;
+template <class _Tp>
+using remove_reference_t = __libcpp_remove_reference_t<_Tp>;
 #endif
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/remove_volatile.h b/libcxx/include/__type_traits/remove_volatile.h
index cc9fb55e04028..7600ae0ec5167 100644
--- a/libcxx/include/__type_traits/remove_volatile.h
+++ b/libcxx/include/__type_traits/remove_volatile.h
@@ -26,15 +26,22 @@ struct remove_volatile {
 template <class _Tp>
 using __remove_volatile_t = __remove_volatile(_Tp);
 #else
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS remove_volatile               {typedef _Tp type;};
-template <class _Tp> struct _LIBCPP_TEMPLATE_VIS remove_volatile<volatile _Tp> {typedef _Tp type;};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS remove_volatile {
+  typedef _Tp type;
+};
+template <class _Tp>
+struct _LIBCPP_TEMPLATE_VIS remove_volatile<volatile _Tp> {
+  typedef _Tp type;
+};
 
 template <class _Tp>
 using __remove_volatile_t = typename remove_volatile<_Tp>::type;
 #endif // __has_builtin(__remove_volatile)
 
 #if _LIBCPP_STD_VER >= 14
-template <class _Tp> using remove_volatile_t = __remove_volatile_t<_Tp>;
+template <class _Tp>
+using remove_volatile_t = __remove_volatile_t<_Tp>;
 #endif
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/result_of.h b/libcxx/include/__type_traits/result_of.h
index dff28286efd35..f00fa8e9be7f7 100644
--- a/libcxx/include/__type_traits/result_of.h
+++ b/libcxx/include/__type_traits/result_of.h
@@ -21,18 +21,17 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 // result_of
 
 #if _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_TYPE_TRAITS)
-template <class _Callable> class _LIBCPP_DEPRECATED_IN_CXX17 result_of;
-
-template <class _Fp, class ..._Args>
-class _LIBCPP_TEMPLATE_VIS result_of<_Fp(_Args...)>
-    : public __invoke_of<_Fp, _Args...>
-{
-};
-
-#if _LIBCPP_STD_VER >= 14
-template <class _Tp> using result_of_t _LIBCPP_DEPRECATED_IN_CXX17 = typename result_of<_Tp>::type;
-#endif // _LIBCPP_STD_VER >= 14
-#endif // _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_TYPE_TRAITS)
+template <class _Callable>
+class _LIBCPP_DEPRECATED_IN_CXX17 result_of;
+
+template <class _Fp, class... _Args>
+class _LIBCPP_TEMPLATE_VIS result_of<_Fp(_Args...)> : public __invoke_of<_Fp, _Args...> {};
+
+#  if _LIBCPP_STD_VER >= 14
+template <class _Tp>
+using result_of_t _LIBCPP_DEPRECATED_IN_CXX17 = typename result_of<_Tp>::type;
+#  endif // _LIBCPP_STD_VER >= 14
+#endif   // _LIBCPP_STD_VER <= 17 || defined(_LIBCPP_ENABLE_CXX20_REMOVED_TYPE_TRAITS)
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__type_traits/strip_signature.h b/libcxx/include/__type_traits/strip_signature.h
index 2bb7393c1ee7f..3fe79592f55b8 100644
--- a/libcxx/include/__type_traits/strip_signature.h
+++ b/libcxx/include/__type_traits/strip_signature.h
@@ -19,23 +19,24 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template<class _Fp>
+template <class _Fp>
 struct __strip_signature;
 
 #  if defined(__cpp_static_call_operator) && __cpp_static_call_operator >= 202207L
 
 template <class _Rp, class... _Args>
-struct __strip_signature<_Rp(*)(_Args...)> {
+struct __strip_signature<_Rp (*)(_Args...)> {
   using type = _Rp(_Args...);
 };
 
 template <class _Rp, class... _Args>
-struct __strip_signature<_Rp(*)(_Args...) noexcept> {
+struct __strip_signature<_Rp (*)(_Args...) noexcept> {
   using type = _Rp(_Args...);
 };
 
 #  endif // defined(__cpp_static_call_operator) && __cpp_static_call_operator >= 202207L
 
+// clang-format off
 template<class _Rp, class _Gp, class ..._Ap>
 struct __strip_signature<_Rp (_Gp::*) (_Ap...)> { using type = _Rp(_Ap...); };
 template<class _Rp, class _Gp, class ..._Ap>
@@ -71,6 +72,7 @@ template<class _Rp, class _Gp, class ..._Ap>
 struct __strip_signature<_Rp (_Gp::*) (_Ap...) volatile & noexcept> { using type = _Rp(_Ap...); };
 template<class _Rp, class _Gp, class ..._Ap>
 struct __strip_signature<_Rp (_Gp::*) (_Ap...) const volatile & noexcept> { using type = _Rp(_Ap...); };
+// clang-format on
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__type_traits/type_identity.h b/libcxx/include/__type_traits/type_identity.h
index 92f288f4c0b1d..b0b5a1277d596 100644
--- a/libcxx/include/__type_traits/type_identity.h
+++ b/libcxx/include/__type_traits/type_identity.h
@@ -18,14 +18,20 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp>
-struct __type_identity { typedef _Tp type; };
+struct __type_identity {
+  typedef _Tp type;
+};
 
 template <class _Tp>
 using __type_identity_t _LIBCPP_NODEBUG = typename __type_identity<_Tp>::type;
 
 #if _LIBCPP_STD_VER >= 20
-template<class _Tp> struct type_identity { typedef _Tp type; };
-template<class _Tp> using type_identity_t = typename type_identity<_Tp>::type;
+template <class _Tp>
+struct type_identity {
+  typedef _Tp type;
+};
+template <class _Tp>
+using type_identity_t = typename type_identity<_Tp>::type;
 #endif
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/type_list.h b/libcxx/include/__type_traits/type_list.h
index 5a9e3319a1d46..02905707ee37a 100644
--- a/libcxx/include/__type_traits/type_list.h
+++ b/libcxx/include/__type_traits/type_list.h
@@ -19,24 +19,22 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Hp, class _Tp>
-struct __type_list
-{
-    typedef _Hp _Head;
-    typedef _Tp _Tail;
+struct __type_list {
+  typedef _Hp _Head;
+  typedef _Tp _Tail;
 };
 
-template <class _TypeList, size_t _Size, bool = _Size <= sizeof(typename _TypeList::_Head)> struct __find_first;
+template <class _TypeList, size_t _Size, bool = _Size <= sizeof(typename _TypeList::_Head)>
+struct __find_first;
 
 template <class _Hp, class _Tp, size_t _Size>
-struct __find_first<__type_list<_Hp, _Tp>, _Size, true>
-{
-    typedef _LIBCPP_NODEBUG _Hp type;
+struct __find_first<__type_list<_Hp, _Tp>, _Size, true> {
+  typedef _LIBCPP_NODEBUG _Hp type;
 };
 
 template <class _Hp, class _Tp, size_t _Size>
-struct __find_first<__type_list<_Hp, _Tp>, _Size, false>
-{
-    typedef _LIBCPP_NODEBUG typename __find_first<_Tp, _Size>::type type;
+struct __find_first<__type_list<_Hp, _Tp>, _Size, false> {
+  typedef _LIBCPP_NODEBUG typename __find_first<_Tp, _Size>::type type;
 };
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/underlying_type.h b/libcxx/include/__type_traits/underlying_type.h
index 6f52d678b3eb2..16e7501dee17d 100644
--- a/libcxx/include/__type_traits/underlying_type.h
+++ b/libcxx/include/__type_traits/underlying_type.h
@@ -18,22 +18,23 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
-template <class _Tp, bool = is_enum<_Tp>::value> struct __underlying_type_impl;
+template <class _Tp, bool = is_enum<_Tp>::value>
+struct __underlying_type_impl;
 
 template <class _Tp>
 struct __underlying_type_impl<_Tp, false> {};
 
 template <class _Tp>
-struct __underlying_type_impl<_Tp, true>
-{
-    typedef __underlying_type(_Tp) type;
+struct __underlying_type_impl<_Tp, true> {
+  typedef __underlying_type(_Tp) type;
 };
 
 template <class _Tp>
 struct underlying_type : __underlying_type_impl<_Tp, is_enum<_Tp>::value> {};
 
 #if _LIBCPP_STD_VER >= 14
-template <class _Tp> using underlying_type_t = typename underlying_type<_Tp>::type;
+template <class _Tp>
+using underlying_type_t = typename underlying_type<_Tp>::type;
 #endif
 
 _LIBCPP_END_NAMESPACE_STD
diff --git a/libcxx/include/__type_traits/unwrap_ref.h b/libcxx/include/__type_traits/unwrap_ref.h
index 94cc8ea722edf..5fed08f7ddda1 100644
--- a/libcxx/include/__type_traits/unwrap_ref.h
+++ b/libcxx/include/__type_traits/unwrap_ref.h
@@ -19,26 +19,30 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 template <class _Tp>
-struct __unwrap_reference { typedef _LIBCPP_NODEBUG _Tp type; };
+struct __unwrap_reference {
+  typedef _LIBCPP_NODEBUG _Tp type;
+};
 
 template <class _Tp>
 class reference_wrapper;
 
 template <class _Tp>
-struct __unwrap_reference<reference_wrapper<_Tp> > { typedef _LIBCPP_NODEBUG _Tp& type; };
+struct __unwrap_reference<reference_wrapper<_Tp> > {
+  typedef _LIBCPP_NODEBUG _Tp& type;
+};
 
 template <class _Tp>
 struct decay;
 
 #if _LIBCPP_STD_VER >= 20
 template <class _Tp>
-struct unwrap_reference : __unwrap_reference<_Tp> { };
+struct unwrap_reference : __unwrap_reference<_Tp> {};
 
 template <class _Tp>
 using unwrap_reference_t = typename unwrap_reference<_Tp>::type;
 
 template <class _Tp>
-struct unwrap_ref_decay : unwrap_reference<__decay_t<_Tp> > { };
+struct unwrap_ref_decay : unwrap_reference<__decay_t<_Tp> > {};
 
 template <class _Tp>
 using unwrap_ref_decay_t = typename unwrap_ref_decay<_Tp>::type;
@@ -51,7 +55,8 @@ struct __unwrap_ref_decay
 #else
     : __unwrap_reference<__decay_t<_Tp> >
 #endif
-{ };
+{
+};
 
 _LIBCPP_END_NAMESPACE_STD
 
diff --git a/libcxx/include/__type_traits/void_t.h b/libcxx/include/__type_traits/void_t.h
index 2fe438cf39cd3..985bba02e72f7 100644
--- a/libcxx/include/__type_traits/void_t.h
+++ b/libcxx/include/__type_traits/void_t.h
@@ -18,7 +18,8 @@
 _LIBCPP_BEGIN_NAMESPACE_STD
 
 #if _LIBCPP_STD_VER >= 17
-template <class...> using void_t = void;
+template <class...>
+using void_t = void;
 #endif
 
 template <class...>
diff --git a/libcxx/utils/data/ignore_format.txt b/libcxx/utils/data/ignore_format.txt
index dfe30a2c3f4c1..5e5a23dfca1b6 100644
--- a/libcxx/utils/data/ignore_format.txt
+++ b/libcxx/utils/data/ignore_format.txt
@@ -1,40 +1,3 @@
-libcxx/benchmarks/CartesianBenchmarks.h
-libcxx/benchmarks/ContainerBenchmarks.h
-libcxx/benchmarks/GenerateInput.h
-libcxx/benchmarks/Utilities.h
-libcxx/benchmarks/VariantBenchmarks.h
-libcxx/benchmarks/algorithms/common.h
-libcxx/benchmarks/algorithms/lower_bound.bench.cpp
-libcxx/benchmarks/algorithms/make_heap.bench.cpp
-libcxx/benchmarks/algorithms/make_heap_then_sort_heap.bench.cpp
-libcxx/benchmarks/algorithms.partition_point.bench.cpp
-libcxx/benchmarks/algorithms/pop_heap.bench.cpp
-libcxx/benchmarks/algorithms/push_heap.bench.cpp
-libcxx/benchmarks/algorithms/ranges_make_heap.bench.cpp
-libcxx/benchmarks/algorithms/ranges_make_heap_then_sort_heap.bench.cpp
-libcxx/benchmarks/algorithms/ranges_pop_heap.bench.cpp
-libcxx/benchmarks/algorithms/ranges_push_heap.bench.cpp
-libcxx/benchmarks/algorithms/ranges_sort.bench.cpp
-libcxx/benchmarks/algorithms/ranges_sort_heap.bench.cpp
-libcxx/benchmarks/algorithms/ranges_stable_sort.bench.cpp
-libcxx/benchmarks/algorithms/sort.bench.cpp
-libcxx/benchmarks/algorithms/sort_heap.bench.cpp
-libcxx/benchmarks/algorithms/stable_sort.bench.cpp
-libcxx/benchmarks/allocation.bench.cpp
-libcxx/benchmarks/deque.bench.cpp
-libcxx/benchmarks/filesystem.bench.cpp
-libcxx/benchmarks/formatter_float.bench.cpp
-libcxx/benchmarks/formatter_int.bench.cpp
-libcxx/benchmarks/format_to.bench.cpp
-libcxx/benchmarks/format_to_n.bench.cpp
-libcxx/benchmarks/function.bench.cpp
-libcxx/benchmarks/map.bench.cpp
-libcxx/benchmarks/ordered_set.bench.cpp
-libcxx/benchmarks/random.bench.cpp
-libcxx/benchmarks/string.bench.cpp
-libcxx/benchmarks/stringstream.bench.cpp
-libcxx/benchmarks/unordered_set_operations.bench.cpp
-libcxx/benchmarks/vector_operations.bench.cpp
 libcxx/include/__algorithm/binary_search.h
 libcxx/include/__algorithm/clamp.h
 libcxx/include/__algorithm/comp_ref_type.h
@@ -94,90 +57,6 @@ libcxx/include/__algorithm/partition_point.h
 libcxx/include/__algorithm/pop_heap.h
 libcxx/include/__algorithm/prev_permutation.h
 libcxx/include/__algorithm/push_heap.h
-libcxx/include/__algorithm/ranges_adjacent_find.h
-libcxx/include/__algorithm/ranges_all_of.h
-libcxx/include/__algorithm/ranges_any_of.h
-libcxx/include/__algorithm/ranges_binary_search.h
-libcxx/include/__algorithm/ranges_clamp.h
-libcxx/include/__algorithm/ranges_copy_backward.h
-libcxx/include/__algorithm/ranges_copy.h
-libcxx/include/__algorithm/ranges_copy_if.h
-libcxx/include/__algorithm/ranges_copy_n.h
-libcxx/include/__algorithm/ranges_count.h
-libcxx/include/__algorithm/ranges_count_if.h
-libcxx/include/__algorithm/ranges_equal.h
-libcxx/include/__algorithm/ranges_equal_range.h
-libcxx/include/__algorithm/ranges_fill.h
-libcxx/include/__algorithm/ranges_fill_n.h
-libcxx/include/__algorithm/ranges_find_end.h
-libcxx/include/__algorithm/ranges_find_first_of.h
-libcxx/include/__algorithm/ranges_find.h
-libcxx/include/__algorithm/ranges_find_if.h
-libcxx/include/__algorithm/ranges_find_if_not.h
-libcxx/include/__algorithm/ranges_for_each.h
-libcxx/include/__algorithm/ranges_for_each_n.h
-libcxx/include/__algorithm/ranges_generate.h
-libcxx/include/__algorithm/ranges_generate_n.h
-libcxx/include/__algorithm/ranges_includes.h
-libcxx/include/__algorithm/ranges_inplace_merge.h
-libcxx/include/__algorithm/ranges_is_heap.h
-libcxx/include/__algorithm/ranges_is_heap_until.h
-libcxx/include/__algorithm/ranges_is_partitioned.h
-libcxx/include/__algorithm/ranges_is_permutation.h
-libcxx/include/__algorithm/ranges_is_sorted.h
-libcxx/include/__algorithm/ranges_is_sorted_until.h
-libcxx/include/__algorithm/ranges_lexicographical_compare.h
-libcxx/include/__algorithm/ranges_lower_bound.h
-libcxx/include/__algorithm/ranges_make_heap.h
-libcxx/include/__algorithm/ranges_max_element.h
-libcxx/include/__algorithm/ranges_max.h
-libcxx/include/__algorithm/ranges_merge.h
-libcxx/include/__algorithm/ranges_min_element.h
-libcxx/include/__algorithm/ranges_min.h
-libcxx/include/__algorithm/ranges_minmax_element.h
-libcxx/include/__algorithm/ranges_minmax.h
-libcxx/include/__algorithm/ranges_mismatch.h
-libcxx/include/__algorithm/ranges_move_backward.h
-libcxx/include/__algorithm/ranges_move.h
-libcxx/include/__algorithm/ranges_none_of.h
-libcxx/include/__algorithm/ranges_nth_element.h
-libcxx/include/__algorithm/ranges_partial_sort_copy.h
-libcxx/include/__algorithm/ranges_partial_sort.h
-libcxx/include/__algorithm/ranges_partition_copy.h
-libcxx/include/__algorithm/ranges_partition.h
-libcxx/include/__algorithm/ranges_partition_point.h
-libcxx/include/__algorithm/ranges_pop_heap.h
-libcxx/include/__algorithm/ranges_prev_permutation.h
-libcxx/include/__algorithm/ranges_push_heap.h
-libcxx/include/__algorithm/ranges_remove_copy.h
-libcxx/include/__algorithm/ranges_remove_copy_if.h
-libcxx/include/__algorithm/ranges_remove.h
-libcxx/include/__algorithm/ranges_remove_if.h
-libcxx/include/__algorithm/ranges_replace_copy.h
-libcxx/include/__algorithm/ranges_replace_copy_if.h
-libcxx/include/__algorithm/ranges_replace.h
-libcxx/include/__algorithm/ranges_replace_if.h
-libcxx/include/__algorithm/ranges_reverse_copy.h
-libcxx/include/__algorithm/ranges_reverse.h
-libcxx/include/__algorithm/ranges_rotate_copy.h
-libcxx/include/__algorithm/ranges_rotate.h
-libcxx/include/__algorithm/ranges_sample.h
-libcxx/include/__algorithm/ranges_search.h
-libcxx/include/__algorithm/ranges_search_n.h
-libcxx/include/__algorithm/ranges_set_difference.h
-libcxx/include/__algorithm/ranges_set_intersection.h
-libcxx/include/__algorithm/ranges_set_symmetric_difference.h
-libcxx/include/__algorithm/ranges_set_union.h
-libcxx/include/__algorithm/ranges_shuffle.h
-libcxx/include/__algorithm/ranges_sort.h
-libcxx/include/__algorithm/ranges_sort_heap.h
-libcxx/include/__algorithm/ranges_stable_partition.h
-libcxx/include/__algorithm/ranges_stable_sort.h
-libcxx/include/__algorithm/ranges_swap_ranges.h
-libcxx/include/__algorithm/ranges_transform.h
-libcxx/include/__algorithm/ranges_unique_copy.h
-libcxx/include/__algorithm/ranges_unique.h
-libcxx/include/__algorithm/ranges_upper_bound.h
 libcxx/include/__algorithm/remove_copy.h
 libcxx/include/__algorithm/remove_copy_if.h
 libcxx/include/__algorithm/remove.h
@@ -269,28 +148,6 @@ libcxx/include/__compare/strong_order.h
 libcxx/include/__compare/three_way_comparable.h
 libcxx/include/__compare/weak_order.h
 libcxx/include/complex
-libcxx/include/__concepts/arithmetic.h
-libcxx/include/__concepts/assignable.h
-libcxx/include/__concepts/boolean_testable.h
-libcxx/include/__concepts/class_or_enum.h
-libcxx/include/__concepts/common_reference_with.h
-libcxx/include/__concepts/common_with.h
-libcxx/include/__concepts/constructible.h
-libcxx/include/__concepts/convertible_to.h
-libcxx/include/__concepts/copyable.h
-libcxx/include/__concepts/derived_from.h
-libcxx/include/__concepts/destructible.h
-libcxx/include/__concepts/different_from.h
-libcxx/include/__concepts/equality_comparable.h
-libcxx/include/__concepts/invocable.h
-libcxx/include/__concepts/movable.h
-libcxx/include/__concepts/predicate.h
-libcxx/include/__concepts/regular.h
-libcxx/include/__concepts/relation.h
-libcxx/include/__concepts/same_as.h
-libcxx/include/__concepts/semiregular.h
-libcxx/include/__concepts/swappable.h
-libcxx/include/__concepts/totally_ordered.h
 libcxx/include/condition_variable
 libcxx/include/__config
 libcxx/include/__coroutine/coroutine_handle.h
@@ -638,125 +495,6 @@ libcxx/include/__tuple/tuple_size.h
 libcxx/include/__tuple/tuple_types.h
 libcxx/include/typeindex
 libcxx/include/typeinfo
-libcxx/include/__type_traits/add_const.h
-libcxx/include/__type_traits/add_cv.h
-libcxx/include/__type_traits/add_lvalue_reference.h
-libcxx/include/__type_traits/add_pointer.h
-libcxx/include/__type_traits/add_volatile.h
-libcxx/include/__type_traits/aligned_storage.h
-libcxx/include/__type_traits/aligned_union.h
-libcxx/include/__type_traits/alignment_of.h
-libcxx/include/__type_traits/can_extract_key.h
-libcxx/include/__type_traits/common_reference.h
-libcxx/include/__type_traits/common_type.h
-libcxx/include/__type_traits/copy_cv.h
-libcxx/include/__type_traits/copy_cvref.h
-libcxx/include/__type_traits/decay.h
-libcxx/include/__type_traits/enable_if.h
-libcxx/include/__type_traits/extent.h
-libcxx/include/__type_traits/has_unique_object_representation.h
-libcxx/include/__type_traits/has_virtual_destructor.h
-libcxx/include/__type_traits/integral_constant.h
-libcxx/include/__type_traits/invoke.h
-libcxx/include/__type_traits/is_abstract.h
-libcxx/include/__type_traits/is_aggregate.h
-libcxx/include/__type_traits/is_allocator.h
-libcxx/include/__type_traits/is_always_bitcastable.h
-libcxx/include/__type_traits/is_arithmetic.h
-libcxx/include/__type_traits/is_array.h
-libcxx/include/__type_traits/is_assignable.h
-libcxx/include/__type_traits/is_base_of.h
-libcxx/include/__type_traits/is_bounded_array.h
-libcxx/include/__type_traits/is_callable.h
-libcxx/include/__type_traits/is_class.h
-libcxx/include/__type_traits/is_compound.h
-libcxx/include/__type_traits/is_constant_evaluated.h
-libcxx/include/__type_traits/is_const.h
-libcxx/include/__type_traits/is_constructible.h
-libcxx/include/__type_traits/is_convertible.h
-libcxx/include/__type_traits/is_copy_assignable.h
-libcxx/include/__type_traits/is_copy_constructible.h
-libcxx/include/__type_traits/is_core_convertible.h
-libcxx/include/__type_traits/is_default_constructible.h
-libcxx/include/__type_traits/is_destructible.h
-libcxx/include/__type_traits/is_empty.h
-libcxx/include/__type_traits/is_enum.h
-libcxx/include/__type_traits/is_final.h
-libcxx/include/__type_traits/is_floating_point.h
-libcxx/include/__type_traits/is_fundamental.h
-libcxx/include/__type_traits/is_implicitly_default_constructible.h
-libcxx/include/__type_traits/is_integral.h
-libcxx/include/__type_traits/is_literal_type.h
-libcxx/include/__type_traits/is_member_function_pointer.h
-libcxx/include/__type_traits/is_member_object_pointer.h
-libcxx/include/__type_traits/is_member_pointer.h
-libcxx/include/__type_traits/is_move_assignable.h
-libcxx/include/__type_traits/is_move_constructible.h
-libcxx/include/__type_traits/is_nothrow_assignable.h
-libcxx/include/__type_traits/is_nothrow_constructible.h
-libcxx/include/__type_traits/is_nothrow_convertible.h
-libcxx/include/__type_traits/is_nothrow_copy_assignable.h
-libcxx/include/__type_traits/is_nothrow_copy_constructible.h
-libcxx/include/__type_traits/is_nothrow_default_constructible.h
-libcxx/include/__type_traits/is_nothrow_destructible.h
-libcxx/include/__type_traits/is_nothrow_move_assignable.h
-libcxx/include/__type_traits/is_nothrow_move_constructible.h
-libcxx/include/__type_traits/is_null_pointer.h
-libcxx/include/__type_traits/is_object.h
-libcxx/include/__type_traits/is_pod.h
-libcxx/include/__type_traits/is_pointer.h
-libcxx/include/__type_traits/is_polymorphic.h
-libcxx/include/__type_traits/is_primary_template.h
-libcxx/include/__type_traits/is_reference.h
-libcxx/include/__type_traits/is_reference_wrapper.h
-libcxx/include/__type_traits/is_same.h
-libcxx/include/__type_traits/is_scalar.h
-libcxx/include/__type_traits/is_scoped_enum.h
-libcxx/include/__type_traits/is_signed.h
-libcxx/include/__type_traits/is_signed_integer.h
-libcxx/include/__type_traits/is_standard_layout.h
-libcxx/include/__type_traits/is_swappable.h
-libcxx/include/__type_traits/is_trivial.h
-libcxx/include/__type_traits/is_trivially_assignable.h
-libcxx/include/__type_traits/is_trivially_constructible.h
-libcxx/include/__type_traits/is_trivially_copyable.h
-libcxx/include/__type_traits/is_trivially_copy_assignable.h
-libcxx/include/__type_traits/is_trivially_copy_constructible.h
-libcxx/include/__type_traits/is_trivially_default_constructible.h
-libcxx/include/__type_traits/is_trivially_destructible.h
-libcxx/include/__type_traits/is_unbounded_array.h
-libcxx/include/__type_traits/is_union.h
-libcxx/include/__type_traits/is_unsigned.h
-libcxx/include/__type_traits/is_unsigned_integer.h
-libcxx/include/__type_traits/is_valid_expansion.h
-libcxx/include/__type_traits/is_void.h
-libcxx/include/__type_traits/is_volatile.h
-libcxx/include/__type_traits/lazy.h
-libcxx/include/__type_traits/make_32_64_or_128_bit.h
-libcxx/include/__type_traits/make_const_lvalue_ref.h
-libcxx/include/__type_traits/make_signed.h
-libcxx/include/__type_traits/make_unsigned.h
-libcxx/include/__type_traits/maybe_const.h
-libcxx/include/__type_traits/nat.h
-libcxx/include/__type_traits/negation.h
-libcxx/include/__type_traits/noexcept_move_assign_container.h
-libcxx/include/__type_traits/promote.h
-libcxx/include/__type_traits/rank.h
-libcxx/include/__type_traits/remove_all_extents.h
-libcxx/include/__type_traits/remove_const.h
-libcxx/include/__type_traits/remove_cv.h
-libcxx/include/__type_traits/remove_cvref.h
-libcxx/include/__type_traits/remove_extent.h
-libcxx/include/__type_traits/remove_pointer.h
-libcxx/include/__type_traits/remove_reference.h
-libcxx/include/__type_traits/remove_volatile.h
-libcxx/include/__type_traits/result_of.h
-libcxx/include/__type_traits/strip_signature.h
-libcxx/include/__type_traits/type_identity.h
-libcxx/include/__type_traits/type_list.h
-libcxx/include/__type_traits/underlying_type.h
-libcxx/include/__type_traits/unwrap_ref.h
-libcxx/include/__type_traits/void_t.h
 libcxx/include/uchar.h
 libcxx/include/unordered_map
 libcxx/include/unordered_set

From 6c93b0d6a6fa885ab0c8e7edce1af8a99008321c Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Mon, 19 Jun 2023 11:21:15 -0400
Subject: [PATCH 050/130] [NFC] Add libc++ formatting commit to the git-blame
 ignore file

---
 .git-blame-ignore-revs | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
index 72923419e0b2f..b0c91c6973126 100644
--- a/.git-blame-ignore-revs
+++ b/.git-blame-ignore-revs
@@ -55,3 +55,6 @@ dd3c26a045c081620375a878159f536758baba6e
 f98ee40f4b5d7474fc67e82824bf6abbaedb7b1c
 2238dcc39358353cac21df75c3c3286ab20b8f53
 f9008e6366c2496b1ca1785b891d5578174ad63e
+
+# [libc++][NFC] Apply clang-format on large parts of the code base
+5aa03b648b827128d439f705cd7d57d59673741d

From 734881a6d546c6065a4aa43c3d876337b8b2a263 Mon Sep 17 00:00:00 2001
From: Krzysztof Parzyszek <kparzysz@quicinc.com>
Date: Mon, 19 Jun 2023 08:18:05 -0700
Subject: [PATCH 051/130] [Hexagon] Fix range checks for immediate operands

The output assembly (textual) contains the instruction
  r29 = add(r29,#4294967136)
The value 4294967136 is -160 when interpreted as a signed 32-bit
integer, so it fits in the range of the immediate operand without
a constant extender. The range check in HexagonInstrInfo was putting
the operand value into an int variable, reporting no need for an
extender. This resulted in a packet with 4 instructions, including
the "add". The corresponding check in HexagonMCInstrInfo was using
an int64_t variable, causing the range check to fail, and an extender
to be emitted when lowering to MCInst, resulting in a packet with
too many instructions.
---
 llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp  | 16 ++++++++----
 .../MCTargetDesc/HexagonMCInstrInfo.cpp       | 14 +++++++---
 llvm/test/CodeGen/Hexagon/imm-range-check.ll  | 26 +++++++++++++++++++
 3 files changed, 48 insertions(+), 8 deletions(-)
 create mode 100644 llvm/test/CodeGen/Hexagon/imm-range-check.ll

diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
index 55ddad2378ac9..15a09195eaca2 100644
--- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
@@ -2177,11 +2177,17 @@ bool HexagonInstrInfo::isConstExtended(const MachineInstr &MI) const {
   // have 'isExtended' flag set.
   assert(MO.isImm() && "Extendable operand must be Immediate type");
 
-  int MinValue = getMinValue(MI);
-  int MaxValue = getMaxValue(MI);
-  int ImmValue = MO.getImm();
-
-  return (ImmValue < MinValue || ImmValue > MaxValue);
+  int64_t Value = MO.getImm();
+  if ((F >> HexagonII::ExtentSignedPos) & HexagonII::ExtentSignedMask) {
+    int32_t SValue = Value;
+    int32_t MinValue = getMinValue(MI);
+    int32_t MaxValue = getMaxValue(MI);
+    return SValue < MinValue || SValue > MaxValue;
+  }
+  uint32_t UValue = Value;
+  uint32_t MinValue = getMinValue(MI);
+  uint32_t MaxValue = getMaxValue(MI);
+  return UValue < MinValue || UValue > MaxValue;
 }
 
 bool HexagonInstrInfo::isDeallocRet(const MachineInstr &MI) const {
diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp
index 0fd002ac56909..9cf004cf4c9a5 100644
--- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp
+++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp
@@ -568,12 +568,20 @@ bool HexagonMCInstrInfo::isConstExtended(MCInstrInfo const &MCII,
   if (isa<HexagonMCExpr>(MO.getExpr()) &&
       HexagonMCInstrInfo::mustNotExtend(*MO.getExpr()))
     return false;
+
   int64_t Value;
   if (!MO.getExpr()->evaluateAsAbsolute(Value))
     return true;
-  int MinValue = HexagonMCInstrInfo::getMinValue(MCII, MCI);
-  int MaxValue = HexagonMCInstrInfo::getMaxValue(MCII, MCI);
-  return (MinValue > Value || Value > MaxValue);
+  if (HexagonMCInstrInfo::isExtentSigned(MCII, MCI)) {
+    int32_t SValue = Value;
+    int32_t MinValue = HexagonMCInstrInfo::getMinValue(MCII, MCI);
+    int32_t MaxValue = HexagonMCInstrInfo::getMaxValue(MCII, MCI);
+    return SValue < MinValue || SValue > MaxValue;
+  }
+  uint32_t UValue = Value;
+  uint32_t MinValue = HexagonMCInstrInfo::getMinValue(MCII, MCI);
+  uint32_t MaxValue = HexagonMCInstrInfo::getMaxValue(MCII, MCI);
+  return UValue < MinValue || UValue > MaxValue;
 }
 
 bool HexagonMCInstrInfo::isCanon(MCInstrInfo const &MCII, MCInst const &MCI) {
diff --git a/llvm/test/CodeGen/Hexagon/imm-range-check.ll b/llvm/test/CodeGen/Hexagon/imm-range-check.ll
new file mode 100644
index 0000000000000..290a5b69d3334
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/imm-range-check.ll
@@ -0,0 +1,26 @@
+; RUN: llc -march=hexagon -filetype=obj < %s | llvm-objdump -d - | FileCheck %s
+
+; The output assembly (textual) contains the instruction
+;   r29 = add(r29,#4294967136)
+; The value 4294967136 is -160 when interpreted as a signed 32-bit
+; integer, so it fits in the range of the immediate operand without
+; a constant extender. The range check in HexagonInstrInfo was putting
+; the operand value into an int variable, reporting no need for an
+; extender. This resulted in a packet with 4 instructions, including
+; the "add". The corresponding check in HexagonMCInstrInfo was using
+; an int64_t variable, causing an extender to be emitted when lowering
+; to MCInst, and resulting in a packet with 5 instructions.
+
+; Check that this doesn't crash.
+; CHECK: r29 = add(r29,#-0xa0)
+
+target triple = "hexagon-unknown-linux-gnu"
+
+define float @f0() {
+b0:
+  %v0 = alloca i8, i32 0, align 1
+  %v1 = alloca float, i32 -42, align 4
+  %v2 = load float, ptr %v1, align 4
+  store i8 0, ptr %v0, align 1
+  ret float %v2
+}

From cb9b01f910ca0fa1a4227e6b2e16f48705929947 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Wed, 24 May 2023 12:47:57 +0100
Subject: [PATCH 052/130] ValueTracking: Handle compare to nan and -inf
 constants in fcmpToClassTest

This will help enable a cleanup of simplifyFCmpInst
---
 llvm/lib/Analysis/ValueTracking.cpp           | 31 +++++-
 llvm/unittests/Analysis/ValueTrackingTest.cpp | 99 +++++++++++++++++++
 2 files changed, 128 insertions(+), 2 deletions(-)

diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp
index e4e1a2b4707dc..88ed92e031892 100644
--- a/llvm/lib/Analysis/ValueTracking.cpp
+++ b/llvm/lib/Analysis/ValueTracking.cpp
@@ -4012,6 +4012,14 @@ std::pair<Value *, FPClassTest> llvm::fcmpToClassTest(FCmpInst::Predicate Pred,
   if (!match(RHS, m_APFloat(ConstRHS)))
     return {nullptr, fcNone};
 
+  // fcmp ord x, zero|normal|subnormal|inf -> ~fcNan
+  if (Pred == FCmpInst::FCMP_ORD && !ConstRHS->isNaN())
+    return {LHS, ~fcNan};
+
+  // fcmp uno x, zero|normal|subnormal|inf -> fcNan
+  if (Pred == FCmpInst::FCMP_UNO && !ConstRHS->isNaN())
+    return {LHS, fcNan};
+
   if (ConstRHS->isZero()) {
     // Compares with fcNone are only exactly equal to fcZero if input denormals
     // are not flushed.
@@ -4118,8 +4126,14 @@ std::pair<Value *, FPClassTest> llvm::fcmpToClassTest(FCmpInst::Predicate Pred,
     }
     case FCmpInst::FCMP_OLT:
     case FCmpInst::FCMP_UGE: {
-      if (ConstRHS->isNegative()) // TODO
-        return {nullptr, fcNone};
+      if (ConstRHS->isNegative()) {
+        // No value is ordered and less than negative infinity.
+        // All values are unordered with or at least negative infinity.
+        // fcmp olt x, -inf -> false
+        // fcmp uge x, -inf -> true
+        Mask = fcNone;
+        break;
+      }
 
       // fcmp olt fabs(x), +inf -> fcFinite
       // fcmp uge fabs(x), +inf -> ~fcFinite
@@ -4144,6 +4158,15 @@ std::pair<Value *, FPClassTest> llvm::fcmpToClassTest(FCmpInst::Predicate Pred,
         Mask |= fcNegInf;
       break;
     }
+    case FCmpInst::FCMP_OGT:
+    case FCmpInst::FCMP_ULE: {
+      if (ConstRHS->isNegative())
+        return {nullptr, fcNone};
+
+      // No value is ordered and greater than infinity.
+      Mask = fcNone;
+      break;
+    }
     default:
       return {nullptr, fcNone};
     }
@@ -4176,6 +4199,10 @@ std::pair<Value *, FPClassTest> llvm::fcmpToClassTest(FCmpInst::Predicate Pred,
     default:
       return {nullptr, fcNone};
     }
+  } else if (ConstRHS->isNaN()) {
+    // fcmp o__ x, nan -> false
+    // fcmp u__ x, nan -> true
+    Mask = fcNone;
   } else
     return {nullptr, fcNone};
 
diff --git a/llvm/unittests/Analysis/ValueTrackingTest.cpp b/llvm/unittests/Analysis/ValueTrackingTest.cpp
index e464407197d63..f8910ea21e6ec 100644
--- a/llvm/unittests/Analysis/ValueTrackingTest.cpp
+++ b/llvm/unittests/Analysis/ValueTrackingTest.cpp
@@ -1791,6 +1791,105 @@ TEST_F(ComputeKnownFPClassTest, CannotBeOrderedLessThanZero) {
           .cannotBeOrderedLessThanZero());
 }
 
+TEST_F(ComputeKnownFPClassTest, FCmpToClassTest_OrdNan) {
+  parseAssembly("define i1 @test(double %arg) {\n"
+                "  %A = fcmp ord double %arg, 0x7FF8000000000000"
+                "  %A2 = fcmp uno double %arg, 0x7FF8000000000000"
+                "  %A3 = fcmp oeq double %arg, 0x7FF8000000000000"
+                "  %A4 = fcmp ueq double %arg, 0x7FF8000000000000"
+                "  ret i1 %A\n"
+                "}\n");
+
+  auto [OrdVal, OrdClass] = fcmpToClassTest(
+      CmpInst::FCMP_ORD, *A->getFunction(), A->getOperand(0), A->getOperand(1));
+  EXPECT_EQ(A->getOperand(0), OrdVal);
+  EXPECT_EQ(fcNone, OrdClass);
+
+  auto [UnordVal, UnordClass] =
+      fcmpToClassTest(CmpInst::FCMP_UNO, *A2->getFunction(), A2->getOperand(0),
+                      A2->getOperand(1));
+  EXPECT_EQ(A2->getOperand(0), UnordVal);
+  EXPECT_EQ(fcAllFlags, UnordClass);
+
+  auto [OeqVal, OeqClass] =
+      fcmpToClassTest(CmpInst::FCMP_OEQ, *A3->getFunction(), A3->getOperand(0),
+                      A3->getOperand(1));
+  EXPECT_EQ(A3->getOperand(0), OeqVal);
+  EXPECT_EQ(fcNone, OeqClass);
+
+  auto [UeqVal, UeqClass] =
+      fcmpToClassTest(CmpInst::FCMP_UEQ, *A3->getFunction(), A3->getOperand(0),
+                      A3->getOperand(1));
+  EXPECT_EQ(A3->getOperand(0), UeqVal);
+  EXPECT_EQ(fcAllFlags, UeqClass);
+}
+
+TEST_F(ComputeKnownFPClassTest, FCmpToClassTest_NInf) {
+  parseAssembly("define i1 @test(double %arg) {\n"
+                "  %A = fcmp olt double %arg, 0xFFF0000000000000"
+                "  %A2 = fcmp uge double %arg, 0xFFF0000000000000"
+                "  %A3 = fcmp ogt double %arg, 0xFFF0000000000000"
+                "  %A4 = fcmp ule double %arg, 0xFFF0000000000000"
+                "  ret i1 %A\n"
+                "}\n");
+
+  auto [OltVal, OltClass] = fcmpToClassTest(
+      CmpInst::FCMP_OLT, *A->getFunction(), A->getOperand(0), A->getOperand(1));
+  EXPECT_EQ(A->getOperand(0), OltVal);
+  EXPECT_EQ(fcNone, OltClass);
+
+  auto [UgeVal, UgeClass] =
+      fcmpToClassTest(CmpInst::FCMP_UGE, *A2->getFunction(), A2->getOperand(0),
+                      A2->getOperand(1));
+  EXPECT_EQ(A2->getOperand(0), UgeVal);
+  EXPECT_EQ(fcAllFlags, UgeClass);
+
+  auto [OgtVal, OgtClass] =
+      fcmpToClassTest(CmpInst::FCMP_OGT, *A3->getFunction(), A3->getOperand(0),
+                      A3->getOperand(1));
+  EXPECT_EQ(nullptr, OgtVal);
+  EXPECT_EQ(fcNone, OgtClass);
+
+  auto [UleVal, UleClass] =
+      fcmpToClassTest(CmpInst::FCMP_ULE, *A4->getFunction(), A4->getOperand(0),
+                      A4->getOperand(1));
+  EXPECT_EQ(nullptr, UleVal);
+  EXPECT_EQ(fcNone, UleClass);
+}
+
+TEST_F(ComputeKnownFPClassTest, FCmpToClassTest_PInf) {
+  parseAssembly("define i1 @test(double %arg) {\n"
+                "  %A = fcmp ogt double %arg, 0x7FF0000000000000"
+                "  %A2 = fcmp ule double %arg, 0x7FF0000000000000"
+                "  %A3 = fcmp ole double %arg, 0x7FF0000000000000"
+                "  %A4 = fcmp ugt double %arg, 0x7FF0000000000000"
+                "  ret i1 %A\n"
+                "}\n");
+
+  auto [OgtVal, OgtClass] = fcmpToClassTest(
+      CmpInst::FCMP_OGT, *A->getFunction(), A->getOperand(0), A->getOperand(1));
+  EXPECT_EQ(A->getOperand(0), OgtVal);
+  EXPECT_EQ(fcNone, OgtClass);
+
+  auto [UleVal, UleClass] =
+      fcmpToClassTest(CmpInst::FCMP_ULE, *A2->getFunction(), A2->getOperand(0),
+                      A2->getOperand(1));
+  EXPECT_EQ(A2->getOperand(0), UleVal);
+  EXPECT_EQ(fcAllFlags, UleClass);
+
+  auto [OleVal, OleClass] =
+      fcmpToClassTest(CmpInst::FCMP_OLE, *A3->getFunction(), A3->getOperand(0),
+                      A3->getOperand(1));
+  EXPECT_EQ(nullptr, OleVal);
+  EXPECT_EQ(fcNone, OleClass);
+
+  auto [UgtVal, UgtClass] =
+      fcmpToClassTest(CmpInst::FCMP_UGT, *A4->getFunction(), A4->getOperand(0),
+                      A4->getOperand(1));
+  EXPECT_EQ(nullptr, UgtVal);
+  EXPECT_EQ(fcNone, UgtClass);
+}
+
 TEST_F(ValueTrackingTest, isNonZeroRecurrence) {
   parseAssembly(R"(
     define i1 @test(i8 %n, i8 %r) {

From 2a488b4443a9c0aa0f368aed901676508ced202f Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Thu, 5 Jan 2023 14:50:11 -0500
Subject: [PATCH 053/130] clang: Add __builtin_elementwise_round

---
 clang/docs/LanguageExtensions.rst             |  4 +++
 clang/docs/ReleaseNotes.rst                   |  4 +++
 clang/include/clang/Basic/Builtins.def        |  1 +
 clang/lib/CodeGen/CGBuiltin.cpp               |  3 +++
 clang/lib/Sema/SemaChecking.cpp               |  1 +
 .../test/CodeGen/builtins-elementwise-math.c  | 16 ++++++++++++
 clang/test/Sema/builtins-elementwise-math.c   | 26 +++++++++++++++++++
 7 files changed, 55 insertions(+)

diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst
index 9944cb28f0487..ab1fe885a3a75 100644
--- a/clang/docs/LanguageExtensions.rst
+++ b/clang/docs/LanguageExtensions.rst
@@ -645,6 +645,10 @@ Unless specified otherwise operation(±0) = ±0 and operation(±infinity) = ±in
                                              rounding halfway cases to even (that is, to the nearest value
                                              that is an even integer), regardless of the current rounding
                                              direction.
+ T __builtin_elementwise_round(T x)          round x to the nearest  integer value in floating point format,      floating point types
+                                             rounding halfway cases away from zero, regardless of the
+                                             current rounding direction. May raise floating-point
+                                             exceptions.
  T __builtin_elementwise_trunc(T x)          return the integral value nearest to but no larger in            floating point types
                                              magnitude than x
  T __builtin_elementwise_canonicalize(T x)   return the platform specific canonical encoding                  floating point types
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 6cee1c83706aa..7c6a350c7eab2 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -228,6 +228,10 @@ Non-comprehensive list of changes in this release
   variable as a way to disable color diagnostics.
 - Clang now supports ``__builtin_isfpclass``, which checks if the specified
   floating-point value falls into any of the specified data classes.
+- Added ``__builtin_elementwise_round`` for  builtin for floating
+  point types. This allows access to ``llvm.round`` for
+  arbitrary floating-point and vector of floating-point types.
+
 
 New Compiler Flags
 ------------------
diff --git a/clang/include/clang/Basic/Builtins.def b/clang/include/clang/Basic/Builtins.def
index 59b61b89bd245..aa0076aab951c 100644
--- a/clang/include/clang/Basic/Builtins.def
+++ b/clang/include/clang/Basic/Builtins.def
@@ -687,6 +687,7 @@ BUILTIN(__builtin_elementwise_log, "v.", "nct")
 BUILTIN(__builtin_elementwise_log2, "v.", "nct")
 BUILTIN(__builtin_elementwise_log10, "v.", "nct")
 BUILTIN(__builtin_elementwise_roundeven, "v.", "nct")
+BUILTIN(__builtin_elementwise_round, "v.", "nct")
 BUILTIN(__builtin_elementwise_sin, "v.", "nct")
 BUILTIN(__builtin_elementwise_trunc, "v.", "nct")
 BUILTIN(__builtin_elementwise_canonicalize, "v.", "nct")
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 69438960ebf23..7a38ac4ba10cb 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -3194,6 +3194,9 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
   case Builtin::BI__builtin_elementwise_roundeven:
     return RValue::get(emitUnaryBuiltin(*this, E, llvm::Intrinsic::roundeven,
                                         "elt.roundeven"));
+  case Builtin::BI__builtin_elementwise_round:
+    return RValue::get(emitUnaryBuiltin(*this, E, llvm::Intrinsic::round,
+                                        "elt.round"));
   case Builtin::BI__builtin_elementwise_sin:
     return RValue::get(
         emitUnaryBuiltin(*this, E, llvm::Intrinsic::sin, "elt.sin"));
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index d3b37c1fa70a2..72b2f1d1edd82 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -2636,6 +2636,7 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
   case Builtin::BI__builtin_elementwise_log2:
   case Builtin::BI__builtin_elementwise_log10:
   case Builtin::BI__builtin_elementwise_roundeven:
+  case Builtin::BI__builtin_elementwise_round:
   case Builtin::BI__builtin_elementwise_sin:
   case Builtin::BI__builtin_elementwise_trunc:
   case Builtin::BI__builtin_elementwise_canonicalize: {
diff --git a/clang/test/CodeGen/builtins-elementwise-math.c b/clang/test/CodeGen/builtins-elementwise-math.c
index deb518b1ba597..4598faaa84163 100644
--- a/clang/test/CodeGen/builtins-elementwise-math.c
+++ b/clang/test/CodeGen/builtins-elementwise-math.c
@@ -468,6 +468,22 @@ void test_builtin_elementwise_roundeven(float f1, float f2, double d1, double d2
   vf2 = __builtin_elementwise_roundeven(vf1);
 }
 
+void test_builtin_elementwise_round(float f1, float f2, double d1, double d2,
+                                        float4 vf1, float4 vf2) {
+  // CHECK-LABEL: define void @test_builtin_elementwise_round(
+  // CHECK:      [[F1:%.+]] = load float, ptr %f1.addr, align 4
+  // CHECK-NEXT:  call float @llvm.round.f32(float [[F1]])
+  f2 = __builtin_elementwise_round(f1);
+
+  // CHECK:      [[D1:%.+]] = load double, ptr %d1.addr, align 8
+  // CHECK-NEXT: call double @llvm.round.f64(double [[D1]])
+  d2 = __builtin_elementwise_round(d1);
+
+  // CHECK:      [[VF1:%.+]] = load <4 x float>, ptr %vf1.addr, align 16
+  // CHECK-NEXT: call <4 x float> @llvm.round.v4f32(<4 x float> [[VF1]])
+  vf2 = __builtin_elementwise_round(vf1);
+}
+
 void test_builtin_elementwise_sin(float f1, float f2, double d1, double d2,
                                   float4 vf1, float4 vf2) {
   // CHECK-LABEL: define void @test_builtin_elementwise_sin(
diff --git a/clang/test/Sema/builtins-elementwise-math.c b/clang/test/Sema/builtins-elementwise-math.c
index 35b065437e54f..030c03eccd646 100644
--- a/clang/test/Sema/builtins-elementwise-math.c
+++ b/clang/test/Sema/builtins-elementwise-math.c
@@ -459,6 +459,32 @@ void test_builtin_elementwise_roundeven(int i, float f, double d, float4 v, int3
   // expected-error@-1 {{1st argument must be a floating point type (was 'unsigned4' (vector of 4 'unsigned int' values))}}
 }
 
+void test_builtin_elementwise_round(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) {
+
+  struct Foo s = __builtin_elementwise_round(f);
+  // expected-error@-1 {{initializing 'struct Foo' with an expression of incompatible type 'float'}}
+
+  i = __builtin_elementwise_round();
+  // expected-error@-1 {{too few arguments to function call, expected 1, have 0}}
+
+  i = __builtin_elementwise_round(i);
+  // expected-error@-1 {{1st argument must be a floating point type (was 'int')}}
+
+  i = __builtin_elementwise_round(f, f);
+  // expected-error@-1 {{too many arguments to function call, expected 1, have 2}}
+
+  u = __builtin_elementwise_round(u);
+  // expected-error@-1 {{1st argument must be a floating point type (was 'unsigned int')}}
+
+  uv = __builtin_elementwise_round(uv);
+  // expected-error@-1 {{1st argument must be a floating point type (was 'unsigned4' (vector of 4 'unsigned int' values))}}
+
+  // FIXME: Error should not mention integer
+  _Complex float c1, c2;
+  c1 = __builtin_elementwise_round(c1);
+  // expected-error@-1 {{1st argument must be a vector, integer or floating point type (was '_Complex float')}}
+}
+
 void test_builtin_elementwise_sin(int i, float f, double d, float4 v, int3 iv, unsigned u, unsigned4 uv) {
 
   struct Foo s = __builtin_elementwise_sin(f);

From 7c8958118c6f695b01c98db6984390d2ee4fdb1a Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault@amd.com>
Date: Fri, 10 Dec 2021 18:48:16 -0500
Subject: [PATCH 054/130] AMDGPU: Remove amdgpu-waves-per-eu support in old
 attribute pass

AMDGPUAttributor now handles this attribute with value merging, so
delete the old approach which could only apply this to functions which
did not set it, or cloned the function.
---
 .../AMDGPU/AMDGPUPropagateAttributes.cpp      | 34 -------------------
 .../AMDGPU/propagate-attributes-clone.ll      | 12 +++----
 ...te-attributes-function-pointer-argument.ll |  2 +-
 3 files changed, 7 insertions(+), 41 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPropagateAttributes.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPropagateAttributes.cpp
index 5a4ab467731ec..f256fd9493aec 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPropagateAttributes.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPropagateAttributes.cpp
@@ -53,12 +53,6 @@ static constexpr const FeatureBitset TargetFeatures = {
   AMDGPU::FeatureWavefrontSize64
 };
 
-// Attributes to propagate.
-// TODO: Support conservative min/max merging instead of cloning.
-static constexpr const char *AttributeNames[] = {"amdgpu-waves-per-eu"};
-
-static constexpr unsigned NumAttr = std::size(AttributeNames);
-
 class AMDGPUPropagateAttributes {
 
   class FnProperties {
@@ -68,30 +62,20 @@ class AMDGPUPropagateAttributes {
   public:
     explicit FnProperties(const TargetMachine &TM, const Function &F) {
       Features = TM.getSubtargetImpl(F)->getFeatureBits();
-
-      for (unsigned I = 0; I < NumAttr; ++I)
-        if (F.hasFnAttribute(AttributeNames[I]))
-          Attributes[I] = F.getFnAttribute(AttributeNames[I]);
     }
 
     bool operator == (const FnProperties &Other) const {
       if ((Features & TargetFeatures) != (Other.Features & TargetFeatures))
         return false;
-      for (unsigned I = 0; I < NumAttr; ++I)
-        if (Attributes[I] != Other.Attributes[I])
-          return false;
       return true;
     }
 
     FnProperties adjustToCaller(const FnProperties &CallerProps) const {
       FnProperties New((Features & ~TargetFeatures) | CallerProps.Features);
-      for (unsigned I = 0; I < NumAttr; ++I)
-        New.Attributes[I] = CallerProps.Attributes[I];
       return New;
     }
 
     FeatureBitset Features;
-    std::optional<Attribute> Attributes[NumAttr];
   };
 
   class Clone {
@@ -126,10 +110,6 @@ class AMDGPUPropagateAttributes {
   // Set new function's features in place.
   void setFeatures(Function &F, const FeatureBitset &NewFeatures);
 
-  // Set new function's attributes in place.
-  void setAttributes(Function &F,
-                     const ArrayRef<std::optional<Attribute>> NewAttrs);
-
   std::string getFeatureString(const FeatureBitset &Features) const;
 
   // Propagate attributes from Roots.
@@ -274,7 +254,6 @@ bool AMDGPUPropagateAttributes::process() {
             // we rely on a second pass running on Module, which is allowed
             // to clone.
             setFeatures(F, NewProps.Features);
-            setAttributes(F, NewProps.Attributes);
             NewRoots.insert(&F);
             Changed = true;
             break;
@@ -317,7 +296,6 @@ AMDGPUPropagateAttributes::cloneWithProperties(Function &F,
   ValueToValueMapTy dummy;
   Function *NewF = CloneFunction(&F, dummy);
   setFeatures(*NewF, NewProps.Features);
-  setAttributes(*NewF, NewProps.Attributes);
   NewF->setVisibility(GlobalValue::DefaultVisibility);
   NewF->setLinkage(GlobalValue::InternalLinkage);
 
@@ -344,18 +322,6 @@ void AMDGPUPropagateAttributes::setFeatures(Function &F,
   F.addFnAttr("target-features", NewFeatureStr);
 }
 
-void AMDGPUPropagateAttributes::setAttributes(
-    Function &F, const ArrayRef<std::optional<Attribute>> NewAttrs) {
-  LLVM_DEBUG(dbgs() << "Set attributes on " << F.getName() << ":\n");
-  for (unsigned I = 0; I < NumAttr; ++I) {
-    F.removeFnAttr(AttributeNames[I]);
-    if (NewAttrs[I]) {
-      LLVM_DEBUG(dbgs() << '\t' << NewAttrs[I]->getAsString() << '\n');
-      F.addFnAttr(*NewAttrs[I]);
-    }
-  }
-}
-
 std::string
 AMDGPUPropagateAttributes::getFeatureString(const FeatureBitset &Features) const
 {
diff --git a/llvm/test/CodeGen/AMDGPU/propagate-attributes-clone.ll b/llvm/test/CodeGen/AMDGPU/propagate-attributes-clone.ll
index 461a2b90ea410..e5b992449b222 100644
--- a/llvm/test/CodeGen/AMDGPU/propagate-attributes-clone.ll
+++ b/llvm/test/CodeGen/AMDGPU/propagate-attributes-clone.ll
@@ -58,11 +58,11 @@
 ; OPT-INT: define internal fastcc void @foo3() unnamed_addr #4
 ; OPT-EXT: define internal fastcc void @foo2.3() unnamed_addr #4
 ; OPT-INT: define internal fastcc void @foo2() unnamed_addr #4
-; OPT: attributes #0 = { {{.*}} "amdgpu-waves-per-eu"="1,1" "target-features"="+wavefrontsize64" }
+; OPT: attributes #0 = { {{.*}} "target-features"="+wavefrontsize64" }
 ; OPT: attributes #1 = { {{.*}} "target-features"="{{.*}},-wavefrontsize16,-wavefrontsize32,+wavefrontsize64{{.*}}" }
-; OPT: attributes #2 = { {{.*}} "amdgpu-waves-per-eu"="2,4" "target-features"="+wavefrontsize32" }
+; OPT: attributes #2 = { {{.*}} "target-features"="+wavefrontsize32" }
 ; OPT: attributes #3 = { {{.*}} "target-features"="+wavefrontsize64" }
-; OPT: attributes #4 = { {{.*}} "amdgpu-waves-per-eu"="2,4" "target-features"="{{.*}},-wavefrontsize16,+wavefrontsize32,-wavefrontsize64{{.*}}" }
+; OPT: attributes #4 = { {{.*}} "target-features"="{{.*}},-wavefrontsize16,+wavefrontsize32,-wavefrontsize64{{.*}}" }
 
 ; LLC: foo3:
 ; LLC: sample asm
@@ -138,8 +138,8 @@ entry:
   ret void
 }
 
-attributes #0 = { nounwind "target-features"="+wavefrontsize32" "amdgpu-waves-per-eu"="2,4" }
-attributes #1 = { noinline nounwind "target-features"="+wavefrontsize64" "amdgpu-waves-per-eu"="1,1" }
+attributes #0 = { nounwind "target-features"="+wavefrontsize32" }
+attributes #1 = { noinline nounwind "target-features"="+wavefrontsize64" }
 attributes #2 = { nounwind "target-features"="+wavefrontsize64" }
 attributes #3 = { nounwind "target-features"="+wavefrontsize64" }
-attributes #4 = { noinline nounwind "target-features"="+wavefrontsize64" "amdgpu-waves-per-eu"="2,4" }
+attributes #4 = { noinline nounwind "target-features"="+wavefrontsize64" }
diff --git a/llvm/test/CodeGen/AMDGPU/propagate-attributes-function-pointer-argument.ll b/llvm/test/CodeGen/AMDGPU/propagate-attributes-function-pointer-argument.ll
index 1584888c75a12..fa0cdc8d37d14 100644
--- a/llvm/test/CodeGen/AMDGPU/propagate-attributes-function-pointer-argument.ll
+++ b/llvm/test/CodeGen/AMDGPU/propagate-attributes-function-pointer-argument.ll
@@ -37,4 +37,4 @@ define private void @g() #1 {
 }
 
 attributes #0 = { noinline }
-attributes #1 = { noinline "amdgpu-waves-per-eu"="1,10" }
+attributes #1 = { noinline "target-features"="+wavefrontsize32" }

From 1288e72d10b045925540b28132ac748dd7754769 Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Mon, 19 Jun 2023 11:53:53 -0400
Subject: [PATCH 055/130] [libc++][NFC] Sort header list in
 header_information.py

---
 libcxx/utils/libcxx/test/header_information.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/libcxx/utils/libcxx/test/header_information.py b/libcxx/utils/libcxx/test/header_information.py
index 7f15571c28718..e505731dcd71b 100644
--- a/libcxx/utils/libcxx/test/header_information.py
+++ b/libcxx/utils/libcxx/test/header_information.py
@@ -10,7 +10,6 @@
 
 lit_header_restrictions = {
     "barrier": "// UNSUPPORTED: no-threads, c++03, c++11, c++14, c++17",
-    "stop_token": "// UNSUPPORTED: no-threads, c++03, c++11, c++14, c++17",
     "clocale": "// UNSUPPORTED: no-localization",
     "codecvt": "// UNSUPPORTED: no-localization",
     "coroutine": "// UNSUPPORTED: c++03, c++11, c++14, c++17",
@@ -52,6 +51,7 @@
     "shared_mutex": "// UNSUPPORTED: no-threads, c++03, c++11",
     "sstream": "// UNSUPPORTED: no-localization",
     "stdatomic.h": "// UNSUPPORTED: no-threads, c++03, c++11, c++14, c++17, c++20",
+    "stop_token": "// UNSUPPORTED: no-threads, c++03, c++11, c++14, c++17",
     "streambuf": "// UNSUPPORTED: no-localization",
     "strstream": "// UNSUPPORTED: no-localization",
     "thread": "// UNSUPPORTED: no-threads, c++03",

From 7972a6e126bf3af19d9871e81c6ce1b4cb2fdea2 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes@amd.com>
Date: Mon, 8 May 2023 11:43:15 -0700
Subject: [PATCH 056/130] [DAGCombiner][NFC] Factor out ByteProvider

Differential Revision: https://reviews.llvm.org/D143018

Change-Id: I3dc03787a3382c0c3fe6b869f869c2946f450874
---
 llvm/include/llvm/CodeGen/ByteProvider.h      | 90 +++++++++++++++++++
 llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 75 +++++-----------
 2 files changed, 112 insertions(+), 53 deletions(-)
 create mode 100644 llvm/include/llvm/CodeGen/ByteProvider.h

diff --git a/llvm/include/llvm/CodeGen/ByteProvider.h b/llvm/include/llvm/CodeGen/ByteProvider.h
new file mode 100644
index 0000000000000..e0ba40b135336
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/ByteProvider.h
@@ -0,0 +1,90 @@
+//===-- include/llvm/CodeGen/ByteProvider.h - Map bytes ---------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// \file
+// This file implements ByteProvider. The purpose of ByteProvider is to provide
+// a map between a target node's byte (byte position is DestOffset) and the
+// source (and byte position) that provides it (in Src and SrcOffset
+// respectively) See CodeGen/SelectionDAG/DAGCombiner.cpp MatchLoadCombine
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_BYTEPROVIDER_H
+#define LLVM_CODEGEN_BYTEPROVIDER_H
+
+#include <optional>
+#include <type_traits>
+
+namespace llvm {
+
+/// Represents known origin of an individual byte in combine pattern. The
+/// value of the byte is either constant zero, or comes from memory /
+/// some other productive instruction (e.g. arithmetic instructions).
+/// Bit manipulation instructions like shifts are not ByteProviders, rather
+/// are used to extract Bytes.
+template <typename ISelOp> class ByteProvider {
+private:
+  ByteProvider<ISelOp>(std::optional<ISelOp> Src, int64_t DestOffset,
+                       int64_t SrcOffset)
+      : Src(Src), DestOffset(DestOffset), SrcOffset(SrcOffset) {}
+
+  // TODO -- use constraint in c++20
+  // Does this type correspond with an operation in selection DAG
+  template <typename T> class is_op {
+  private:
+    using yes = std::true_type;
+    using no = std::false_type;
+
+    // Only allow classes with member function getOpcode
+    template <typename U>
+    static auto test(int) -> decltype(std::declval<U>().getOpcode(), yes());
+
+    template <typename> static no test(...);
+
+  public:
+    using remove_pointer_t = typename std::remove_pointer<T>::type;
+    static constexpr bool value =
+        std::is_same<decltype(test<remove_pointer_t>(0)), yes>::value;
+  };
+
+public:
+  // For constant zero providers Src is set to nullopt. For actual providers
+  // Src represents the node which originally produced the relevant bits.
+  std::optional<ISelOp> Src = std::nullopt;
+  // DestOffset is the offset of the byte in the dest we are trying to map for.
+  int64_t DestOffset = 0;
+  // SrcOffset is the offset in the ultimate source node that maps to the
+  // DestOffset
+  int64_t SrcOffset = 0;
+
+  ByteProvider() = default;
+
+  static ByteProvider getSrc(std::optional<ISelOp> Val, int64_t ByteOffset,
+                             int64_t VectorOffset) {
+    static_assert(is_op<ISelOp>().value,
+                  "ByteProviders must contain an operation in selection DAG.");
+    return ByteProvider(Val, ByteOffset, VectorOffset);
+  }
+
+  static ByteProvider getConstantZero() {
+    return ByteProvider<ISelOp>(std::nullopt, 0, 0);
+  }
+  bool isConstantZero() const { return !Src; }
+
+  bool hasSrc() const { return Src.has_value(); }
+
+  bool hasSameSrc(const ByteProvider &Other) const { return Other.Src == Src; }
+
+  bool operator==(const ByteProvider &Other) const {
+    return Other.Src == Src && Other.DestOffset == DestOffset &&
+           Other.SrcOffset == SrcOffset;
+  }
+};
+} // end namespace llvm
+
+#endif // LLVM_CODEGEN_BYTEPROVIDER_H
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 454944419293c..56dc284adf709 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -32,6 +32,7 @@
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Analysis/VectorUtils.h"
+#include "llvm/CodeGen/ByteProvider.h"
 #include "llvm/CodeGen/DAGCombine.h"
 #include "llvm/CodeGen/ISDOpcodes.h"
 #include "llvm/CodeGen/MachineFunction.h"
@@ -8402,42 +8403,6 @@ SDValue DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL) {
   return SDValue();
 }
 
-namespace {
-
-/// Represents known origin of an individual byte in load combine pattern. The
-/// value of the byte is either constant zero or comes from memory.
-struct ByteProvider {
-  // For constant zero providers Load is set to nullptr. For memory providers
-  // Load represents the node which loads the byte from memory.
-  // ByteOffset is the offset of the byte in the value produced by the load.
-  LoadSDNode *Load = nullptr;
-  unsigned ByteOffset = 0;
-  unsigned VectorOffset = 0;
-
-  ByteProvider() = default;
-
-  static ByteProvider getMemory(LoadSDNode *Load, unsigned ByteOffset,
-                                unsigned VectorOffset) {
-    return ByteProvider(Load, ByteOffset, VectorOffset);
-  }
-
-  static ByteProvider getConstantZero() { return ByteProvider(nullptr, 0, 0); }
-
-  bool isConstantZero() const { return !Load; }
-  bool isMemory() const { return Load; }
-
-  bool operator==(const ByteProvider &Other) const {
-    return Other.Load == Load && Other.ByteOffset == ByteOffset &&
-           Other.VectorOffset == VectorOffset;
-  }
-
-private:
-  ByteProvider(LoadSDNode *Load, unsigned ByteOffset, unsigned VectorOffset)
-      : Load(Load), ByteOffset(ByteOffset), VectorOffset(VectorOffset) {}
-};
-
-} // end anonymous namespace
-
 /// Recursively traverses the expression calculating the origin of the requested
 /// byte of the given value. Returns std::nullopt if the provider can't be
 /// calculated.
@@ -8479,7 +8444,9 @@ struct ByteProvider {
 ///                 LOAD
 ///
 /// *ExtractVectorElement
-static const std::optional<ByteProvider>
+using SDByteProvider = ByteProvider<SDNode *>;
+
+static const std::optional<SDByteProvider>
 calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth,
                       std::optional<uint64_t> VectorIndex,
                       unsigned StartingIndex = 0) {
@@ -8538,7 +8505,7 @@ calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth,
     // provide, then do not provide anything. Otherwise, subtract the index by
     // the amount we shifted by.
     return Index < ByteShift
-               ? ByteProvider::getConstantZero()
+               ? SDByteProvider::getConstantZero()
                : calculateByteProvider(Op->getOperand(0), Index - ByteShift,
                                        Depth + 1, VectorIndex, Index);
   }
@@ -8553,7 +8520,8 @@ calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth,
 
     if (Index >= NarrowByteWidth)
       return Op.getOpcode() == ISD::ZERO_EXTEND
-                 ? std::optional<ByteProvider>(ByteProvider::getConstantZero())
+                 ? std::optional<SDByteProvider>(
+                       SDByteProvider::getConstantZero())
                  : std::nullopt;
     return calculateByteProvider(NarrowOp, Index, Depth + 1, VectorIndex,
                                  StartingIndex);
@@ -8603,11 +8571,12 @@ calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth,
     // question
     if (Index >= NarrowByteWidth)
       return L->getExtensionType() == ISD::ZEXTLOAD
-                 ? std::optional<ByteProvider>(ByteProvider::getConstantZero())
+                 ? std::optional<SDByteProvider>(
+                       SDByteProvider::getConstantZero())
                  : std::nullopt;
 
     unsigned BPVectorIndex = VectorIndex.value_or(0U);
-    return ByteProvider::getMemory(L, Index, BPVectorIndex);
+    return SDByteProvider::getSrc(L, Index, BPVectorIndex);
   }
   }
 
@@ -8901,23 +8870,24 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
   unsigned ByteWidth = VT.getSizeInBits() / 8;
 
   bool IsBigEndianTarget = DAG.getDataLayout().isBigEndian();
-  auto MemoryByteOffset = [&] (ByteProvider P) {
-    assert(P.isMemory() && "Must be a memory byte provider");
-    unsigned LoadBitWidth = P.Load->getMemoryVT().getScalarSizeInBits();
+  auto MemoryByteOffset = [&](SDByteProvider P) {
+    assert(P.hasSrc() && "Must be a memory byte provider");
+    auto *Load = cast<LoadSDNode>(P.Src.value());
+
+    unsigned LoadBitWidth = Load->getMemoryVT().getScalarSizeInBits();
 
     assert(LoadBitWidth % 8 == 0 &&
            "can only analyze providers for individual bytes not bit");
     unsigned LoadByteWidth = LoadBitWidth / 8;
-    return IsBigEndianTarget
-            ? bigEndianByteAt(LoadByteWidth, P.ByteOffset)
-            : littleEndianByteAt(LoadByteWidth, P.ByteOffset);
+    return IsBigEndianTarget ? bigEndianByteAt(LoadByteWidth, P.DestOffset)
+                             : littleEndianByteAt(LoadByteWidth, P.DestOffset);
   };
 
   std::optional<BaseIndexOffset> Base;
   SDValue Chain;
 
   SmallPtrSet<LoadSDNode *, 8> Loads;
-  std::optional<ByteProvider> FirstByteProvider;
+  std::optional<SDByteProvider> FirstByteProvider;
   int64_t FirstOffset = INT64_MAX;
 
   // Check if all the bytes of the OR we are looking at are loaded from the same
@@ -8938,9 +8908,8 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
         return SDValue();
       continue;
     }
-    assert(P->isMemory() && "provenance should either be memory or zero");
-
-    LoadSDNode *L = P->Load;
+    assert(P->hasSrc() && "provenance should either be memory or zero");
+    auto *L = cast<LoadSDNode>(P->Src.value());
 
     // All loads must share the same chain
     SDValue LChain = L->getChain();
@@ -8964,7 +8933,7 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
       unsigned LoadWidthInBit = L->getMemoryVT().getScalarSizeInBits();
       if (LoadWidthInBit % 8 != 0)
         return SDValue();
-      unsigned ByteOffsetFromVector = P->VectorOffset * LoadWidthInBit / 8;
+      unsigned ByteOffsetFromVector = P->SrcOffset * LoadWidthInBit / 8;
       Ptr.addToOffset(ByteOffsetFromVector);
     }
 
@@ -9021,7 +8990,7 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) {
   // So the combined value can be loaded from the first load address.
   if (MemoryByteOffset(*FirstByteProvider) != 0)
     return SDValue();
-  LoadSDNode *FirstLoad = FirstByteProvider->Load;
+  auto *FirstLoad = cast<LoadSDNode>(FirstByteProvider->Src.value());
 
   // The node we are looking at matches with the pattern, check if we can
   // replace it with a single (possibly zero-extended) load and bswap + shift if

From 632ccc538cf454a00010fa83e0734b562a34faa8 Mon Sep 17 00:00:00 2001
From: Kun Wu <kunww@google.com>
Date: Mon, 19 Jun 2023 15:57:39 +0000
Subject: [PATCH 057/130] [mlir][sparse][gpu] remove tuple as one of the
 spmm_buffer_size output type

Reviewed By: aartbik

Differential Revision: https://reviews.llvm.org/D153188
---
 mlir/include/mlir/Dialect/GPU/IR/GPUOps.td    |  5 ++--
 .../GPUCommon/GPUToLLVMConversion.cpp         | 25 ++++++++++++++-----
 .../ExecutionEngine/CudaRuntimeWrappers.cpp   | 18 ++++++-------
 ...ower-2to4-sparse-to-gpu-runtime-calls.mlir |  2 +-
 .../CUDA/sm80-lt/sparse-matmul-2-4-lib.mlir   | 20 ++++++---------
 5 files changed, 39 insertions(+), 31 deletions(-)

diff --git a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
index d7b985443297a..0e13295b1db07 100644
--- a/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
+++ b/mlir/include/mlir/Dialect/GPU/IR/GPUOps.td
@@ -1963,7 +1963,7 @@ def GPU_SpMVOp : GPU_Op<"spmv", [GPU_AsyncOpInterface]> {
   }];
 }
 
-def GPU_SpMMBufferSizeOp : GPU_Op<"spmm_buffer_size", [GPU_AsyncOpInterface]> {
+def GPU_SpMMBufferSizeOp : GPU_Op<"spmm_buffer_size", [GPU_AsyncOpInterface, AttrSizedResultSegments]> {
   let summary = "Precompute buffersize for SpMM operation";
   let description = [{
     The `gpu.spmm_buffer_size` operation returns the buffer size required
@@ -1994,8 +1994,7 @@ def GPU_SpMMBufferSizeOp : GPU_Op<"spmm_buffer_size", [GPU_AsyncOpInterface]> {
                        GPU_SparseDnTensorHandle:$dnmatB,
                        GPU_SparseDnTensorHandle:$dnmatC,
                        TypeAttr:$computeType);
-  let results = (outs Res<AnyTypeOf<[Index, TupleOf<[Index, Index,
-                                                     Index]>]>>:$bufferSzs,
+  let results = (outs Variadic<Index>:$bufferSzs,
                       Optional<GPU_AsyncToken>:$asyncToken);
 
   let builders = [OpBuilder<(ins
diff --git a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
index 05800aba8690f..580de21f40797 100644
--- a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
+++ b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp
@@ -1746,18 +1746,31 @@ LogicalResult ConvertSpMMBufferSizeOpToGpuRuntimeCallPattern::matchAndRewrite(
         rewriter, loc, getCuSparseLtDataTypeFrom(adaptor.getComputeType()));
     auto three = rewriter.create<LLVM::ConstantOp>(loc, getIndexType(),
                                                    rewriter.getIndexAttr(3));
-    bufferSize = rewriter.create<LLVM::AllocaOp>(loc, llvmInt64PointerType,
-                                                 llvmInt64Type, three);
-    bufferSize =
-        rewriter.create<LLVM::BitcastOp>(loc, llvmPointerType, bufferSize);
-
+    auto bufferSize = rewriter.create<LLVM::AllocaOp>(loc, llvmInt64PointerType,
+                                                      llvmInt64Type, three);
     createCuSparseLtSpMMBufferSizeBuilder
         .create(loc, rewriter,
                 {bufferSize, adaptor.getEnv(), modeA, modeB,
                  adaptor.getSpmatA(), adaptor.getDnmatB(), adaptor.getDnmatC(),
                  computeType, stream})
         .getResult();
-    rewriter.replaceOp(op, {bufferSize, stream});
+
+    auto bufferSizePtr1 = rewriter.create<LLVM::GEPOp>(
+        loc, llvmInt64PointerType, llvmInt64PointerType, bufferSize,
+        ValueRange{rewriter.create<LLVM::ConstantOp>(
+            loc, getIndexType(), rewriter.getIndexAttr(1))});
+    auto bufferSizePtr2 = rewriter.create<LLVM::GEPOp>(
+        loc, llvmInt64PointerType, llvmInt64PointerType, bufferSize,
+        ValueRange{rewriter.create<LLVM::ConstantOp>(
+            loc, getIndexType(), rewriter.getIndexAttr(2))});
+    auto bufferSize0 =
+        rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, bufferSize);
+    auto bufferSize1 =
+        rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, bufferSizePtr1);
+    auto bufferSize2 =
+        rewriter.create<LLVM::LoadOp>(loc, llvmInt64Type, bufferSizePtr2);
+
+    rewriter.replaceOp(op, {bufferSize0, bufferSize1, bufferSize2, stream});
   } else {
     auto computeType = genConstInt32From(
         rewriter, loc, getCuSparseDataTypeFrom(adaptor.getComputeType()));
diff --git a/mlir/lib/ExecutionEngine/CudaRuntimeWrappers.cpp b/mlir/lib/ExecutionEngine/CudaRuntimeWrappers.cpp
index abb4d05760fba..00cfb65e99a87 100644
--- a/mlir/lib/ExecutionEngine/CudaRuntimeWrappers.cpp
+++ b/mlir/lib/ExecutionEngine/CudaRuntimeWrappers.cpp
@@ -556,9 +556,10 @@ mgpuCuSparseLtSpMMBufferSize(void *bs, void *h, int32_t ma, int32_t mb, void *a,
   auto matA = reinterpret_cast<cusparseLtSpMatHandleAndData *>(a);
   auto matB = reinterpret_cast<cusparseLtDnMatHandleAndData *>(b);
   auto matC = reinterpret_cast<cusparseLtDnMatHandleAndData *>(c);
-  auto workspace_size = reinterpret_cast<size_t *>(bs);
-  auto compressed_size = &(reinterpret_cast<size_t *>(bs)[1]);
-  auto compressed_buffer_size = &(reinterpret_cast<size_t *>(bs)[2]);
+  auto workspace_size = reinterpret_cast<int64_t *>(bs);
+  auto compressed_size = &(reinterpret_cast<int64_t *>(bs)[1]);
+  auto compressed_buffer_size = &(reinterpret_cast<int64_t *>(bs)[2]);
+  size_t workspace_size_, compressed_size_, compressed_buffer_size_;
   auto cTp = static_cast<cusparseComputeType>(ctp);
 
   cusparseOperation_t modeA = static_cast<cusparseOperation_t>(ma);
@@ -577,15 +578,14 @@ mgpuCuSparseLtSpMMBufferSize(void *bs, void *h, int32_t ma, int32_t mb, void *a,
       handle, &(matA->plan), &(matA->matmul), &(matA->alg_sel)))
 
   CUSPARSE_REPORT_IF_ERROR(
-      cusparseLtMatmulGetWorkspace(handle, &(matA->plan), workspace_size))
+      cusparseLtMatmulGetWorkspace(handle, &(matA->plan), &workspace_size_))
   CUSPARSE_REPORT_IF_ERROR(cusparseLtSpMMACompressedSize(
-      handle, &(matA->plan), compressed_size, compressed_buffer_size))
-
+      handle, &(matA->plan), &compressed_size_, &compressed_buffer_size_))
   // avoid zero-alloc
-  *workspace_size = (*workspace_size == 0 ? 1 : *workspace_size);
-  *compressed_size = (*compressed_size == 0 ? 1 : *compressed_size);
+  *workspace_size = (workspace_size_ == 0 ? 1 : workspace_size_);
+  *compressed_size = (compressed_size_ == 0 ? 1 : compressed_size_);
   *compressed_buffer_size =
-      (*compressed_buffer_size == 0 ? 1 : *compressed_buffer_size);
+      (compressed_buffer_size_ == 0 ? 1 : compressed_buffer_size_);
 }
 
 extern "C" MLIR_CUDA_WRAPPERS_EXPORT void
diff --git a/mlir/test/Conversion/GPUCommon/lower-2to4-sparse-to-gpu-runtime-calls.mlir b/mlir/test/Conversion/GPUCommon/lower-2to4-sparse-to-gpu-runtime-calls.mlir
index 8fa28cfeae3b0..d46baa7c4ef66 100644
--- a/mlir/test/Conversion/GPUCommon/lower-2to4-sparse-to-gpu-runtime-calls.mlir
+++ b/mlir/test/Conversion/GPUCommon/lower-2to4-sparse-to-gpu-runtime-calls.mlir
@@ -23,7 +23,7 @@ module attributes {gpu.container_module} {
     %env, %token3 = gpu.create_sparse_env async [%token2]
     %spmat, %token4 = gpu.create_2to4_spmat async [%token3] %env, %arg0, %arg0, %mem1:  memref<?xf16>
     %dnmat, %token5 = gpu.create_dn_tensor async [%token4] %env, %mem2, %arg0, %arg0 : index, index into memref<?xf16>
-    %bufferSzs, %token6 = gpu.spmm_buffer_size async [%token5] %env, %spmat, %dnmat, %dnmat : tuple<index,index,index> into f16
+    %bufferSz0, %bufferSz1, %bufferSz2, %token6 = gpu.spmm_buffer_size async [%token5] %env, %spmat, %dnmat, %dnmat : index,index,index into f16
     %token7 = gpu.spmm async [%token6] %env, %spmat, %dnmat, %dnmat, %mem2, %mem2, %mem2 : memref<?xf16>,memref<?xf16>,memref<?xf16> into f16
     %token8 = gpu.destroy_sp_mat async [%token7] %spmat
     %token9 = gpu.destroy_dn_tensor async [%token8] %dnmat
diff --git a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sm80-lt/sparse-matmul-2-4-lib.mlir b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sm80-lt/sparse-matmul-2-4-lib.mlir
index 676454003e472..0ce978c4d7cec 100644
--- a/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sm80-lt/sparse-matmul-2-4-lib.mlir
+++ b/mlir/test/Integration/Dialect/SparseTensor/GPU/CUDA/sm80-lt/sparse-matmul-2-4-lib.mlir
@@ -28,18 +28,14 @@ module {
     %token4 = gpu.memcpy async [%token3] %d_a, %a : memref<16x32xf16>, memref<16x32xf16>
     %token5 = gpu.memcpy async [%token4] %d_b, %b : memref<32x16xf16>, memref<32x16xf16>
     %token6 = gpu.memcpy async [%token5] %d_c, %c : memref<16x16xf16>, memref<16x16xf16>
-    // Allocating larger memory than enough for workspace and storing compressed 
-    // matrices as we haven't implemented the op to unpack tuple %bufferSzs to 
-    // retrieve these three sizes.
-    // TODO: implement the op to unpack tuple %bufferSzs. 
-    %mem1, %token7 = gpu.alloc async [%token6] (%c1048576) : memref<?xf16>
-    %mem2, %token8 = gpu.alloc async [%token7] (%c1048576) : memref<?xf16>
-    %mem3, %token9 = gpu.alloc async [%token8] (%c1048576) : memref<?xf16>
-    %env, %token10 = gpu.create_sparse_env async [%token9]
-    %spmat, %token11 = gpu.create_2to4_spmat async [%token10] %env, %c16, %c32, %d_a: memref<16x32xf16>
-    %dnmat, %token12 = gpu.create_dn_tensor async [%token11] %env, %d_b, %c32, %c16: index, index into memref<32x16xf16>
-    %dnmat2, %token13 = gpu.create_dn_tensor async [%token12] %env, %d_c, %c16, %c16: index, index into memref<16x16xf16>
-    %bufferSzs, %token14 = gpu.spmm_buffer_size async [%token13] %env, %spmat{NON_TRANSPOSE}, %dnmat{NON_TRANSPOSE}, %dnmat2 : tuple<index, index,index> into f16
+    %env, %token7 = gpu.create_sparse_env async [%token6]
+    %spmat, %token8 = gpu.create_2to4_spmat async [%token7] %env, %c16, %c32, %d_a: memref<16x32xf16>
+    %dnmat, %token9 = gpu.create_dn_tensor async [%token8] %env, %d_b, %c32, %c16: index, index into memref<32x16xf16>
+    %dnmat2, %token10 = gpu.create_dn_tensor async [%token9] %env, %d_c, %c16, %c16: index, index into memref<16x16xf16>
+    %bufferSz0, %bufferSz1, %bufferSz2, %token11 = gpu.spmm_buffer_size async [%token10] %env, %spmat{NON_TRANSPOSE}, %dnmat{NON_TRANSPOSE}, %dnmat2 : index, index,index into f16
+    %mem1, %token12 = gpu.alloc async [%token11] (%bufferSz0) : memref<?xf16>
+    %mem2, %token13 = gpu.alloc async [%token12] (%bufferSz1) : memref<?xf16>
+    %mem3, %token14 = gpu.alloc async [%token13] (%bufferSz2) : memref<?xf16>
     %token15 = gpu.spmm async [%token14] %env, %spmat{NON_TRANSPOSE}, %dnmat{NON_TRANSPOSE}, %dnmat2, %mem1, %mem2, %mem3 : memref<?xf16>, memref<?xf16>,memref<?xf16> into f16
     %token16 = gpu.destroy_sp_mat async [%token15] %spmat
     %token17 = gpu.destroy_dn_tensor async [%token16] %dnmat

From eb7491769a511476ffae596a8f58abfd12ed46d4 Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Mon, 19 Jun 2023 15:39:45 +0100
Subject: [PATCH 058/130] [AMDGPU] Reimplement the GFX11 early release VGPRs
 optimization

Implement this optimization in SIInsertWaitcnts, where we already have
information about whether there might be outstanding VMEM store
instructions. This has the following advantages:
- Correctly handles atomics-with-return.
- Correctly handles call instructions.
- Should be faster because it does not require running a separate pass.

Differential Revision: https://reviews.llvm.org/D153279
---
 llvm/lib/Target/AMDGPU/AMDGPU.h               |   3 -
 llvm/lib/Target/AMDGPU/AMDGPUReleaseVGPRs.cpp | 156 -------
 .../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp |   4 -
 llvm/lib/Target/AMDGPU/CMakeLists.txt         |   1 -
 llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp   |  22 +
 .../llvm.amdgcn.global.atomic.csub.ll         |   1 -
 .../llvm.amdgcn.image.atomic.dim.ll           |  80 ++--
 .../CodeGen/AMDGPU/call-argument-types.ll     |   3 -
 llvm/test/CodeGen/AMDGPU/cc-update.ll         |   2 -
 llvm/test/CodeGen/AMDGPU/llc-pipeline.ll      |   2 -
 .../AMDGPU/llvm.amdgcn.s.buffer.load.ll       |   1 +
 llvm/test/CodeGen/AMDGPU/release-vgprs.mir    | 411 ------------------
 .../AMDGPU/waitcnt-preexisting-vscnt.mir      |   5 +
 13 files changed, 54 insertions(+), 637 deletions(-)
 delete mode 100644 llvm/lib/Target/AMDGPU/AMDGPUReleaseVGPRs.cpp
 delete mode 100644 llvm/test/CodeGen/AMDGPU/release-vgprs.mir

diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 07d50df317b77..e4d70abd15f71 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -316,9 +316,6 @@ extern char &SIMemoryLegalizerID;
 void initializeSIModeRegisterPass(PassRegistry&);
 extern char &SIModeRegisterID;
 
-void initializeAMDGPUReleaseVGPRsPass(PassRegistry &);
-extern char &AMDGPUReleaseVGPRsID;
-
 void initializeAMDGPUInsertDelayAluPass(PassRegistry &);
 extern char &AMDGPUInsertDelayAluID;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUReleaseVGPRs.cpp b/llvm/lib/Target/AMDGPU/AMDGPUReleaseVGPRs.cpp
deleted file mode 100644
index b7521540c0205..0000000000000
--- a/llvm/lib/Target/AMDGPU/AMDGPUReleaseVGPRs.cpp
+++ /dev/null
@@ -1,156 +0,0 @@
-//===- AMDGPUReleaseVGPRs.cpp - Automatically release vgprs on GFX11+ -----===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-/// \file
-/// Insert S_SENDMSG instructions to release vgprs on GFX11+.
-//
-//===----------------------------------------------------------------------===//
-
-#include "AMDGPU.h"
-#include "AMDGPUSubtarget.h"
-#include "GCNSubtarget.h"
-#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
-#include "SIDefines.h"
-#include "llvm/ADT/DepthFirstIterator.h"
-#include "llvm/CodeGen/MachineBasicBlock.h"
-#include "llvm/CodeGen/MachineOperand.h"
-#include <optional>
-using namespace llvm;
-
-#define DEBUG_TYPE "release-vgprs"
-
-namespace {
-
-class AMDGPUReleaseVGPRs : public MachineFunctionPass {
-public:
-  static char ID;
-
-  AMDGPUReleaseVGPRs() : MachineFunctionPass(ID) {}
-
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.setPreservesAll();
-    MachineFunctionPass::getAnalysisUsage(AU);
-  }
-
-  // Track if the last instruction referencing a vgpr in a MBB is a VMEM
-  // store. Because this pass is late in the pipeline, it is expected that the
-  // last vgpr use will likely be one of vmem store, ds, exp.
-  // Loads and others vgpr operations would have been
-  // deleted by this point, except for complex control flow involving loops.
-  // This is why we are just testing the type of instructions rather
-  // than the operands.
-  class LastVGPRUseIsVMEMStore {
-    BitVector BlockVMEMStore;
-
-    static std::optional<bool>
-    lastVGPRUseIsStore(const MachineBasicBlock &MBB) {
-      for (auto &MI : reverse(MBB.instrs())) {
-        // If it's a VMEM store, a VGPR will be used, return true.
-        if ((SIInstrInfo::isVMEM(MI) || SIInstrInfo::isFLAT(MI)) &&
-            MI.mayStore())
-          return true;
-
-        // If it's referencing a VGPR but is not a VMEM store, return false.
-        if (SIInstrInfo::isDS(MI) || SIInstrInfo::isEXP(MI) ||
-            SIInstrInfo::isVMEM(MI) || SIInstrInfo::isFLAT(MI) ||
-            SIInstrInfo::isVALU(MI))
-          return false;
-      }
-      // Wait until the values are propagated from the predecessors
-      return std::nullopt;
-    }
-
-  public:
-    LastVGPRUseIsVMEMStore(const MachineFunction &MF)
-        : BlockVMEMStore(MF.getNumBlockIDs()) {
-
-      df_iterator_default_set<const MachineBasicBlock *> Visited;
-      SmallVector<const MachineBasicBlock *> EndWithVMEMStoreBlocks;
-
-      for (const auto &MBB : MF) {
-        auto LastUseIsStore = lastVGPRUseIsStore(MBB);
-        if (!LastUseIsStore.has_value())
-          continue;
-
-        if (*LastUseIsStore) {
-          EndWithVMEMStoreBlocks.push_back(&MBB);
-        } else {
-          Visited.insert(&MBB);
-        }
-      }
-
-      for (const auto *MBB : EndWithVMEMStoreBlocks) {
-        for (const auto *Succ : depth_first_ext(MBB, Visited)) {
-          BlockVMEMStore[Succ->getNumber()] = true;
-        }
-      }
-    }
-
-    // Return true if the last instruction referencing a vgpr in this MBB
-    // is a VMEM store, otherwise return false.
-    bool isLastVGPRUseVMEMStore(const MachineBasicBlock &MBB) const {
-      return BlockVMEMStore[MBB.getNumber()];
-    }
-  };
-
-  static bool
-  runOnMachineBasicBlock(MachineBasicBlock &MBB, const SIInstrInfo *SII,
-                         const LastVGPRUseIsVMEMStore &BlockVMEMStore) {
-
-    bool Changed = false;
-
-    for (auto &MI : MBB.terminators()) {
-      // Look for S_ENDPGM instructions
-      if (MI.getOpcode() == AMDGPU::S_ENDPGM ||
-          MI.getOpcode() == AMDGPU::S_ENDPGM_SAVED) {
-        // If the last instruction using a VGPR in the block is a VMEM store,
-        // release VGPRs. The VGPRs release will be placed just before ending
-        // the program
-        if (BlockVMEMStore.isLastVGPRUseVMEMStore(MBB)) {
-          BuildMI(MBB, MI, DebugLoc(), SII->get(AMDGPU::S_SENDMSG))
-              .addImm(AMDGPU::SendMsg::ID_DEALLOC_VGPRS_GFX11Plus);
-          Changed = true;
-        }
-      }
-    }
-
-    return Changed;
-  }
-
-  bool runOnMachineFunction(MachineFunction &MF) override {
-    Function &F = MF.getFunction();
-    if (skipFunction(F) || !AMDGPU::isEntryFunctionCC(F.getCallingConv()))
-      return false;
-
-    // This pass only runs on GFX11+
-    const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
-    if (ST.getGeneration() < AMDGPUSubtarget::GFX11)
-      return false;
-
-    LLVM_DEBUG(dbgs() << "AMDGPUReleaseVGPRs running on " << MF.getName()
-                      << "\n");
-
-    const SIInstrInfo *SII = ST.getInstrInfo();
-    LastVGPRUseIsVMEMStore BlockVMEMStore(MF);
-
-    bool Changed = false;
-    for (auto &MBB : MF) {
-      Changed |= runOnMachineBasicBlock(MBB, SII, BlockVMEMStore);
-    }
-
-    return Changed;
-  }
-};
-
-} // namespace
-
-char AMDGPUReleaseVGPRs::ID = 0;
-
-char &llvm::AMDGPUReleaseVGPRsID = AMDGPUReleaseVGPRs::ID;
-
-INITIALIZE_PASS(AMDGPUReleaseVGPRs, DEBUG_TYPE, "Release VGPRs", false, false)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index c3f7d753e1979..704f2e0b19bb9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -405,7 +405,6 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   initializeAMDGPURewriteUndefForPHIPass(*PR);
   initializeAMDGPUUnifyMetadataPass(*PR);
   initializeSIAnnotateControlFlowPass(*PR);
-  initializeAMDGPUReleaseVGPRsPass(*PR);
   initializeAMDGPUInsertDelayAluPass(*PR);
   initializeSIInsertHardClausesPass(*PR);
   initializeSIInsertWaitcntsPass(*PR);
@@ -1431,9 +1430,6 @@ void GCNPassConfig::addPreEmitPass() {
   // cases.
   addPass(&PostRAHazardRecognizerID);
 
-  if (getOptLevel() > CodeGenOpt::Less)
-    addPass(&AMDGPUReleaseVGPRsID);
-
   if (isPassEnabled(EnableInsertDelayAlu, CodeGenOpt::Less))
     addPass(&AMDGPUInsertDelayAluID);
 
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index 8df156f24dcb6..ae88d934ad52b 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -88,7 +88,6 @@ add_llvm_target(AMDGPUCodeGen
   AMDGPURegBankCombiner.cpp
   AMDGPURegBankSelect.cpp
   AMDGPURegisterBankInfo.cpp
-  AMDGPUReleaseVGPRs.cpp
   AMDGPURemoveIncompatibleFunctions.cpp
   AMDGPUResourceUsageAnalysis.cpp
   AMDGPURewriteOutArguments.cpp
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 6f0be07a3e23e..05247d3ff1603 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -386,6 +386,10 @@ class SIInsertWaitcnts : public MachineFunctionPass {
   bool ForceEmitZeroWaitcnts;
   bool ForceEmitWaitcnt[NUM_INST_CNTS];
 
+  // S_ENDPGM instructions before which we should insert a DEALLOC_VGPRS
+  // message.
+  DenseSet<MachineInstr *> ReleaseVGPRInsts;
+
 public:
   static char ID;
 
@@ -1032,6 +1036,15 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(MachineInstr &MI,
       (MI.isReturn() && MI.isCall() && !callWaitsOnFunctionEntry(MI))) {
     Wait = Wait.combined(allZeroWaitcnt());
   }
+  // Identify S_ENDPGM instructions which may have to wait for outstanding VMEM
+  // stores. In this case it can be useful to send a message to explicitly
+  // release all VGPRs before the stores have completed.
+  else if (MI.getOpcode() == AMDGPU::S_ENDPGM ||
+           MI.getOpcode() == AMDGPU::S_ENDPGM_SAVED) {
+    if (ST->getGeneration() >= AMDGPUSubtarget::GFX11 &&
+        ScoreBrackets.getScoreRange(VS_CNT) != 0)
+      ReleaseVGPRInsts.insert(&MI);
+  }
   // Resolve vm waits before gs-done.
   else if ((MI.getOpcode() == AMDGPU::S_SENDMSG ||
             MI.getOpcode() == AMDGPU::S_SENDMSGHALT) &&
@@ -1930,5 +1943,14 @@ bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) {
     }
   }
 
+  // Insert DEALLOC_VGPR messages before previously identified S_ENDPGM
+  // instructions.
+  for (MachineInstr *MI : ReleaseVGPRInsts) {
+    BuildMI(*MI->getParent(), MI, DebugLoc(), TII->get(AMDGPU::S_SENDMSG))
+        .addImm(AMDGPU::SendMsg::ID_DEALLOC_VGPRS_GFX11Plus);
+    Modified = true;
+  }
+  ReleaseVGPRInsts.clear();
+
   return Modified;
 }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.global.atomic.csub.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.global.atomic.csub.ll
index f4a3234c73ee6..2f79135fe610d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.global.atomic.csub.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.global.atomic.csub.ll
@@ -159,7 +159,6 @@ define amdgpu_kernel void @global_atomic_csub_sgpr_base_offset_nortn(ptr addrspa
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    v_dual_mov_b32 v1, 0x1000 :: v_dual_mov_b32 v0, s2
 ; GFX11-NEXT:    global_atomic_csub_u32 v0, v1, v0, s[0:1] glc
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr i32, ptr addrspace(1) %ptr, i64 1024
   %ret = call i32 @llvm.amdgcn.global.atomic.csub.p1(ptr addrspace(1) %gep, i32 %data)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.ll
index d95f8eb5d7a45..00f6b7ac9342e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.image.atomic.dim.ll
@@ -3,8 +3,8 @@
 ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=fiji -o - %s | FileCheck -check-prefix=GFX8 %s
 ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx900 -o - %s | FileCheck -check-prefix=GFX900 %s
 ; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx90a -o - %s | FileCheck -check-prefix=GFX90A %s
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX10 %s
-; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 -o - %s | FileCheck -check-prefixes=GFX10PLUS,GFX11 %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1010 -o - %s | FileCheck -check-prefix=GFX10PLUS %s
+; RUN: llc -global-isel -mtriple=amdgcn-mesa-mesa3d -mcpu=gfx1100 -o - %s | FileCheck -check-prefix=GFX10PLUS %s
 
 define amdgpu_ps float @atomic_swap_i32_1d(<8 x i32> inreg %rsrc, i32 %data, i32 %s) {
 ; GFX6-LABEL: atomic_swap_i32_1d:
@@ -1059,32 +1059,18 @@ define amdgpu_ps void @atomic_cmpswap_i32_1d_no_return(<8 x i32> inreg %rsrc, i3
 ; GFX90A-NEXT:    image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 unorm glc
 ; GFX90A-NEXT:    s_endpgm
 ;
-; GFX10-LABEL: atomic_cmpswap_i32_1d_no_return:
-; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    s_mov_b32 s0, s2
-; GFX10-NEXT:    s_mov_b32 s1, s3
-; GFX10-NEXT:    s_mov_b32 s2, s4
-; GFX10-NEXT:    s_mov_b32 s3, s5
-; GFX10-NEXT:    s_mov_b32 s4, s6
-; GFX10-NEXT:    s_mov_b32 s5, s7
-; GFX10-NEXT:    s_mov_b32 s6, s8
-; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
-; GFX10-NEXT:    s_endpgm
-;
-; GFX11-LABEL: atomic_cmpswap_i32_1d_no_return:
-; GFX11:       ; %bb.0: ; %main_body
-; GFX11-NEXT:    s_mov_b32 s0, s2
-; GFX11-NEXT:    s_mov_b32 s1, s3
-; GFX11-NEXT:    s_mov_b32 s2, s4
-; GFX11-NEXT:    s_mov_b32 s3, s5
-; GFX11-NEXT:    s_mov_b32 s4, s6
-; GFX11-NEXT:    s_mov_b32 s5, s7
-; GFX11-NEXT:    s_mov_b32 s6, s8
-; GFX11-NEXT:    s_mov_b32 s7, s9
-; GFX11-NEXT:    image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX11-NEXT:    s_endpgm
+; GFX10PLUS-LABEL: atomic_cmpswap_i32_1d_no_return:
+; GFX10PLUS:       ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT:    s_mov_b32 s0, s2
+; GFX10PLUS-NEXT:    s_mov_b32 s1, s3
+; GFX10PLUS-NEXT:    s_mov_b32 s2, s4
+; GFX10PLUS-NEXT:    s_mov_b32 s3, s5
+; GFX10PLUS-NEXT:    s_mov_b32 s4, s6
+; GFX10PLUS-NEXT:    s_mov_b32 s5, s7
+; GFX10PLUS-NEXT:    s_mov_b32 s6, s8
+; GFX10PLUS-NEXT:    s_mov_b32 s7, s9
+; GFX10PLUS-NEXT:    image_atomic_cmpswap v[0:1], v2, s[0:7] dmask:0x3 dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10PLUS-NEXT:    s_endpgm
 main_body:
   %v = call i32 @llvm.amdgcn.image.atomic.cmpswap.1d.i32.i32(i32 %cmp, i32 %swap, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
@@ -2760,32 +2746,18 @@ define amdgpu_ps void @atomic_cmpswap_i64_1d_no_return(<8 x i32> inreg %rsrc, i6
 ; GFX90A-NEXT:    image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf unorm glc
 ; GFX90A-NEXT:    s_endpgm
 ;
-; GFX10-LABEL: atomic_cmpswap_i64_1d_no_return:
-; GFX10:       ; %bb.0: ; %main_body
-; GFX10-NEXT:    s_mov_b32 s0, s2
-; GFX10-NEXT:    s_mov_b32 s1, s3
-; GFX10-NEXT:    s_mov_b32 s2, s4
-; GFX10-NEXT:    s_mov_b32 s3, s5
-; GFX10-NEXT:    s_mov_b32 s4, s6
-; GFX10-NEXT:    s_mov_b32 s5, s7
-; GFX10-NEXT:    s_mov_b32 s6, s8
-; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm glc
-; GFX10-NEXT:    s_endpgm
-;
-; GFX11-LABEL: atomic_cmpswap_i64_1d_no_return:
-; GFX11:       ; %bb.0: ; %main_body
-; GFX11-NEXT:    s_mov_b32 s0, s2
-; GFX11-NEXT:    s_mov_b32 s1, s3
-; GFX11-NEXT:    s_mov_b32 s2, s4
-; GFX11-NEXT:    s_mov_b32 s3, s5
-; GFX11-NEXT:    s_mov_b32 s4, s6
-; GFX11-NEXT:    s_mov_b32 s5, s7
-; GFX11-NEXT:    s_mov_b32 s6, s8
-; GFX11-NEXT:    s_mov_b32 s7, s9
-; GFX11-NEXT:    image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm glc
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GFX11-NEXT:    s_endpgm
+; GFX10PLUS-LABEL: atomic_cmpswap_i64_1d_no_return:
+; GFX10PLUS:       ; %bb.0: ; %main_body
+; GFX10PLUS-NEXT:    s_mov_b32 s0, s2
+; GFX10PLUS-NEXT:    s_mov_b32 s1, s3
+; GFX10PLUS-NEXT:    s_mov_b32 s2, s4
+; GFX10PLUS-NEXT:    s_mov_b32 s3, s5
+; GFX10PLUS-NEXT:    s_mov_b32 s4, s6
+; GFX10PLUS-NEXT:    s_mov_b32 s5, s7
+; GFX10PLUS-NEXT:    s_mov_b32 s6, s8
+; GFX10PLUS-NEXT:    s_mov_b32 s7, s9
+; GFX10PLUS-NEXT:    image_atomic_cmpswap v[0:3], v4, s[0:7] dmask:0xf dim:SQ_RSRC_IMG_1D unorm glc
+; GFX10PLUS-NEXT:    s_endpgm
 main_body:
   %v = call i64 @llvm.amdgcn.image.atomic.cmpswap.1d.i64.i32(i64 %cmp, i64 %swap, i32 %s, <8 x i32> %rsrc, i32 0, i32 0)
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
index ded7081033f0e..acf69e7e7b2ad 100644
--- a/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/call-argument-types.ll
@@ -4382,7 +4382,6 @@ define amdgpu_kernel void @test_call_external_void_func_v32i32() #0 {
 ; GFX11-NEXT:    s_waitcnt vmcnt(7)
 ; GFX11-NEXT:    scratch_store_b32 off, v31, s32
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 ;
 ; HSA-LABEL: test_call_external_void_func_v32i32:
@@ -4552,7 +4551,6 @@ define amdgpu_kernel void @test_call_external_void_func_v32i32_i32(i32) #0 {
 ; GFX11-NEXT:    s_waitcnt vmcnt(7)
 ; GFX11-NEXT:    scratch_store_b32 off, v32, s4
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 ;
 ; HSA-LABEL: test_call_external_void_func_v32i32_i32:
@@ -4926,7 +4924,6 @@ define amdgpu_kernel void @test_call_external_void_func_byval_struct_i8_i32() #0
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    scratch_store_b64 off, v[0:1], s32
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 ;
 ; HSA-LABEL: test_call_external_void_func_byval_struct_i8_i32:
diff --git a/llvm/test/CodeGen/AMDGPU/cc-update.ll b/llvm/test/CodeGen/AMDGPU/cc-update.ll
index a3e22a3bc7db6..fdb8bec944cb3 100644
--- a/llvm/test/CodeGen/AMDGPU/cc-update.ll
+++ b/llvm/test/CodeGen/AMDGPU/cc-update.ll
@@ -238,7 +238,6 @@ define amdgpu_kernel void @test_kern_stack_and_call() local_unnamed_addr #0 {
 ; GFX1100-NEXT:    s_add_u32 s16, s16, ex@rel32@lo+4
 ; GFX1100-NEXT:    s_addc_u32 s17, s17, ex@rel32@hi+12
 ; GFX1100-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1100-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX1100-NEXT:    s_endpgm
 
 entry:
@@ -517,7 +516,6 @@ define amdgpu_kernel void @test_force_fp_kern_stack_and_call() local_unnamed_add
 ; GFX1100-NEXT:    s_add_u32 s16, s16, ex@rel32@lo+4
 ; GFX1100-NEXT:    s_addc_u32 s17, s17, ex@rel32@hi+12
 ; GFX1100-NEXT:    s_swappc_b64 s[30:31], s[16:17]
-; GFX1100-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX1100-NEXT:    s_endpgm
 entry:
   %x = alloca i32, align 4, addrspace(5)
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index fe4d0224b038b..4d9ed0a784bfe 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -1026,7 +1026,6 @@
 ; GCN-O2-NEXT:        SI Final Branch Preparation
 ; GCN-O2-NEXT:        SI peephole optimizations
 ; GCN-O2-NEXT:        Post RA hazard recognizer
-; GCN-O2-NEXT:        Release VGPRs
 ; GCN-O2-NEXT:        AMDGPU Insert Delay ALU
 ; GCN-O2-NEXT:        Branch relaxation pass
 ; GCN-O2-NEXT:        Register Usage Information Collector Pass
@@ -1350,7 +1349,6 @@
 ; GCN-O3-NEXT:        SI Final Branch Preparation
 ; GCN-O3-NEXT:        SI peephole optimizations
 ; GCN-O3-NEXT:        Post RA hazard recognizer
-; GCN-O3-NEXT:        Release VGPRs
 ; GCN-O3-NEXT:        AMDGPU Insert Delay ALU
 ; GCN-O3-NEXT:        Branch relaxation pass
 ; GCN-O3-NEXT:        Register Usage Information Collector Pass
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.ll
index cbf0ca740c76b..4b16e9b2ce846 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.buffer.load.ll
@@ -591,6 +591,7 @@ define amdgpu_ps void @s_buffer_load_index_across_bb(<4 x i32> inreg %desc, i32
 ; GFX11-NEXT:    buffer_load_b32 v0, v0, s[0:3], 0 offen
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    exp mrt0 v0, v0, v0, v0 done
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 main_body:
   %tmp = shl i32 %index, 4
diff --git a/llvm/test/CodeGen/AMDGPU/release-vgprs.mir b/llvm/test/CodeGen/AMDGPU/release-vgprs.mir
deleted file mode 100644
index 7adb2e4cac416..0000000000000
--- a/llvm/test/CodeGen/AMDGPU/release-vgprs.mir
+++ /dev/null
@@ -1,411 +0,0 @@
-# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-
-# RUN: llc -march=amdgcn -mcpu=gfx1100 -run-pass=release-vgprs -verify-machineinstrs -o - %s | FileCheck %s
-
---- |
-  define amdgpu_ps void @tbuffer_store1() { ret void }
-  define amdgpu_ps void @tbuffer_store2() { ret void }
-  define amdgpu_ps void @flat_store() { ret void }
-  define amdgpu_ps void @global_store() { ret void }
-  define amdgpu_ps void @buffer_store_format() { ret void }
-  define amdgpu_ps void @ds_write_b32() { ret void }
-  define amdgpu_ps void @global_store_dword() { ret void }
-  define amdgpu_ps void @multiple_basic_blocks1() { ret void }
-  define amdgpu_ps void @multiple_basic_blocks2() { ret void }
-  define amdgpu_ps void @multiple_basic_blocks3() { ret void }
-  define amdgpu_ps void @recursive_loop() { ret void }
-  define amdgpu_ps void @recursive_loop_vmem() { ret void }
-  define amdgpu_ps void @image_store() { ret void }
-  define amdgpu_ps void @scratch_store() { ret void }
-  define amdgpu_ps void @buffer_atomic() { ret void }
-  define amdgpu_ps void @flat_atomic() { ret void }
-  define amdgpu_ps void @global_atomic() { ret void }
-  define amdgpu_ps void @image_atomic() { ret void }
-...
-
----
-name:            tbuffer_store1
-body:             |
-  bb.0:
-    ; CHECK-LABEL: name: tbuffer_store1
-    ; CHECK: TBUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, killed renamable $sgpr4, 42, 117, 0, 0, implicit $exec
-    ; CHECK-NEXT: S_SENDMSG 3, implicit $exec, implicit $m0
-    ; CHECK-NEXT: S_ENDPGM 0
-    TBUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, killed renamable $sgpr4, 42, 117, 0, 0, implicit $exec
-    S_ENDPGM 0
-...
-
----
-name:            tbuffer_store2
-body:             |
-  bb.0:
-    ; CHECK-LABEL: name: tbuffer_store2
-    ; CHECK: TBUFFER_STORE_FORMAT_XYZW_OFFEN_exact killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, killed renamable $vgpr4, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 115, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
-    ; CHECK-NEXT: S_SENDMSG 3, implicit $exec, implicit $m0
-    ; CHECK-NEXT: S_ENDPGM 0
-    TBUFFER_STORE_FORMAT_XYZW_OFFEN_exact killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, killed renamable $vgpr4, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 115, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 7)
-    S_ENDPGM 0
-...
-
----
-name:            flat_store
-body:             |
-  bb.0:
-    ; CHECK-LABEL: name: flat_store
-    ; CHECK: FLAT_STORE_DWORDX4 $vgpr49_vgpr50, $vgpr26_vgpr27_vgpr28_vgpr29, 0, 0, implicit $exec, implicit $flat_scr
-    ; CHECK-NEXT: S_SENDMSG 3, implicit $exec, implicit $m0
-    ; CHECK-NEXT: S_ENDPGM 0
-    FLAT_STORE_DWORDX4 $vgpr49_vgpr50, $vgpr26_vgpr27_vgpr28_vgpr29, 0, 0, implicit $exec, implicit $flat_scr
-    S_ENDPGM 0
-...
-
----
-name:            global_store
-body:             |
-  bb.0:
-    ; CHECK-LABEL: name: global_store
-    ; CHECK: GLOBAL_STORE_DWORD undef renamable $vgpr0_vgpr1, killed renamable $vgpr1, 0, 4, implicit $exec
-    ; CHECK-NEXT: S_WAITCNT_VSCNT undef $sgpr_null, 0
-    ; CHECK-NEXT: S_SENDMSG 3, implicit $exec, implicit $m0
-    ; CHECK-NEXT: S_ENDPGM 0
-    GLOBAL_STORE_DWORD undef renamable $vgpr0_vgpr1, killed renamable $vgpr1, 0, 4, implicit $exec
-    S_WAITCNT_VSCNT undef $sgpr_null, 0
-    S_ENDPGM 0
-...
-
----
-name:            buffer_store_format
-body:             |
-  bb.0:
-    ; CHECK-LABEL: name: buffer_store_format
-    ; CHECK: BUFFER_STORE_FORMAT_D16_X_OFFEN_exact killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, killed renamable $sgpr4, 0, 0, 0, implicit $exec
-    ; CHECK-NEXT: S_SENDMSG 3, implicit $exec, implicit $m0
-    ; CHECK-NEXT: S_ENDPGM 0
-    BUFFER_STORE_FORMAT_D16_X_OFFEN_exact killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, killed renamable $sgpr4, 0, 0, 0, implicit $exec
-    S_ENDPGM 0
-...
-
----
-name:            ds_write_b32
-body:             |
-  bb.0:
-    ; CHECK-LABEL: name: ds_write_b32
-    ; CHECK: renamable $vgpr0 = IMPLICIT_DEF
-    ; CHECK-NEXT: renamable $vgpr1 = IMPLICIT_DEF
-    ; CHECK-NEXT: DS_WRITE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 12, 0, implicit $exec
-    ; CHECK-NEXT: S_ENDPGM 0
-    renamable $vgpr0 = IMPLICIT_DEF
-    renamable $vgpr1 = IMPLICIT_DEF
-    DS_WRITE_B32_gfx9 killed renamable $vgpr0, killed renamable $vgpr1, 12, 0, implicit $exec
-    S_ENDPGM 0
-
-...
----
-name:            global_store_dword
-body:             |
-  bb.0:
-    liveins: $vgpr0, $sgpr0_sgpr1
-
-    ; CHECK-LABEL: name: global_store_dword
-    ; CHECK: liveins: $vgpr0, $sgpr0_sgpr1
-    ; CHECK-NEXT: {{  $}}
-    ; CHECK-NEXT: renamable $vgpr0 = V_MAD_I32_I24_e64 killed $vgpr1, killed $vgpr0, killed $sgpr2, 0, implicit $exec
-    ; CHECK-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr2, killed renamable $vgpr0, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec
-    ; CHECK-NEXT: S_SENDMSG 3, implicit $exec, implicit $m0
-    ; CHECK-NEXT: S_ENDPGM 0
-    renamable $vgpr0 = V_MAD_I32_I24_e64 killed $vgpr1, killed $vgpr0, killed $sgpr2, 0, implicit $exec
-    GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr2, killed renamable $vgpr0, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec
-    S_ENDPGM 0
-...
-
----
-name:            multiple_basic_blocks1
-body:             |
-  ; CHECK-LABEL: name: multiple_basic_blocks1
-  ; CHECK: bb.0:
-  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $vgpr0 = BUFFER_LOAD_FORMAT_X_IDXEN killed renamable $vgpr0, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec
-  ; CHECK-NEXT:   S_BRANCH %bb.1
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.1:
-  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   $vgpr1 = V_ADD_U32_e32 $vgpr0, $vgpr2, implicit $exec
-  ; CHECK-NEXT:   S_CMP_LG_U32 killed renamable $sgpr3, renamable $sgpr4, implicit-def $scc
-  ; CHECK-NEXT:   S_CBRANCH_SCC1 %bb.1, implicit killed $scc
-  ; CHECK-NEXT:   S_BRANCH %bb.2
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.2:
-  ; CHECK-NEXT:   S_ENDPGM 0
-  bb.0:
-    successors: %bb.1
-
-    renamable $vgpr0 = BUFFER_LOAD_FORMAT_X_IDXEN killed renamable $vgpr0, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec
-    S_BRANCH %bb.1
-
-  bb.1:
-    successors: %bb.1, %bb.2
-
-    $vgpr1 = V_ADD_U32_e32 renamable $vgpr0, renamable $vgpr2, implicit $exec
-    S_CMP_LG_U32 killed renamable $sgpr3, renamable $sgpr4, implicit-def $scc
-    S_CBRANCH_SCC1 %bb.1, implicit killed $scc
-    S_BRANCH %bb.2
-
-  bb.2:
-    S_ENDPGM 0
-
-...
-
-
-# One block has a VMEM store as the last instruction, we should release the VGPRS
-...
----
-name:            multiple_basic_blocks2
-body:             |
-  ; CHECK-LABEL: name: multiple_basic_blocks2
-  ; CHECK: bb.0:
-  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFSET_exact killed renamable $vgpr0, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 125, 0, 0, implicit $exec
-  ; CHECK-NEXT:   $vgpr1 = V_ADD_U32_e32 $vgpr0, $vgpr2, implicit $exec
-  ; CHECK-NEXT:   S_BRANCH %bb.2
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.1:
-  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   $vgpr1 = V_ADD_U32_e32 $vgpr0, $vgpr2, implicit $exec
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFSET_exact killed renamable $vgpr0, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 125, 0, 0, implicit $exec
-  ; CHECK-NEXT:   S_BRANCH %bb.2
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.2:
-  ; CHECK-NEXT:   S_SENDMSG 3, implicit $exec, implicit $m0
-  ; CHECK-NEXT:   S_ENDPGM 0
-  bb.0:
-    successors: %bb.2
-
-    TBUFFER_STORE_FORMAT_X_OFFSET_exact killed renamable $vgpr0, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 125, 0, 0, implicit $exec
-    $vgpr1 = V_ADD_U32_e32 renamable $vgpr0, renamable $vgpr2, implicit $exec
-    S_BRANCH %bb.2
-
-  bb.1:
-    successors: %bb.2
-
-    $vgpr1 = V_ADD_U32_e32 renamable $vgpr0, renamable $vgpr2, implicit $exec
-    TBUFFER_STORE_FORMAT_X_OFFSET_exact killed renamable $vgpr0, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 125, 0, 0, implicit $exec
-    S_BRANCH %bb.2
-
-  bb.2:
-    S_ENDPGM 0
-...
-
-
-# One parent block has a VMEM store, release VGPRs
----
-name:            multiple_basic_blocks3
-body:             |
-  ; CHECK-LABEL: name: multiple_basic_blocks3
-  ; CHECK: bb.0:
-  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   $vgpr1 = V_ADD_U32_e32 $vgpr0, $vgpr2, implicit $exec
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_X_OFFSET_exact killed renamable $vgpr0, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 125, 0, 0, implicit $exec
-  ; CHECK-NEXT:   S_BRANCH %bb.2
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.1:
-  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   $vgpr1 = V_ADD_U32_e32 $vgpr0, $vgpr2, implicit $exec
-  ; CHECK-NEXT:   S_BRANCH %bb.2
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.2:
-  ; CHECK-NEXT:   successors: %bb.4(0x80000000)
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   S_BRANCH %bb.4
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.3:
-  ; CHECK-NEXT:   successors: %bb.4(0x80000000)
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   $vgpr1 = V_ADD_U32_e32 $vgpr0, $vgpr2, implicit $exec
-  ; CHECK-NEXT:   S_BRANCH %bb.4
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.4:
-  ; CHECK-NEXT:   S_SENDMSG 3, implicit $exec, implicit $m0
-  ; CHECK-NEXT:   S_ENDPGM 0
-  bb.0:
-    successors: %bb.2
-
-    $vgpr1 = V_ADD_U32_e32 renamable $vgpr0, renamable $vgpr2, implicit $exec
-    TBUFFER_STORE_FORMAT_X_OFFSET_exact killed renamable $vgpr0, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 125, 0, 0, implicit $exec
-    S_BRANCH %bb.2
-
-  bb.1:
-    successors: %bb.2
-
-    $vgpr1 = V_ADD_U32_e32 renamable $vgpr0, renamable $vgpr2, implicit $exec
-    S_BRANCH %bb.2
-
-  bb.2:
-    successors: %bb.4
-
-    S_BRANCH %bb.4
-
-  bb.3:
-    successors: %bb.4
-
-    $vgpr1 = V_ADD_U32_e32 renamable $vgpr0, renamable $vgpr2, implicit $exec
-    S_BRANCH %bb.4
-
-  bb.4:
-    S_ENDPGM 0
-...
-
----
-name:            recursive_loop
-body:             |
-  ; CHECK-LABEL: name: recursive_loop
-  ; CHECK: bb.0:
-  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $vgpr0 = BUFFER_LOAD_FORMAT_X_IDXEN killed renamable $vgpr0, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec
-  ; CHECK-NEXT:   S_BRANCH %bb.1
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.1:
-  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   S_CMP_LG_U32 killed renamable $sgpr3, renamable $sgpr4, implicit-def $scc
-  ; CHECK-NEXT:   S_CBRANCH_SCC1 %bb.1, implicit killed $scc
-  ; CHECK-NEXT:   S_BRANCH %bb.2
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.2:
-  ; CHECK-NEXT:   S_ENDPGM 0
-  bb.0:
-    successors: %bb.1
-
-    renamable $vgpr0 = BUFFER_LOAD_FORMAT_X_IDXEN killed renamable $vgpr0, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec
-    S_BRANCH %bb.1
-
-  bb.1:
-    successors: %bb.1, %bb.2
-
-    S_CMP_LG_U32 killed renamable $sgpr3, renamable $sgpr4, implicit-def $scc
-    S_CBRANCH_SCC1 %bb.1, implicit killed $scc
-    S_BRANCH %bb.2
-
-  bb.2:
-    S_ENDPGM 0
-...
-
----
-name:            recursive_loop_vmem
-body:             |
-  ; CHECK-LABEL: name: recursive_loop_vmem
-  ; CHECK: bb.0:
-  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   renamable $vgpr0 = BUFFER_LOAD_FORMAT_X_IDXEN killed renamable $vgpr0, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec
-  ; CHECK-NEXT:   S_BRANCH %bb.1
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.1:
-  ; CHECK-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT:   TBUFFER_STORE_FORMAT_XYZW_OFFEN_exact killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, killed renamable $vgpr4, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 115, 0, 0, implicit $exec
-  ; CHECK-NEXT:   S_CMP_LG_U32 killed renamable $sgpr3, renamable $sgpr4, implicit-def $scc
-  ; CHECK-NEXT:   S_CBRANCH_SCC1 %bb.1, implicit killed $scc
-  ; CHECK-NEXT:   S_BRANCH %bb.2
-  ; CHECK-NEXT: {{  $}}
-  ; CHECK-NEXT: bb.2:
-  ; CHECK-NEXT:   S_SENDMSG 3, implicit $exec, implicit $m0
-  ; CHECK-NEXT:   S_ENDPGM 0
-  bb.0:
-    successors: %bb.1
-
-    renamable $vgpr0 = BUFFER_LOAD_FORMAT_X_IDXEN killed renamable $vgpr0, renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec
-    S_BRANCH %bb.1
-
-  bb.1:
-    successors: %bb.1, %bb.2
-
-    TBUFFER_STORE_FORMAT_XYZW_OFFEN_exact killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, killed renamable $vgpr4, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 115, 0, 0, implicit $exec
-    S_CMP_LG_U32 killed renamable $sgpr3, renamable $sgpr4, implicit-def $scc
-    S_CBRANCH_SCC1 %bb.1, implicit killed $scc
-    S_BRANCH %bb.2
-
-  bb.2:
-    S_ENDPGM 0
-...
-
----
-name:            image_store
-body:             |
-  bb.0:
-    ; CHECK-LABEL: name: image_store
-    ; CHECK: IMAGE_STORE_V2_V1_gfx11 killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, 12, 0, 1, 0, 0, -1, 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), addrspace 7)
-    ; CHECK-NEXT: S_SENDMSG 3, implicit $exec, implicit $m0
-    ; CHECK-NEXT: S_ENDPGM 0
-  IMAGE_STORE_V2_V1_gfx11 killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, 12, 0, 1, 0, 0, -1, 0, 0, 0, implicit $exec :: (dereferenceable store (<2 x s32>), addrspace 7)
-  S_ENDPGM 0
-...
-
----
-name:            scratch_store
-body:             |
-  bb.0:
-    ; CHECK-LABEL: name: scratch_store
-    ; CHECK: renamable $sgpr0 = S_AND_B32 killed renamable $sgpr0, -16, implicit-def dead $scc
-    ; CHECK-NEXT: SCRATCH_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $sgpr0, 0, 0, implicit $exec, implicit $flat_scr
-    ; CHECK-NEXT: S_SENDMSG 3, implicit $exec, implicit $m0
-    ; CHECK-NEXT: S_ENDPGM 0
-    renamable $sgpr0 = S_AND_B32 killed renamable $sgpr0, -16, implicit-def dead $scc
-    SCRATCH_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $sgpr0, 0, 0, implicit $exec, implicit $flat_scr
-    S_ENDPGM 0
-...
-
----
-name:            buffer_atomic
-body:             |
-  bb.0:
-    ; CHECK-LABEL: name: buffer_atomic
-    ; CHECK: BUFFER_ATOMIC_ADD_F32_OFFEN killed renamable $vgpr0, killed renamable $vgpr2, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 7)
-    ; CHECK-NEXT: S_SENDMSG 3, implicit $exec, implicit $m0
-    ; CHECK-NEXT: S_ENDPGM 0
-    BUFFER_ATOMIC_ADD_F32_OFFEN killed renamable $vgpr0, killed renamable $vgpr2, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), align 1, addrspace 7)
-    S_ENDPGM 0
-...
-
----
-name:            flat_atomic
-body:             |
-  bb.0:
-    ; CHECK-LABEL: name: flat_atomic
-    ; CHECK: renamable $vgpr0_vgpr1 = FLAT_ATOMIC_DEC_X2_RTN killed renamable $vgpr0_vgpr1, killed renamable $vgpr2_vgpr3, 40, 1, implicit $exec, implicit $flat_scr
-    ; CHECK-NEXT: S_SENDMSG 3, implicit $exec, implicit $m0
-    ; CHECK-NEXT: S_ENDPGM 0
-    renamable $vgpr0_vgpr1 = FLAT_ATOMIC_DEC_X2_RTN killed renamable $vgpr0_vgpr1, killed renamable $vgpr2_vgpr3, 40, 1, implicit $exec, implicit $flat_scr
-    S_ENDPGM 0
-...
-
-
----
-name:            global_atomic
-body:             |
-  bb.0:
-    ; CHECK-LABEL: name: global_atomic
-    ; CHECK: renamable $vgpr0_vgpr1 = GLOBAL_ATOMIC_INC_X2_SADDR_RTN killed renamable $vgpr0, killed renamable $vgpr1_vgpr2, killed renamable $sgpr0_sgpr1, 40, 1, implicit $exec
-    ; CHECK-NEXT: S_SENDMSG 3, implicit $exec, implicit $m0
-    ; CHECK-NEXT: S_ENDPGM 0
-    renamable $vgpr0_vgpr1 = GLOBAL_ATOMIC_INC_X2_SADDR_RTN killed renamable $vgpr0, killed renamable $vgpr1_vgpr2, killed renamable $sgpr0_sgpr1, 40, 1, implicit $exec
-    S_ENDPGM 0
-...
-
----
-name:            image_atomic
-body:             |
-  bb.0:
-    ; CHECK-LABEL: name: image_atomic
-    ; CHECK: renamable $vgpr0_vgpr1_vgpr2_vgpr3 = IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx11 killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, killed renamable $vgpr4, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, 15, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 7)
-    ; CHECK-NEXT: S_SENDMSG 3, implicit $exec, implicit $m0
-    ; CHECK-NEXT: S_ENDPGM 0
-    renamable $vgpr0_vgpr1_vgpr2_vgpr3 = IMAGE_ATOMIC_CMPSWAP_V2_V1_gfx11 killed renamable $vgpr0_vgpr1_vgpr2_vgpr3, killed renamable $vgpr4, killed renamable $sgpr0_sgpr1_sgpr2_sgpr3_sgpr4_sgpr5_sgpr6_sgpr7, 15, 0, 1, 1, 0, 0, 0, 0, implicit $exec :: (volatile dereferenceable load store (s64), addrspace 7)
-    S_ENDPGM 0
-...
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-preexisting-vscnt.mir b/llvm/test/CodeGen/AMDGPU/waitcnt-preexisting-vscnt.mir
index 5f04e8cff880e..97dc6b60c0b4a 100644
--- a/llvm/test/CodeGen/AMDGPU/waitcnt-preexisting-vscnt.mir
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-preexisting-vscnt.mir
@@ -31,6 +31,7 @@ body:             |
     ; GFX11-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
     ; GFX11-NEXT: S_WAITCNT 7
     ; GFX11-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX11-NEXT: S_SENDMSG 3, implicit $exec, implicit $m0
     ; GFX11-NEXT: S_ENDPGM 0
     GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
     S_WAITCNT_VSCNT undef $sgpr_null, 0
@@ -69,6 +70,7 @@ body:             |
     ; GFX11-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
     ; GFX11-NEXT: S_WAITCNT 7
     ; GFX11-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX11-NEXT: S_SENDMSG 3, implicit $exec, implicit $m0
     ; GFX11-NEXT: S_ENDPGM 0
     GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
     S_WAITCNT_VSCNT undef $sgpr_null, 1
@@ -109,6 +111,7 @@ body:             |
     ; GFX11-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
     ; GFX11-NEXT: S_WAITCNT 7
     ; GFX11-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX11-NEXT: S_SENDMSG 3, implicit $exec, implicit $m0
     ; GFX11-NEXT: S_ENDPGM 0
     GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
     S_WAITCNT 112
@@ -148,6 +151,7 @@ body:             |
     ; GFX11-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
     ; GFX11-NEXT: S_WAITCNT 7
     ; GFX11-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX11-NEXT: S_SENDMSG 3, implicit $exec, implicit $m0
     ; GFX11-NEXT: S_ENDPGM 0
     GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
     S_WAITCNT_VSCNT undef $sgpr_null, 0
@@ -190,6 +194,7 @@ body:             |
     ; GFX11-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
     ; GFX11-NEXT: S_WAITCNT 7
     ; GFX11-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
+    ; GFX11-NEXT: S_SENDMSG 3, implicit $exec, implicit $m0
     ; GFX11-NEXT: S_ENDPGM 0
     GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
     S_WAITCNT 0

From c4039c10282f6cc88650f99e05fc1c382605463a Mon Sep 17 00:00:00 2001
From: LLVM GN Syncbot <llvmgnsyncbot@gmail.com>
Date: Mon, 19 Jun 2023 16:13:28 +0000
Subject: [PATCH 059/130] [gn build] Port eb7491769a51

---
 llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn | 1 -
 1 file changed, 1 deletion(-)

diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
index 36af931b713a9..e2b491202a5fa 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
@@ -173,7 +173,6 @@ static_library("LLVMAMDGPUCodeGen") {
     "AMDGPURegBankCombiner.cpp",
     "AMDGPURegBankSelect.cpp",
     "AMDGPURegisterBankInfo.cpp",
-    "AMDGPUReleaseVGPRs.cpp",
     "AMDGPURemoveIncompatibleFunctions.cpp",
     "AMDGPUResourceUsageAnalysis.cpp",
     "AMDGPURewriteOutArguments.cpp",

From a8a811997062d38c6e16e190ecd6377213b77be6 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <benny.kra@googlemail.com>
Date: Mon, 19 Jun 2023 18:18:07 +0200
Subject: [PATCH 060/130] [mlir] Fix a rare use-after free in dialect loading

applyExtensions can load further dialects, invalidating the reference to
the dialect pointer in the dialects DenseMap. Capture the pointer to
prevent that from happening.
---
 mlir/lib/IR/MLIRContext.cpp | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/mlir/lib/IR/MLIRContext.cpp b/mlir/lib/IR/MLIRContext.cpp
index cf355703aaf47..cc4b33f9ca669 100644
--- a/mlir/lib/IR/MLIRContext.cpp
+++ b/mlir/lib/IR/MLIRContext.cpp
@@ -457,8 +457,9 @@ MLIRContext::getOrLoadDialect(StringRef dialectNamespace, TypeID dialectID,
     // dialect is currently being loaded. Re-lookup the address in
     // loadedDialects because the table might have been rehashed by recursive
     // dialect loading in ctor().
-    std::unique_ptr<Dialect> &dialect = impl.loadedDialects[dialectNamespace] =
-        ctor();
+    std::unique_ptr<Dialect> &dialectOwned =
+        impl.loadedDialects[dialectNamespace] = ctor();
+    Dialect *dialect = dialectOwned.get();
     assert(dialect && "dialect ctor failed");
 
     // Refresh all the identifiers dialect field, this catches cases where a
@@ -467,13 +468,13 @@ MLIRContext::getOrLoadDialect(StringRef dialectNamespace, TypeID dialectID,
     auto stringAttrsIt = impl.dialectReferencingStrAttrs.find(dialectNamespace);
     if (stringAttrsIt != impl.dialectReferencingStrAttrs.end()) {
       for (StringAttrStorage *storage : stringAttrsIt->second)
-        storage->referencedDialect = dialect.get();
+        storage->referencedDialect = dialect;
       impl.dialectReferencingStrAttrs.erase(stringAttrsIt);
     }
 
     // Apply any extensions to this newly loaded dialect.
-    impl.dialectsRegistry.applyExtensions(dialect.get());
-    return dialect.get();
+    impl.dialectsRegistry.applyExtensions(dialect);
+    return dialect;
   }
 
 #ifndef NDEBUG

From ac2d6df2d6a88e007b6c4c059a28e7cf9dc38a03 Mon Sep 17 00:00:00 2001
From: Jeffrey Byrnes <Jeffrey.Byrnes@amd.com>
Date: Mon, 8 May 2023 11:58:14 -0700
Subject: [PATCH 061/130] [AMDGPU] Add basic support for extended i8 perm
 matching

Differential Revision: https://reviews.llvm.org/D142782

Change-Id: Ibb95224f7885839e8b77a705f487f10b47a258a6
---
 llvm/lib/Target/AMDGPU/SIISelLowering.cpp     |  427 ++-
 .../CodeGen/AMDGPU/combine-vload-extract.ll   |   13 +-
 llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll     |  172 +
 .../fast-unaligned-load-store.private.ll      |   32 -
 .../CodeGen/AMDGPU/fneg-modifier-casting.ll   |   16 +-
 .../CodeGen/AMDGPU/insert_vector_elt.v2i16.ll |   24 +-
 llvm/test/CodeGen/AMDGPU/load-lo16.ll         |   56 +-
 llvm/test/CodeGen/AMDGPU/pack.v2f16.ll        |    8 +-
 llvm/test/CodeGen/AMDGPU/pack.v2i16.ll        |    8 +-
 llvm/test/CodeGen/AMDGPU/permute_i8.ll        | 2811 +++++++++++++++++
 10 files changed, 3456 insertions(+), 111 deletions(-)
 create mode 100644 llvm/test/CodeGen/AMDGPU/permute_i8.ll

diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index ab7a543ed5fef..5725672460b34 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -25,6 +25,7 @@
 #include "llvm/Analysis/UniformityAnalysis.h"
 #include "llvm/BinaryFormat/ELF.h"
 #include "llvm/CodeGen/Analysis.h"
+#include "llvm/CodeGen/ByteProvider.h"
 #include "llvm/CodeGen/FunctionLoweringInfo.h"
 #include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
 #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
@@ -39,6 +40,7 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/KnownBits.h"
 #include "llvm/Support/ModRef.h"
+#include <optional>
 
 using namespace llvm;
 
@@ -9924,7 +9926,7 @@ static uint32_t getConstantPermuteMask(uint32_t C) {
 // value 0-3 selects corresponding source byte;
 // value 0xc selects zero;
 // value 0xff selects 0xff.
-static uint32_t getPermuteMask(SelectionDAG &DAG, SDValue V) {
+static uint32_t getPermuteMask(SDValue V) {
   assert(V.getValueSizeInBits() == 32);
 
   if (V.getNumOperands() != 2)
@@ -9940,15 +9942,13 @@ static uint32_t getPermuteMask(SelectionDAG &DAG, SDValue V) {
   default:
     break;
   case ISD::AND:
-    if (uint32_t ConstMask = getConstantPermuteMask(C)) {
+    if (uint32_t ConstMask = getConstantPermuteMask(C))
       return (0x03020100 & ConstMask) | (0x0c0c0c0c & ~ConstMask);
-    }
     break;
 
   case ISD::OR:
-    if (uint32_t ConstMask = getConstantPermuteMask(C)) {
+    if (uint32_t ConstMask = getConstantPermuteMask(C))
       return (0x03020100 & ~ConstMask) | ConstMask;
-    }
     break;
 
   case ISD::SHL:
@@ -10107,8 +10107,8 @@ SDValue SITargetLowering::performAndCombine(SDNode *N,
   const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
   if (VT == MVT::i32 && LHS.hasOneUse() && RHS.hasOneUse() &&
       N->isDivergent() && TII->pseudoToMCOpcode(AMDGPU::V_PERM_B32_e64) != -1) {
-    uint32_t LHSMask = getPermuteMask(DAG, LHS);
-    uint32_t RHSMask = getPermuteMask(DAG, RHS);
+    uint32_t LHSMask = getPermuteMask(LHS);
+    uint32_t RHSMask = getPermuteMask(RHS);
     if (LHSMask != ~0u && RHSMask != ~0u) {
       // Canonicalize the expression in an attempt to have fewer unique masks
       // and therefore fewer registers used to hold the masks.
@@ -10154,6 +10154,322 @@ SDValue SITargetLowering::performAndCombine(SDNode *N,
   return SDValue();
 }
 
+// A key component of v_perm is a mapping between byte position of the src
+// operands, and the byte position of the dest. To provide such, we need: 1. the
+// node that provides x byte of the dest of the OR, and 2. the byte of the node
+// used to provide that x byte. calculateByteProvider finds which node provides
+// a certain byte of the dest of the OR, and calculateSrcByte takes that node,
+// and finds an ultimate src and byte position For example: The supported
+// LoadCombine pattern for vector loads is as follows
+//                                t1
+//                                or
+//                      /                  \
+//                      t2                 t3
+//                     zext                shl
+//                      |                   |     \
+//                     t4                  t5     16
+//                     or                 anyext
+//                 /        \               |
+//                t6        t7             t8
+//               srl        shl             or
+//            /    |      /     \         /     \
+//           t9   t10    t11   t12      t13    t14
+//         trunc*  8    trunc*  8      and     and
+//           |            |          /    |     |    \
+//          t15          t16        t17  t18   t19   t20
+//                                trunc*  255   srl   -256
+//                                   |         /   \
+//                                  t15       t15  16
+//
+// *In this example, the truncs are from i32->i16
+//
+// calculateByteProvider would find t6, t7, t13, and t14 for bytes 0-3
+// respectively. calculateSrcByte would find (given node) -> ultimate src &
+// byteposition: t6 -> t15 & 1, t7 -> t16 & 0, t13 -> t15 & 0, t14 -> t15 & 3.
+// After finding the mapping, we can combine the tree into vperm t15, t16,
+// 0x05000407
+
+// Find the source and byte position from a node.
+// \p DestByte is the byte position of the dest of the or that the src
+// ultimately provides. \p SrcIndex is the byte of the src that maps to this
+// dest of the or byte. \p Depth tracks how many recursive iterations we have
+// performed.
+static const std::optional<ByteProvider<SDValue>>
+calculateSrcByte(const SDValue Op, uint64_t DestByte, uint64_t SrcIndex = 0,
+                 unsigned Depth = 0) {
+  // We may need to recursively traverse a series of SRLs
+  if (Depth >= 6)
+    return std::nullopt;
+
+  switch (Op->getOpcode()) {
+  case ISD::TRUNCATE: {
+    if (Op->getOperand(0).getScalarValueSizeInBits() != 32)
+      return std::nullopt;
+    return calculateSrcByte(Op->getOperand(0), DestByte, SrcIndex, Depth + 1);
+  }
+
+  case ISD::SRL: {
+    auto ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
+    if (!ShiftOp)
+      return std::nullopt;
+
+    uint64_t BitShift = ShiftOp->getZExtValue();
+
+    if (BitShift % 8 != 0)
+      return std::nullopt;
+
+    SrcIndex += BitShift / 8;
+
+    return calculateSrcByte(Op->getOperand(0), DestByte, SrcIndex, Depth + 1);
+  }
+
+  default: {
+    if (Op.getScalarValueSizeInBits() != 32)
+      return std::nullopt;
+
+    return ByteProvider<SDValue>::getSrc(Op, DestByte, SrcIndex);
+  }
+  }
+  llvm_unreachable("fully handled switch");
+}
+
+// For a byte position in the result of an Or, traverse the tree and find the
+// node (and the byte of the node) which ultimately provides this {Or,
+// BytePosition}. \p Op is the operand we are currently examining. \p Index is
+// the byte position of the Op that corresponds with the originally requested
+// byte of the Or \p Depth tracks how many recursive iterations we have
+// performed. \p StartingIndex is the originally requested byte of the Or
+static const std::optional<ByteProvider<SDValue>>
+calculateByteProvider(const SDValue &Op, unsigned Index, unsigned Depth,
+                      unsigned StartingIndex = 0) {
+  // Finding Src tree of RHS of or typically requires at least 1 additional
+  // depth
+  if (Depth > 6)
+    return std::nullopt;
+
+  unsigned BitWidth = Op.getScalarValueSizeInBits();
+  if (BitWidth % 8 != 0)
+    return std::nullopt;
+  assert(Index < BitWidth / 8 && "invalid index requested");
+
+  switch (Op.getOpcode()) {
+  case ISD::OR: {
+    auto RHS = calculateByteProvider(Op.getOperand(1), Index, Depth + 1,
+                                     StartingIndex);
+    if (!RHS)
+      return std::nullopt;
+    auto LHS = calculateByteProvider(Op.getOperand(0), Index, Depth + 1,
+                                     StartingIndex);
+    if (!LHS)
+      return std::nullopt;
+    // A well formed Or will have two ByteProviders for each byte, one of which
+    // is constant zero
+    if (!LHS->isConstantZero() && !RHS->isConstantZero())
+      return std::nullopt;
+    if (!LHS || LHS->isConstantZero())
+      return RHS;
+    if (!RHS || RHS->isConstantZero())
+      return LHS;
+    return std::nullopt;
+  }
+
+  case ISD::AND: {
+    auto BitMaskOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
+    if (!BitMaskOp)
+      return std::nullopt;
+
+    uint32_t BitMask = BitMaskOp->getZExtValue();
+    // Bits we expect for our StartingIndex
+    uint32_t IndexMask = 0xFF << (Index * 8);
+
+    if ((IndexMask & BitMask) != IndexMask) {
+      // If the result of the and partially provides the byte, then it
+      // is not well formatted
+      if (IndexMask & BitMask)
+        return std::nullopt;
+      return ByteProvider<SDValue>::getConstantZero();
+    }
+
+    return calculateSrcByte(Op->getOperand(0), StartingIndex, Index);
+  }
+
+  case ISD::SRL: {
+    auto ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
+    if (!ShiftOp)
+      return std::nullopt;
+
+    uint64_t BitShift = ShiftOp->getZExtValue();
+    if (BitShift % 8)
+      return std::nullopt;
+
+    auto BitsProvided = Op.getScalarValueSizeInBits();
+    if (BitsProvided % 8 != 0)
+      return std::nullopt;
+
+    uint64_t BytesProvided = BitsProvided / 8;
+    uint64_t ByteShift = BitShift / 8;
+    // The dest of shift will have good [0 : (BytesProvided - ByteShift)] bytes.
+    // If the byte we are trying to provide (as tracked by index) falls in this
+    // range, then the SRL provides the byte. The byte of interest of the src of
+    // the SRL is Index + ByteShift
+    return BytesProvided - ByteShift > Index
+               ? calculateSrcByte(Op->getOperand(0), StartingIndex,
+                                  Index + ByteShift)
+               : ByteProvider<SDValue>::getConstantZero();
+  }
+
+  case ISD::SHL: {
+    auto ShiftOp = dyn_cast<ConstantSDNode>(Op->getOperand(1));
+    if (!ShiftOp)
+      return std::nullopt;
+
+    uint64_t BitShift = ShiftOp->getZExtValue();
+    if (BitShift % 8 != 0)
+      return std::nullopt;
+    uint64_t ByteShift = BitShift / 8;
+
+    // If we are shifting by an amount greater than (or equal to)
+    // the index we are trying to provide, then it provides 0s. If not,
+    // then this bytes are not definitively 0s, and the corresponding byte
+    // of interest is Index - ByteShift of the src
+    return Index < ByteShift
+               ? ByteProvider<SDValue>::getConstantZero()
+               : calculateByteProvider(Op.getOperand(0), Index - ByteShift,
+                                       Depth + 1, StartingIndex);
+  }
+  case ISD::ANY_EXTEND:
+  case ISD::SIGN_EXTEND:
+  case ISD::ZERO_EXTEND: {
+    SDValue NarrowOp = Op->getOperand(0);
+    unsigned NarrowBitWidth = NarrowOp.getScalarValueSizeInBits();
+    if (NarrowBitWidth % 8 != 0)
+      return std::nullopt;
+    uint64_t NarrowByteWidth = NarrowBitWidth / 8;
+
+    if (Index >= NarrowByteWidth)
+      return Op.getOpcode() == ISD::ZERO_EXTEND
+                 ? std::optional<ByteProvider<SDValue>>(
+                       ByteProvider<SDValue>::getConstantZero())
+                 : std::nullopt;
+    return calculateByteProvider(NarrowOp, Index, Depth + 1, StartingIndex);
+  }
+
+  case ISD::TRUNCATE: {
+    unsigned NarrowBitWidth = Op.getScalarValueSizeInBits();
+    if (NarrowBitWidth % 8 != 0)
+      return std::nullopt;
+    uint64_t NarrowByteWidth = NarrowBitWidth / 8;
+
+    if (NarrowByteWidth >= Index) {
+      return calculateByteProvider(Op.getOperand(0), Index, Depth + 1,
+                                   StartingIndex);
+    }
+
+    return std::nullopt;
+  }
+
+  case ISD::LOAD: {
+    auto L = cast<LoadSDNode>(Op.getNode());
+    unsigned NarrowBitWidth = L->getMemoryVT().getSizeInBits();
+    if (NarrowBitWidth % 8 != 0)
+      return std::nullopt;
+    uint64_t NarrowByteWidth = NarrowBitWidth / 8;
+
+    // If the width of the load does not reach byte we are trying to provide for
+    // and it is not a ZEXTLOAD, then the load does not provide for the byte in
+    // question
+    if (Index >= NarrowByteWidth) {
+      return L->getExtensionType() == ISD::ZEXTLOAD
+                 ? std::optional<ByteProvider<SDValue>>(
+                       ByteProvider<SDValue>::getConstantZero())
+                 : std::nullopt;
+    }
+
+    if (NarrowByteWidth > Index) {
+      return calculateSrcByte(Op, StartingIndex, Index);
+    }
+
+    return std::nullopt;
+  }
+
+  default: {
+    return std::nullopt;
+  }
+  }
+
+  llvm_unreachable("fully handled switch");
+}
+
+// Returns true if the Operand is a scalar and is 16 bits
+static bool is16BitScalarOp(SDValue &Operand) {
+  switch (Operand.getOpcode()) {
+  case ISD::ANY_EXTEND:
+  case ISD::SIGN_EXTEND:
+  case ISD::ZERO_EXTEND: {
+    auto OpVT = Operand.getOperand(0).getValueType();
+    return !OpVT.isVector() && OpVT.getSizeInBits() == 16;
+  }
+  case ISD::LOAD: {
+    LoadSDNode *L = cast<LoadSDNode>(Operand.getNode());
+    auto ExtType = cast<LoadSDNode>(L)->getExtensionType();
+    if (ExtType == ISD::ZEXTLOAD || ExtType == ISD::SEXTLOAD ||
+        ExtType == ISD::EXTLOAD) {
+      auto MemVT = L->getMemoryVT();
+      return !MemVT.isVector() && MemVT.getSizeInBits() == 16;
+    }
+    return false;
+  }
+  default:
+    return false;
+  }
+}
+
+// Returns true if the mask matches consecutive bytes, and the first byte
+// begins at a power of 2 byte offset from 0th byte
+static bool addresses16Bits(int Mask) {
+  int Low8 = Mask & 0xff;
+  int Hi8 = (Mask & 0xff00) >> 8;
+
+  assert(Low8 < 8 && Hi8 < 8);
+  // Are the bytes contiguous in the order of increasing addresses.
+  bool IsConsecutive = (Hi8 - Low8 == 1);
+  // Is the first byte at location that is aligned for 16 bit instructions.
+  // A counter example is taking 2 consecutive bytes starting at the 8th bit.
+  // In this case, we still need code to extract the 16 bit operand, so it
+  // is better to use i8 v_perm
+  bool Is16Aligned = !(Low8 % 2);
+
+  return IsConsecutive && Is16Aligned;
+}
+
+// Do not lower into v_perm if the operands are actually 16 bit
+// and the selected bits (based on PermMask) correspond with two
+// easily addressable 16 bit operands.
+static bool hasEightBitAccesses(uint64_t PermMask, SDValue &Op,
+                                SDValue &OtherOp) {
+  int Low16 = PermMask & 0xffff;
+  int Hi16 = (PermMask & 0xffff0000) >> 16;
+
+  // ByteProvider only accepts 32 bit operands
+  assert(Op.getValueType().getSizeInBits() == 32);
+  assert(OtherOp.getValueType().getSizeInBits() == 32);
+
+  auto OpIs16Bit = is16BitScalarOp(Op);
+  auto OtherOpIs16Bit = is16BitScalarOp(Op);
+
+  // If there is a size mismatch, then we must use masking on at least one
+  // operand
+  if (OpIs16Bit != OtherOpIs16Bit)
+    return true;
+
+  // If both operands are 16 bit, return whether or not we cleanly address both
+  if (is16BitScalarOp(Op) && is16BitScalarOp(OtherOp))
+    return !addresses16Bits(Low16) || !addresses16Bits(Hi16);
+
+  // Both are 32 bit operands
+  return true;
+}
+
 SDValue SITargetLowering::performOrCombine(SDNode *N,
                                            DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
@@ -10204,8 +10520,36 @@ SDValue SITargetLowering::performOrCombine(SDNode *N,
   const SIInstrInfo *TII = getSubtarget()->getInstrInfo();
   if (VT == MVT::i32 && LHS.hasOneUse() && RHS.hasOneUse() &&
       N->isDivergent() && TII->pseudoToMCOpcode(AMDGPU::V_PERM_B32_e64) != -1) {
-    uint32_t LHSMask = getPermuteMask(DAG, LHS);
-    uint32_t RHSMask = getPermuteMask(DAG, RHS);
+
+    // If all the uses of an or need to extract the individual elements, do not
+    // attempt to lower into v_perm
+    auto usesCombinedOperand = [](SDNode *OrUse) {
+      // If we have any non-vectorized use, then it is a candidate for v_perm
+      if (OrUse->getOpcode() != ISD::BITCAST ||
+          !OrUse->getValueType(0).isVector())
+        return true;
+
+      // If we have any non-vectorized use, then it is a candidate for v_perm
+      for (auto VUse : OrUse->uses()) {
+        if (!VUse->getValueType(0).isVector())
+          return true;
+
+        // If the use of a vector is a store, then combining via a v_perm
+        // is beneficial.
+        // TODO -- whitelist more uses
+        for (auto VectorwiseOp : {ISD::STORE, ISD::CopyToReg, ISD::CopyFromReg})
+          if (VUse->getOpcode() == VectorwiseOp)
+            return true;
+      }
+      return false;
+    };
+
+    if (!any_of(N->uses(), usesCombinedOperand))
+      return SDValue();
+
+    uint32_t LHSMask = getPermuteMask(LHS);
+    uint32_t RHSMask = getPermuteMask(RHS);
+
     if (LHSMask != ~0u && RHSMask != ~0u) {
       // Canonicalize the expression in an attempt to have fewer unique masks
       // and therefore fewer registers used to hold the masks.
@@ -10238,6 +10582,71 @@ SDValue SITargetLowering::performOrCombine(SDNode *N,
                            DAG.getConstant(Sel, DL, MVT::i32));
       }
     }
+    if (LHSMask == ~0u || RHSMask == ~0u) {
+      SmallVector<ByteProvider<SDValue>, 8> PermNodes;
+
+      // VT is known to be MVT::i32, so we need to provide 4 bytes.
+      assert(VT == MVT::i32);
+      for (int i = 0; i < 4; i++) {
+        // Find the ByteProvider that provides the ith byte of the result of OR
+        std::optional<ByteProvider<SDValue>> P =
+            calculateByteProvider(SDValue(N, 0), i, 0, /*StartingIndex = */ i);
+        // TODO support constantZero
+        if (!P || P->isConstantZero())
+          return SDValue();
+
+        PermNodes.push_back(*P);
+      }
+      if (PermNodes.size() != 4)
+        return SDValue();
+
+      int FirstSrc = 0;
+      std::optional<int> SecondSrc;
+      uint64_t permMask = 0x00000000;
+      for (size_t i = 0; i < PermNodes.size(); i++) {
+        auto PermOp = PermNodes[i];
+        // Since the mask is applied to Src1:Src2, Src1 bytes must be offset
+        // by sizeof(Src2) = 4
+        int SrcByteAdjust = 4;
+
+        if (!PermOp.hasSameSrc(PermNodes[FirstSrc])) {
+          if (SecondSrc.has_value())
+            if (!PermOp.hasSameSrc(PermNodes[*SecondSrc]))
+              return SDValue();
+          // Set the index of the second distinct Src node
+          SecondSrc = i;
+          assert(PermNodes[*SecondSrc].Src->getValueType().getSizeInBits() ==
+                 32);
+          SrcByteAdjust = 0;
+        }
+        assert(PermOp.SrcOffset + SrcByteAdjust < 8);
+        assert(!DAG.getDataLayout().isBigEndian());
+        permMask |= (PermOp.SrcOffset + SrcByteAdjust) << (i * 8);
+      }
+
+      SDValue Op = *PermNodes[FirstSrc].Src;
+      SDValue OtherOp = SecondSrc.has_value() ? *PermNodes[*SecondSrc].Src
+                                              : *PermNodes[FirstSrc].Src;
+
+      // Check that we are not just extracting the bytes in order from an op
+      if (Op == OtherOp) {
+        int Low16 = permMask & 0xffff;
+        int Hi16 = (permMask & 0xffff0000) >> 16;
+
+        bool WellFormedLow = (Low16 == 0x0504) || (Low16 == 0x0100);
+        bool WellFormedHi = (Hi16 == 0x0706) || (Hi16 == 0x0302);
+
+        // The perm op would really just produce Op. So combine into Op
+        if (WellFormedLow && WellFormedHi)
+          return Op;
+      }
+
+      if (hasEightBitAccesses(permMask, Op, OtherOp)) {
+        SDLoc DL(N);
+        return DAG.getNode(AMDGPUISD::PERM, DL, MVT::i32, Op, OtherOp,
+                           DAG.getConstant(permMask, DL, MVT::i32));
+      }
+    }
   }
 
   if (VT != MVT::i64 || DCI.isBeforeLegalizeOps())
diff --git a/llvm/test/CodeGen/AMDGPU/combine-vload-extract.ll b/llvm/test/CodeGen/AMDGPU/combine-vload-extract.ll
index e2796c316e6eb..738c9b4ed6034 100644
--- a/llvm/test/CodeGen/AMDGPU/combine-vload-extract.ll
+++ b/llvm/test/CodeGen/AMDGPU/combine-vload-extract.ll
@@ -42,16 +42,11 @@ define amdgpu_kernel void @vectorLoadShuffle(ptr %in, ptr %out) {
 ; GCN-NEXT:    v_mov_b32_e32 v0, s0
 ; GCN-NEXT:    v_mov_b32_e32 v1, s1
 ; GCN-NEXT:    flat_load_dword v2, v[0:1]
-; GCN-NEXT:    s_mov_b32 s0, 0x6050400
+; GCN-NEXT:    s_mov_b32 s0, 0x7050604
 ; GCN-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GCN-NEXT:    v_bfe_u32 v3, v2, 16, 8
-; GCN-NEXT:    v_lshlrev_b32_e32 v4, 8, v2
-; GCN-NEXT:    v_perm_b32 v3, v3, v2, s0
-; GCN-NEXT:    v_and_b32_e32 v4, 0xff0000, v4
-; GCN-NEXT:    v_and_b32_e32 v2, 0xff000000, v2
-; GCN-NEXT:    v_or3_b32 v2, v3, v4, v2
+; GCN-NEXT:    v_perm_b32 v2, v2, v2, s0
 ; GCN-NEXT:    flat_store_dword v[0:1], v2
 ; GCN-NEXT:    s_endpgm
 entry:
@@ -238,9 +233,9 @@ define i64 @load_3xi16_noncombine(ptr addrspace(1) %p) #0 {
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GCN-NEXT:    global_load_ushort v2, v[0:1], off
 ; GCN-NEXT:    global_load_dword v3, v[0:1], off offset:4
-; GCN-NEXT:    s_mov_b32 s4, 0xffff0000
+; GCN-NEXT:    s_mov_b32 s4, 0x3020504
 ; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    v_and_or_b32 v0, v3, s4, v2
+; GCN-NEXT:    v_perm_b32 v0, v2, v3, s4
 ; GCN-NEXT:    v_and_b32_e32 v1, 0xffff, v3
 ; GCN-NEXT:    s_setpc_b64 s[30:31]
   %gep.p = getelementptr i16, ptr addrspace(1) %p, i32 3
diff --git a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
index 102bd99b50857..8e1241d86949f 100644
--- a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
@@ -1426,6 +1426,178 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_unaligned(ptr addrspace(1) noalias
   ret void
 }
 
+; The other use of shuffle0_0 make it profitable to lower into v_perm
+
+define amdgpu_kernel void @load_v4i8_to_v4f32_unaligned_multiuse(<4 x float> addrspace(1)* noalias %out, <4 x i8> addrspace(1)* noalias %out1, <4 x i8> addrspace(1)* noalias %in, <4 x i8> addrspace(1)* noalias %in1) nounwind {
+; SI-LABEL: load_v4i8_to_v4f32_unaligned_multiuse:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_mov_b32 s14, 0
+; SI-NEXT:    s_mov_b32 s15, s11
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[12:13], s[4:5]
+; SI-NEXT:    buffer_load_ubyte v2, v[0:1], s[12:15], 0 addr64 offset:3
+; SI-NEXT:    buffer_load_ubyte v3, v[0:1], s[12:15], 0 addr64 offset:2
+; SI-NEXT:    s_mov_b64 s[12:13], s[6:7]
+; SI-NEXT:    buffer_load_ubyte v4, v[0:1], s[12:15], 0 addr64 offset:2
+; SI-NEXT:    s_mov_b32 s10, -1
+; SI-NEXT:    s_mov_b32 s8, s2
+; SI-NEXT:    s_mov_b32 s9, s3
+; SI-NEXT:    s_mov_b32 s2, s10
+; SI-NEXT:    s_mov_b32 s3, s11
+; SI-NEXT:    s_waitcnt vmcnt(2)
+; SI-NEXT:    v_lshlrev_b32_e32 v5, 24, v2
+; SI-NEXT:    s_waitcnt vmcnt(1)
+; SI-NEXT:    v_lshlrev_b32_e32 v6, 8, v3
+; SI-NEXT:    v_cvt_f32_ubyte0_e32 v1, v3
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_or_b32_e32 v6, v4, v6
+; SI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v2
+; SI-NEXT:    v_alignbit_b32 v5, v3, v5, 24
+; SI-NEXT:    v_cvt_f32_ubyte0_e32 v2, v4
+; SI-NEXT:    v_mov_b32_e32 v3, v1
+; SI-NEXT:    v_lshlrev_b32_e32 v4, 16, v6
+; SI-NEXT:    v_or_b32_e32 v4, v5, v4
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT:    buffer_store_dword v4, off, s[8:11], 0
+; SI-NEXT:    s_endpgm
+;
+; VI-LABEL: load_v4i8_to_v4f32_unaligned_multiuse:
+; VI:       ; %bb.0:
+; VI-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
+; VI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; VI-NEXT:    s_mov_b32 s8, 0x4000405
+; VI-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-NEXT:    v_mov_b32_e32 v1, s5
+; VI-NEXT:    v_add_u32_e32 v2, vcc, s4, v0
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v1, vcc
+; VI-NEXT:    v_mov_b32_e32 v1, s7
+; VI-NEXT:    v_add_u32_e32 v4, vcc, s6, v0
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v1, vcc
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 3, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; VI-NEXT:    flat_load_ubyte v6, v[0:1]
+; VI-NEXT:    v_add_u32_e32 v0, vcc, 2, v2
+; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v3, vcc
+; VI-NEXT:    v_add_u32_e32 v2, vcc, 3, v4
+; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v5, vcc
+; VI-NEXT:    v_add_u32_e32 v4, vcc, 2, v4
+; VI-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; VI-NEXT:    flat_load_ubyte v2, v[2:3]
+; VI-NEXT:    flat_load_ubyte v3, v[4:5]
+; VI-NEXT:    flat_load_ubyte v4, v[0:1]
+; VI-NEXT:    s_mov_b32 s7, 0xf000
+; VI-NEXT:    s_mov_b32 s6, -1
+; VI-NEXT:    s_mov_b32 s4, s2
+; VI-NEXT:    s_mov_b32 s5, s3
+; VI-NEXT:    s_mov_b32 s2, s6
+; VI-NEXT:    s_mov_b32 s3, s7
+; VI-NEXT:    s_waitcnt vmcnt(3)
+; VI-NEXT:    v_lshlrev_b32_e32 v5, 8, v6
+; VI-NEXT:    v_cvt_f32_ubyte0_e32 v0, v6
+; VI-NEXT:    s_waitcnt vmcnt(2)
+; VI-NEXT:    v_lshlrev_b32_e32 v7, 8, v2
+; VI-NEXT:    s_waitcnt vmcnt(1)
+; VI-NEXT:    v_cvt_f32_ubyte0_e32 v2, v3
+; VI-NEXT:    s_waitcnt vmcnt(0)
+; VI-NEXT:    v_cvt_f32_ubyte0_e32 v1, v4
+; VI-NEXT:    v_or_b32_e32 v4, v5, v4
+; VI-NEXT:    v_or_b32_e32 v5, v7, v3
+; VI-NEXT:    v_mov_b32_e32 v3, v1
+; VI-NEXT:    v_perm_b32 v4, v4, v5, s8
+; VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; VI-NEXT:    buffer_store_dword v4, off, s[4:7], 0
+; VI-NEXT:    s_endpgm
+;
+; GFX10-LABEL: load_v4i8_to_v4f32_unaligned_multiuse:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x24
+; GFX10-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX10-NEXT:    v_mov_b32_e32 v7, 0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_clause 0x3
+; GFX10-NEXT:    global_load_ubyte v1, v0, s[4:5] offset:2
+; GFX10-NEXT:    global_load_ubyte v3, v0, s[4:5] offset:3
+; GFX10-NEXT:    global_load_ubyte v2, v0, s[6:7] offset:3
+; GFX10-NEXT:    global_load_ubyte v4, v0, s[6:7] offset:2
+; GFX10-NEXT:    s_waitcnt vmcnt(2)
+; GFX10-NEXT:    v_lshl_or_b32 v5, v3, 8, v1
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v1, v1
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_lshl_or_b32 v6, v2, 8, v4
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v2, v4
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v0, v3
+; GFX10-NEXT:    v_mov_b32_e32 v3, v1
+; GFX10-NEXT:    v_perm_b32 v4, v5, v6, 0x4000405
+; GFX10-NEXT:    global_store_dwordx4 v7, v[0:3], s[0:1]
+; GFX10-NEXT:    global_store_dword v7, v4, s[2:3]
+; GFX10-NEXT:    s_endpgm
+;
+; GFX9-LABEL: load_v4i8_to_v4f32_unaligned_multiuse:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_load_dwordx8 s[0:7], s[4:5], 0x24
+; GFX9-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX9-NEXT:    v_mov_b32_e32 v5, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    global_load_ubyte v1, v0, s[4:5] offset:2
+; GFX9-NEXT:    global_load_ubyte v2, v0, s[6:7] offset:3
+; GFX9-NEXT:    global_load_ubyte v3, v0, s[4:5] offset:3
+; GFX9-NEXT:    global_load_ubyte v4, v0, s[6:7] offset:2
+; GFX9-NEXT:    s_mov_b32 s4, 0x4000405
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_lshl_or_b32 v6, v3, 8, v1
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v1, v1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshl_or_b32 v7, v2, 8, v4
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v2, v4
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v0, v3
+; GFX9-NEXT:    v_mov_b32_e32 v3, v1
+; GFX9-NEXT:    v_perm_b32 v4, v6, v7, s4
+; GFX9-NEXT:    global_store_dwordx4 v5, v[0:3], s[0:1]
+; GFX9-NEXT:    global_store_dword v5, v4, s[2:3]
+; GFX9-NEXT:    s_endpgm
+;
+; GFX11-LABEL: load_v4i8_to_v4f32_unaligned_multiuse:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b256 s[0:7], s[0:1], 0x24
+; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-NEXT:    v_mov_b32_e32 v6, 0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_clause 0x3
+; GFX11-NEXT:    global_load_u8 v1, v0, s[4:5] offset:2
+; GFX11-NEXT:    global_load_u8 v3, v0, s[4:5] offset:3
+; GFX11-NEXT:    global_load_u8 v2, v0, s[6:7] offset:3
+; GFX11-NEXT:    global_load_u8 v0, v0, s[6:7] offset:2
+; GFX11-NEXT:    s_waitcnt vmcnt(2)
+; GFX11-NEXT:    v_lshl_or_b32 v4, v3, 8, v1
+; GFX11-NEXT:    v_cvt_f32_ubyte0_e32 v1, v1
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_lshl_or_b32 v5, v2, 8, v0
+; GFX11-NEXT:    v_cvt_f32_ubyte0_e32 v2, v0
+; GFX11-NEXT:    v_cvt_f32_ubyte0_e32 v0, v3
+; GFX11-NEXT:    v_mov_b32_e32 v3, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT:    v_perm_b32 v4, v4, v5, 0x4000405
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    global_store_b128 v6, v[0:3], s[0:1]
+; GFX11-NEXT:    global_store_b32 v6, v4, s[2:3]
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %in, i32 %tid
+  %gep1 = getelementptr <4 x i8>, <4 x i8> addrspace(1)* %in1, i32 %tid
+  %load = load <4 x i8>, <4 x i8> addrspace(1)* %gep, align 1
+  %load1 = load <4 x i8>, <4 x i8> addrspace(1)* %gep1, align 1
+  %shuffle0_0 = shufflevector <4 x i8> %load, <4 x i8> %load1, <4 x i32> <i32 3, i32 2, i32 6, i32 2>
+  %cvt = uitofp <4 x i8> %shuffle0_0 to <4 x float>
+  store <4 x float> %cvt, <4 x float> addrspace(1)* %out, align 16
+  store <4 x i8> %shuffle0_0, <4 x i8> addrspace(1)* %out1, align 4
+  ret void
+}
+
 ; FIXME: Need to handle non-uniform case for function below (load without gep).
 ; Instructions still emitted to repack bytes for add use.
 define amdgpu_kernel void @load_v4i8_to_v4f32_2_uses(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %out2, ptr addrspace(1) noalias %in) nounwind {
diff --git a/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.private.ll b/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.private.ll
index ee377bacfb161..a150908f2fa0f 100644
--- a/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.private.ll
+++ b/llvm/test/CodeGen/AMDGPU/fast-unaligned-load-store.private.ll
@@ -33,20 +33,14 @@ define i32 @private_load_2xi16_align2(ptr addrspace(5) %p) #0 {
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_load_dword v0, v0, s[0:3], 0 offen
-; GFX9-NEXT:    s_mov_b32 s4, 0xffff
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff0000, v0
-; GFX9-NEXT:    v_and_or_b32 v0, v0, s4, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-FLASTSCR-LABEL: private_load_2xi16_align2:
 ; GFX9-FLASTSCR:       ; %bb.0:
 ; GFX9-FLASTSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-FLASTSCR-NEXT:    scratch_load_dword v0, v0, off
-; GFX9-FLASTSCR-NEXT:    s_mov_b32 s0, 0xffff
 ; GFX9-FLASTSCR-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-FLASTSCR-NEXT:    v_and_b32_e32 v1, 0xffff0000, v0
-; GFX9-FLASTSCR-NEXT:    v_and_or_b32 v0, v0, s0, v1
 ; GFX9-FLASTSCR-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: private_load_2xi16_align2:
@@ -55,8 +49,6 @@ define i32 @private_load_2xi16_align2(ptr addrspace(5) %p) #0 {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    buffer_load_dword v0, v0, s[0:3], 0 offen
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_and_b32_e32 v1, 0xffff0000, v0
-; GFX10-NEXT:    v_and_or_b32 v0, 0xffff, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-FLASTSCR-LABEL: private_load_2xi16_align2:
@@ -65,8 +57,6 @@ define i32 @private_load_2xi16_align2(ptr addrspace(5) %p) #0 {
 ; GFX10-FLASTSCR-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-FLASTSCR-NEXT:    scratch_load_dword v0, v0, off
 ; GFX10-FLASTSCR-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-FLASTSCR-NEXT:    v_and_b32_e32 v1, 0xffff0000, v0
-; GFX10-FLASTSCR-NEXT:    v_and_or_b32 v0, 0xffff, v0, v1
 ; GFX10-FLASTSCR-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: private_load_2xi16_align2:
@@ -75,9 +65,6 @@ define i32 @private_load_2xi16_align2(ptr addrspace(5) %p) #0 {
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    scratch_load_b32 v0, v0, off
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_and_b32_e32 v1, 0xffff0000, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_and_or_b32 v0, 0xffff, v0, v1
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FLASTSCR-LABEL: private_load_2xi16_align2:
@@ -86,9 +73,6 @@ define i32 @private_load_2xi16_align2(ptr addrspace(5) %p) #0 {
 ; GFX11-FLASTSCR-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-FLASTSCR-NEXT:    scratch_load_b32 v0, v0, off
 ; GFX11-FLASTSCR-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-FLASTSCR-NEXT:    v_and_b32_e32 v1, 0xffff0000, v0
-; GFX11-FLASTSCR-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FLASTSCR-NEXT:    v_and_or_b32 v0, 0xffff, v0, v1
 ; GFX11-FLASTSCR-NEXT:    s_setpc_b64 s[30:31]
   %gep.p = getelementptr i16, ptr addrspace(5) %p, i64 1
   %p.0 = load i16, ptr addrspace(5) %p, align 2
@@ -213,20 +197,14 @@ define i32 @private_load_2xi16_align1(ptr addrspace(5) %p) #0 {
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-NEXT:    buffer_load_dword v0, v0, s[0:3], 0 offen
-; GFX9-NEXT:    s_mov_b32 s4, 0xffff
 ; GFX9-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff0000, v0
-; GFX9-NEXT:    v_and_or_b32 v0, v0, s4, v1
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX9-FLASTSCR-LABEL: private_load_2xi16_align1:
 ; GFX9-FLASTSCR:       ; %bb.0:
 ; GFX9-FLASTSCR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX9-FLASTSCR-NEXT:    scratch_load_dword v0, v0, off
-; GFX9-FLASTSCR-NEXT:    s_mov_b32 s0, 0xffff
 ; GFX9-FLASTSCR-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-FLASTSCR-NEXT:    v_and_b32_e32 v1, 0xffff0000, v0
-; GFX9-FLASTSCR-NEXT:    v_and_or_b32 v0, v0, s0, v1
 ; GFX9-FLASTSCR-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-LABEL: private_load_2xi16_align1:
@@ -235,8 +213,6 @@ define i32 @private_load_2xi16_align1(ptr addrspace(5) %p) #0 {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    buffer_load_dword v0, v0, s[0:3], 0 offen
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_and_b32_e32 v1, 0xffff0000, v0
-; GFX10-NEXT:    v_and_or_b32 v0, 0xffff, v0, v1
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX10-FLASTSCR-LABEL: private_load_2xi16_align1:
@@ -245,8 +221,6 @@ define i32 @private_load_2xi16_align1(ptr addrspace(5) %p) #0 {
 ; GFX10-FLASTSCR-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-FLASTSCR-NEXT:    scratch_load_dword v0, v0, off
 ; GFX10-FLASTSCR-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-FLASTSCR-NEXT:    v_and_b32_e32 v1, 0xffff0000, v0
-; GFX10-FLASTSCR-NEXT:    v_and_or_b32 v0, 0xffff, v0, v1
 ; GFX10-FLASTSCR-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: private_load_2xi16_align1:
@@ -255,9 +229,6 @@ define i32 @private_load_2xi16_align1(ptr addrspace(5) %p) #0 {
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    scratch_load_b32 v0, v0, off
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_and_b32_e32 v1, 0xffff0000, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_and_or_b32 v0, 0xffff, v0, v1
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-FLASTSCR-LABEL: private_load_2xi16_align1:
@@ -266,9 +237,6 @@ define i32 @private_load_2xi16_align1(ptr addrspace(5) %p) #0 {
 ; GFX11-FLASTSCR-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-FLASTSCR-NEXT:    scratch_load_b32 v0, v0, off
 ; GFX11-FLASTSCR-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-FLASTSCR-NEXT:    v_and_b32_e32 v1, 0xffff0000, v0
-; GFX11-FLASTSCR-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FLASTSCR-NEXT:    v_and_or_b32 v0, 0xffff, v0, v1
 ; GFX11-FLASTSCR-NEXT:    s_setpc_b64 s[30:31]
   %gep.p = getelementptr i16, ptr addrspace(5) %p, i64 1
   %p.0 = load i16, ptr addrspace(5) %p, align 1
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll b/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
index 45a2f271ffec9..f32f8ae39cacd 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-modifier-casting.ll
@@ -1280,10 +1280,9 @@ define double @fneg_f64_bitcast_build_vector_v4bf16_to_f64(bfloat %elt0, bfloat
 ; GFX9-LABEL: fneg_f64_bitcast_build_vector_v4bf16_to_f64:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
-; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
-; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    s_mov_b32 s4, 0x3020706
+; GFX9-NEXT:    v_perm_b32 v2, v2, v3, s4
+; GFX9-NEXT:    v_perm_b32 v0, v0, v1, s4
 ; GFX9-NEXT:    v_xor_b32_e32 v1, 0x80000000, v2
 ; GFX9-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -1291,13 +1290,8 @@ define double @fneg_f64_bitcast_build_vector_v4bf16_to_f64(bfloat %elt0, bfloat
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT:    v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-NEXT:    v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11-NEXT:    v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_or_b32_e32 v2, v2, v3
-; GFX11-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX11-NEXT:    v_perm_b32 v2, v2, v3, 0x3020706
+; GFX11-NEXT:    v_perm_b32 v0, v0, v1, 0x3020706
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-NEXT:    v_xor_b32_e32 v1, 0x80000000, v2
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll
index 6bcebdad0eb79..426150bd0c26f 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2i16.ll
@@ -755,10 +755,10 @@ define amdgpu_kernel void @v_insertelement_v2i16_0_reghi(ptr addrspace(1) %out,
 ; VI-NEXT:    flat_load_dword v3, v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v1, s1
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s0, v2
+; VI-NEXT:    v_mov_b32_e32 v2, 0x3020706
 ; VI-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_lshrrev_b32_e32 v2, 16, v3
-; VI-NEXT:    v_alignbit_b32 v2, v2, s4, 16
+; VI-NEXT:    v_perm_b32 v2, s4, v3, v2
 ; VI-NEXT:    flat_store_dword v[0:1], v2
 ; VI-NEXT:    s_endpgm
 ;
@@ -1594,6 +1594,7 @@ define amdgpu_kernel void @v_insertelement_v4f16_0(ptr addrspace(1) %out, ptr ad
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; VI-NEXT:    s_load_dword s4, s[4:5], 0x30
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT:    v_mov_b32_e32 v4, 0x3020504
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
@@ -1601,11 +1602,9 @@ define amdgpu_kernel void @v_insertelement_v4f16_0(ptr addrspace(1) %out, ptr ad
 ; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
-; VI-NEXT:    s_mov_b32 s0, 0xffff
-; VI-NEXT:    v_mov_b32_e32 v4, s4
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_bfi_b32 v0, s0, v4, v0
+; VI-NEXT:    v_perm_b32 v0, s4, v0, v4
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
 ;
@@ -1751,6 +1750,7 @@ define amdgpu_kernel void @v_insertelement_v4f16_2(ptr addrspace(1) %out, ptr ad
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; VI-NEXT:    s_load_dword s4, s[4:5], 0x30
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT:    v_mov_b32_e32 v4, 0x3020504
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
@@ -1758,11 +1758,9 @@ define amdgpu_kernel void @v_insertelement_v4f16_2(ptr addrspace(1) %out, ptr ad
 ; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
-; VI-NEXT:    s_mov_b32 s0, 0xffff
-; VI-NEXT:    v_mov_b32_e32 v4, s4
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_bfi_b32 v1, s0, v4, v1
+; VI-NEXT:    v_perm_b32 v1, s4, v1, v4
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
 ;
@@ -1908,6 +1906,7 @@ define amdgpu_kernel void @v_insertelement_v4i16_2(ptr addrspace(1) %out, ptr ad
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; VI-NEXT:    s_load_dword s4, s[4:5], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; VI-NEXT:    v_mov_b32_e32 v4, 0x3020504
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v2
@@ -1915,11 +1914,9 @@ define amdgpu_kernel void @v_insertelement_v4i16_2(ptr addrspace(1) %out, ptr ad
 ; VI-NEXT:    flat_load_dwordx2 v[0:1], v[0:1]
 ; VI-NEXT:    v_mov_b32_e32 v3, s1
 ; VI-NEXT:    v_add_u32_e32 v2, vcc, s0, v2
-; VI-NEXT:    s_mov_b32 s0, 0xffff
-; VI-NEXT:    v_mov_b32_e32 v4, s4
 ; VI-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(0)
-; VI-NEXT:    v_bfi_b32 v1, s0, v4, v1
+; VI-NEXT:    v_perm_b32 v1, s4, v1, v4
 ; VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
 ; VI-NEXT:    s_endpgm
 ;
@@ -2689,6 +2686,7 @@ define amdgpu_kernel void @v_insertelement_v16i16_6(ptr addrspace(1) %out, ptr a
 ; VI-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; VI-NEXT:    s_load_dword s4, s[4:5], 0x10
 ; VI-NEXT:    v_lshlrev_b32_e32 v8, 5, v0
+; VI-NEXT:    v_mov_b32_e32 v12, 0x3020504
 ; VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; VI-NEXT:    v_mov_b32_e32 v1, s3
 ; VI-NEXT:    v_add_u32_e32 v0, vcc, s2, v8
@@ -2701,11 +2699,9 @@ define amdgpu_kernel void @v_insertelement_v16i16_6(ptr addrspace(1) %out, ptr a
 ; VI-NEXT:    v_add_u32_e32 v8, vcc, s0, v8
 ; VI-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
 ; VI-NEXT:    v_add_u32_e32 v10, vcc, 16, v8
-; VI-NEXT:    s_mov_b32 s2, 0xffff
-; VI-NEXT:    v_mov_b32_e32 v12, s4
 ; VI-NEXT:    v_addc_u32_e32 v11, vcc, 0, v9, vcc
 ; VI-NEXT:    s_waitcnt vmcnt(1)
-; VI-NEXT:    v_bfi_b32 v3, s2, v12, v3
+; VI-NEXT:    v_perm_b32 v3, s4, v3, v12
 ; VI-NEXT:    s_waitcnt vmcnt(0)
 ; VI-NEXT:    flat_store_dwordx4 v[10:11], v[4:7]
 ; VI-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
diff --git a/llvm/test/CodeGen/AMDGPU/load-lo16.ll b/llvm/test/CodeGen/AMDGPU/load-lo16.ll
index 8bf28868156ea..3e8e3dec7f44c 100644
--- a/llvm/test/CodeGen/AMDGPU/load-lo16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-lo16.ll
@@ -221,9 +221,9 @@ define void @load_local_lo_v2f16_reghi_vreg(ptr addrspace(3) %in, i32 %reg) #0 {
 ; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX803-NEXT:    s_mov_b32 m0, -1
 ; GFX803-NEXT:    ds_read_u16 v0, v0
-; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
+; GFX803-NEXT:    s_mov_b32 s4, 0x3020504
 ; GFX803-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX803-NEXT:    v_perm_b32 v0, v0, v1, s4
 ; GFX803-NEXT:    flat_store_dword v[0:1], v0
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
 ; GFX803-NEXT:    s_setpc_b64 s[30:31]
@@ -682,12 +682,12 @@ define void @load_local_lo_v2i16_reghi_vreg_multi_use_hi(ptr addrspace(3) %in, <
 ; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX803-NEXT:    s_mov_b32 m0, -1
 ; GFX803-NEXT:    ds_read_u16 v0, v0
+; GFX803-NEXT:    s_mov_b32 s4, 0x3020504
 ; GFX803-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
-; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
 ; GFX803-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX803-NEXT:    ds_write_b16 v3, v2
 ; GFX803-NEXT:    s_waitcnt lgkmcnt(1)
-; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX803-NEXT:    v_perm_b32 v0, v0, v1, s4
 ; GFX803-NEXT:    flat_store_dword v[0:1], v0
 ; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX803-NEXT:    s_setpc_b64 s[30:31]
@@ -794,9 +794,9 @@ define void @load_global_lo_v2i16_reglo_vreg(ptr addrspace(1) %in, i32 %reg) #0
 ; GFX803-NEXT:    v_add_u32_e32 v0, vcc, 0xfffff002, v0
 ; GFX803-NEXT:    v_addc_u32_e32 v1, vcc, -1, v1, vcc
 ; GFX803-NEXT:    flat_load_ushort v0, v[0:1]
-; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
+; GFX803-NEXT:    s_mov_b32 s4, 0x3020504
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
-; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX803-NEXT:    v_perm_b32 v0, v0, v2, s4
 ; GFX803-NEXT:    flat_store_dword v[0:1], v0
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
 ; GFX803-NEXT:    s_setpc_b64 s[30:31]
@@ -836,9 +836,9 @@ define void @load_global_lo_v2f16_reglo_vreg(ptr addrspace(1) %in, i32 %reg) #0
 ; GFX803-NEXT:    v_add_u32_e32 v0, vcc, 0xfffff002, v0
 ; GFX803-NEXT:    v_addc_u32_e32 v1, vcc, -1, v1, vcc
 ; GFX803-NEXT:    flat_load_ushort v0, v[0:1]
-; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
+; GFX803-NEXT:    s_mov_b32 s4, 0x3020504
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
-; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX803-NEXT:    v_perm_b32 v0, v0, v2, s4
 ; GFX803-NEXT:    flat_store_dword v[0:1], v0
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
 ; GFX803-NEXT:    s_setpc_b64 s[30:31]
@@ -1050,9 +1050,9 @@ define void @load_flat_lo_v2i16_reghi_vreg(ptr %in, i32 %reg) #0 {
 ; GFX803:       ; %bb.0: ; %entry
 ; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX803-NEXT:    flat_load_ushort v0, v[0:1]
-; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
+; GFX803-NEXT:    s_mov_b32 s4, 0x3020504
 ; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX803-NEXT:    v_perm_b32 v0, v0, v2, s4
 ; GFX803-NEXT:    flat_store_dword v[0:1], v0
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
 ; GFX803-NEXT:    s_setpc_b64 s[30:31]
@@ -1089,9 +1089,9 @@ define void @load_flat_lo_v2f16_reghi_vreg(ptr %in, i32 %reg) #0 {
 ; GFX803:       ; %bb.0: ; %entry
 ; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX803-NEXT:    flat_load_ushort v0, v[0:1]
-; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
+; GFX803-NEXT:    s_mov_b32 s4, 0x3020504
 ; GFX803-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX803-NEXT:    v_perm_b32 v0, v0, v2, s4
 ; GFX803-NEXT:    flat_store_dword v[0:1], v0
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
 ; GFX803-NEXT:    s_setpc_b64 s[30:31]
@@ -1292,9 +1292,9 @@ define void @load_private_lo_v2i16_reglo_vreg(ptr addrspace(5) byval(i16) %in, i
 ; GFX803:       ; %bb.0: ; %entry
 ; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX803-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:4094
-; GFX803-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX803-NEXT:    s_mov_b32 s4, 0x3020504
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
-; GFX803-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX803-NEXT:    v_perm_b32 v0, v1, v0, s4
 ; GFX803-NEXT:    flat_store_dword v[0:1], v0
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
 ; GFX803-NEXT:    s_setpc_b64 s[30:31]
@@ -1394,9 +1394,9 @@ define void @load_private_lo_v2f16_reglo_vreg(ptr addrspace(5) byval(half) %in,
 ; GFX803:       ; %bb.0: ; %entry
 ; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX803-NEXT:    buffer_load_ushort v1, off, s[0:3], s32 offset:4094
-; GFX803-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
+; GFX803-NEXT:    s_mov_b32 s4, 0x3020504
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
-; GFX803-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX803-NEXT:    v_perm_b32 v0, v1, v0, s4
 ; GFX803-NEXT:    flat_store_dword v[0:1], v0
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
 ; GFX803-NEXT:    s_setpc_b64 s[30:31]
@@ -1444,8 +1444,8 @@ define void @load_private_lo_v2i16_reglo_vreg_nooff(ptr addrspace(5) %in, i32 %r
 ; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX803-NEXT:    buffer_load_ushort v0, off, s[0:3], 0 offset:4094 glc
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
-; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
-; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX803-NEXT:    s_mov_b32 s4, 0x3020504
+; GFX803-NEXT:    v_perm_b32 v0, v0, v1, s4
 ; GFX803-NEXT:    flat_store_dword v[0:1], v0
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
 ; GFX803-NEXT:    s_setpc_b64 s[30:31]
@@ -1493,8 +1493,8 @@ define void @load_private_lo_v2i16_reghi_vreg_nooff(ptr addrspace(5) %in, i32 %r
 ; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX803-NEXT:    buffer_load_ushort v0, off, s[0:3], 0 offset:4094 glc
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
-; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
-; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX803-NEXT:    s_mov_b32 s4, 0x3020504
+; GFX803-NEXT:    v_perm_b32 v0, v0, v1, s4
 ; GFX803-NEXT:    flat_store_dword v[0:1], v0
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
 ; GFX803-NEXT:    s_setpc_b64 s[30:31]
@@ -1542,8 +1542,8 @@ define void @load_private_lo_v2f16_reglo_vreg_nooff(ptr addrspace(5) %in, i32 %r
 ; GFX803-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX803-NEXT:    buffer_load_ushort v0, off, s[0:3], 0 offset:4094 glc
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
-; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v1
-; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX803-NEXT:    s_mov_b32 s4, 0x3020504
+; GFX803-NEXT:    v_perm_b32 v0, v0, v1, s4
 ; GFX803-NEXT:    flat_store_dword v[0:1], v0
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
 ; GFX803-NEXT:    s_setpc_b64 s[30:31]
@@ -1843,9 +1843,9 @@ define void @load_constant_lo_v2i16_reglo_vreg(ptr addrspace(4) %in, i32 %reg) #
 ; GFX803-NEXT:    v_add_u32_e32 v0, vcc, 0xfffff002, v0
 ; GFX803-NEXT:    v_addc_u32_e32 v1, vcc, -1, v1, vcc
 ; GFX803-NEXT:    flat_load_ushort v0, v[0:1]
-; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
+; GFX803-NEXT:    s_mov_b32 s4, 0x3020504
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
-; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX803-NEXT:    v_perm_b32 v0, v0, v2, s4
 ; GFX803-NEXT:    flat_store_dword v[0:1], v0
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
 ; GFX803-NEXT:    s_setpc_b64 s[30:31]
@@ -1885,9 +1885,9 @@ define void @load_constant_lo_v2f16_reglo_vreg(ptr addrspace(4) %in, i32 %reg) #
 ; GFX803-NEXT:    v_add_u32_e32 v0, vcc, 0xfffff002, v0
 ; GFX803-NEXT:    v_addc_u32_e32 v1, vcc, -1, v1, vcc
 ; GFX803-NEXT:    flat_load_ushort v0, v[0:1]
-; GFX803-NEXT:    v_and_b32_e32 v1, 0xffff0000, v2
+; GFX803-NEXT:    s_mov_b32 s4, 0x3020504
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
-; GFX803-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX803-NEXT:    v_perm_b32 v0, v0, v2, s4
 ; GFX803-NEXT:    flat_store_dword v[0:1], v0
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
 ; GFX803-NEXT:    s_setpc_b64 s[30:31]
@@ -2026,8 +2026,8 @@ define void @load_private_lo_v2i16_reglo_vreg_to_offset(i32 %reg) #0 {
 ; GFX803-NEXT:    v_mov_b32_e32 v2, 44
 ; GFX803-NEXT:    buffer_load_ushort v1, v2, s[0:3], s32 offen offset:4054 glc
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
-; GFX803-NEXT:    v_and_b32_e32 v0, 0xffff0000, v0
-; GFX803-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX803-NEXT:    s_mov_b32 s4, 0x3020504
+; GFX803-NEXT:    v_perm_b32 v0, v1, v0, s4
 ; GFX803-NEXT:    flat_store_dword v[0:1], v0
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
 ; GFX803-NEXT:    s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/pack.v2f16.ll b/llvm/test/CodeGen/AMDGPU/pack.v2f16.ll
index d8d327259dec6..e21b93a386c3e 100644
--- a/llvm/test/CodeGen/AMDGPU/pack.v2f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/pack.v2f16.ll
@@ -191,8 +191,8 @@ define amdgpu_kernel void @v_pack_v2f16(ptr addrspace(1) %in0, ptr addrspace(1)
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    flat_load_dword v1, v[2:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    s_mov_b32 s0, 0x1000504
+; GFX8-NEXT:    v_perm_b32 v0, v0, v1, s0
 ; GFX8-NEXT:    ;;#ASMSTART
 ; GFX8-NEXT:    ; use v0
 ; GFX8-NEXT:    ;;#ASMEND
@@ -271,10 +271,10 @@ define amdgpu_kernel void @v_pack_v2f16_user(ptr addrspace(1) %in0, ptr addrspac
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    flat_load_dword v1, v[2:3] glc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
+; GFX8-NEXT:    s_mov_b32 s0, 0x1000504
 ; GFX8-NEXT:    s_mov_b32 s3, 0x1100f000
 ; GFX8-NEXT:    s_mov_b32 s2, -1
-; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX8-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX8-NEXT:    v_perm_b32 v0, v0, v1, s0
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, 9, v0
 ; GFX8-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/pack.v2i16.ll b/llvm/test/CodeGen/AMDGPU/pack.v2i16.ll
index 754a925fd38f0..4b21493bd7ca6 100644
--- a/llvm/test/CodeGen/AMDGPU/pack.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/pack.v2i16.ll
@@ -187,8 +187,8 @@ define amdgpu_kernel void @v_pack_v2i16(ptr addrspace(1) %in0, ptr addrspace(1)
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
 ; GFX803-NEXT:    flat_load_dword v1, v[2:3] glc
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
-; GFX803-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX803-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX803-NEXT:    s_mov_b32 s0, 0x1000504
+; GFX803-NEXT:    v_perm_b32 v0, v0, v1, s0
 ; GFX803-NEXT:    ;;#ASMSTART
 ; GFX803-NEXT:    ; use v0
 ; GFX803-NEXT:    ;;#ASMEND
@@ -265,10 +265,10 @@ define amdgpu_kernel void @v_pack_v2i16_user(ptr addrspace(1) %in0, ptr addrspac
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
 ; GFX803-NEXT:    flat_load_dword v1, v[2:3] glc
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
+; GFX803-NEXT:    s_mov_b32 s0, 0x1000504
 ; GFX803-NEXT:    s_mov_b32 s3, 0x1100f000
 ; GFX803-NEXT:    s_mov_b32 s2, -1
-; GFX803-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
-; GFX803-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX803-NEXT:    v_perm_b32 v0, v0, v1, s0
 ; GFX803-NEXT:    v_add_u32_e32 v0, vcc, 9, v0
 ; GFX803-NEXT:    buffer_store_dword v0, off, s[0:3], 0
 ; GFX803-NEXT:    s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/permute_i8.ll b/llvm/test/CodeGen/AMDGPU/permute_i8.ll
new file mode 100644
index 0000000000000..1c7fe793bb4b9
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/permute_i8.ll
@@ -0,0 +1,2811 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX10
+; RUN: llc -mtriple=amdgcn -mcpu=gfx908  -verify-machineinstrs < %s | FileCheck %s -check-prefixes=GFX9
+
+define hidden void @shuffle6766(ptr addrspace(1) %in0, ptr addrspace(1) %in1, ptr addrspace(1) %out0) {
+; GFX10-LABEL: shuffle6766:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    global_load_dword v0, v[2:3], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_perm_b32 v0, v0, v0, 0x6060706
+; GFX10-NEXT:    global_store_dword v[4:5], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: shuffle6766:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v0, v[2:3], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x6060706
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:    global_store_dword v[4:5], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %in1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 6, i32 7, i32 6, i32 6>
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0, align 4
+  ret void
+}
+
+define hidden void @shuffle3744(ptr addrspace(1) %in0, ptr addrspace(1) %in1, ptr addrspace(1) %out0) {
+; GFX10-LABEL: shuffle3744:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    global_load_dword v6, v[0:1], off
+; GFX10-NEXT:    global_load_dword v7, v[2:3], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_perm_b32 v0, v6, v7, 0x307
+; GFX10-NEXT:    global_store_dword v[4:5], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: shuffle3744:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v6, v[0:1], off
+; GFX9-NEXT:    global_load_dword v7, v[2:3], off
+; GFX9-NEXT:    s_movk_i32 s4, 0x307
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v0, v6, v7, s4
+; GFX9-NEXT:    global_store_dword v[4:5], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %in1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 3, i32 7, i32 4, i32 4>
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0, align 4
+  ret void
+}
+
+define hidden void @shuffle4445(ptr addrspace(1) %in0, ptr addrspace(1) %in1, ptr addrspace(1) %out0) {
+; GFX10-LABEL: shuffle4445:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    global_load_dword v0, v[2:3], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_perm_b32 v0, v0, v0, 0x5040404
+; GFX10-NEXT:    global_store_dword v[4:5], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: shuffle4445:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v0, v[2:3], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040404
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:    global_store_dword v[4:5], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %in1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 4, i32 4, i32 4, i32 5>
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0, align 4
+  ret void
+}
+
+define hidden void @shuffle0101(ptr addrspace(1) %in0, ptr addrspace(1) %in1, ptr addrspace(1) %out0) {
+; GFX10-LABEL: shuffle0101:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    global_load_dword v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_perm_b32 v0, v0, v0, 0x5040504
+; GFX10-NEXT:    global_store_dword v[4:5], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: shuffle0101:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v0, v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040504
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:    global_store_dword v[4:5], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %in1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 0, i32 1, i32 0, i32 1>
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0, align 4
+  ret void
+}
+
+define hidden void @shuffle1004(ptr addrspace(1) %in0, ptr addrspace(1) %in1, ptr addrspace(1) %out0) {
+; GFX10-LABEL: shuffle1004:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    global_load_dword v6, v[0:1], off
+; GFX10-NEXT:    global_load_dword v7, v[2:3], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_perm_b32 v0, v6, v7, 0x40405
+; GFX10-NEXT:    global_store_dword v[4:5], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: shuffle1004:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v6, v[0:1], off
+; GFX9-NEXT:    global_load_dword v7, v[2:3], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x40405
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v0, v6, v7, s4
+; GFX9-NEXT:    global_store_dword v[4:5], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %in1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 1, i32 0, i32 0, i32 4>
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0, align 4
+  ret void
+}
+
+
+
+define hidden void @shuffle7533(ptr addrspace(0) %in0, ptr addrspace(0) %in1, ptr addrspace(0) %out0) {
+; GFX10-LABEL: shuffle7533:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    flat_load_dword v6, v[0:1]
+; GFX10-NEXT:    flat_load_dword v7, v[2:3]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_perm_b32 v0, v7, v6, 0x3030507
+; GFX10-NEXT:    flat_store_dword v[4:5], v0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: shuffle7533:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_load_dword v6, v[0:1]
+; GFX9-NEXT:    flat_load_dword v7, v[2:3]
+; GFX9-NEXT:    s_mov_b32 s4, 0x3030507
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v0, v7, v6, s4
+; GFX9-NEXT:    flat_store_dword v[4:5], v0
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %vec0 = load <4 x i8>, ptr addrspace(0) %in0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(0) %in1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 7, i32 5, i32 3, i32 3>
+  store <4 x i8> %shuffle0_0, ptr addrspace(0) %out0, align 4
+  ret void
+}
+
+define hidden void @shuffle7767(ptr addrspace(0) %in0, ptr addrspace(0) %in1, ptr addrspace(0) %out0) {
+; GFX10-LABEL: shuffle7767:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    flat_load_dword v0, v[2:3]
+; GFX10-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_perm_b32 v0, v0, v0, 0x7060707
+; GFX10-NEXT:    flat_store_dword v[4:5], v0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: shuffle7767:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    flat_load_dword v0, v[2:3]
+; GFX9-NEXT:    s_mov_b32 s4, 0x7060707
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:    flat_store_dword v[4:5], v0
+; GFX9-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %vec0 = load <4 x i8>, ptr addrspace(0) %in0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(0) %in1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 7, i32 7, i32 6, i32 7>
+  store <4 x i8> %shuffle0_0, ptr addrspace(0) %out0, align 4
+  ret void
+}
+
+define hidden void @shuffle0554(ptr addrspace(3) %in0, ptr addrspace(3) %in1, ptr addrspace(3) %out0) {
+; GFX10-LABEL: shuffle0554:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    ds_read_b32 v0, v0
+; GFX10-NEXT:    ds_read_b32 v1, v1
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_perm_b32 v0, v0, v1, 0x10104
+; GFX10-NEXT:    ds_write_b32 v2, v0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: shuffle0554:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_read_b32 v0, v0
+; GFX9-NEXT:    ds_read_b32 v1, v1
+; GFX9-NEXT:    s_mov_b32 s4, 0x10104
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT:    ds_write_b32 v2, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %vec0 = load <4 x i8>, ptr addrspace(3) %in0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(3) %in1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 0, i32 5, i32 5, i32 4>
+  store <4 x i8> %shuffle0_0, ptr addrspace(3) %out0, align 4
+  ret void
+}
+
+define hidden void @shuffle2127(ptr addrspace(3) %in0, ptr addrspace(3) %in1, ptr addrspace(3) %out0) {
+; GFX10-LABEL: shuffle2127:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    ds_read_b32 v0, v0
+; GFX10-NEXT:    ds_read_b32 v1, v1
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_perm_b32 v0, v0, v1, 0x3060506
+; GFX10-NEXT:    ds_write_b32 v2, v0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: shuffle2127:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    ds_read_b32 v0, v0
+; GFX9-NEXT:    ds_read_b32 v1, v1
+; GFX9-NEXT:    s_mov_b32 s4, 0x3060506
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v0, v0, v1, s4
+; GFX9-NEXT:    ds_write_b32 v2, v0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %vec0 = load <4 x i8>, ptr addrspace(3) %in0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(3) %in1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 2, i32 1, i32 2, i32 7>
+  store <4 x i8> %shuffle0_0, ptr addrspace(3) %out0, align 4
+  ret void
+}
+
+define hidden void @shuffle5047(ptr addrspace(5) %in0, ptr addrspace(5) %in1, ptr addrspace(5) %out0) {
+; GFX10-LABEL: shuffle5047:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_clause 0x1
+; GFX10-NEXT:    buffer_load_dword v3, v0, s[0:3], 0 offen
+; GFX10-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_perm_b32 v0, v4, v3, 0x7040005
+; GFX10-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: shuffle5047:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    buffer_load_dword v3, v0, s[0:3], 0 offen
+; GFX9-NEXT:    buffer_load_dword v4, v1, s[0:3], 0 offen
+; GFX9-NEXT:    s_mov_b32 s4, 0x7040005
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v0, v4, v3, s4
+; GFX9-NEXT:    buffer_store_dword v0, v2, s[0:3], 0 offen
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %vec0 = load <4 x i8>, ptr addrspace(5) %in0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(5) %in1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 5, i32 0, i32 4, i32 7>
+  store <4 x i8> %shuffle0_0, ptr addrspace(5) %out0, align 4
+  ret void
+}
+
+define hidden void @shuffle3546(ptr addrspace(1) %in0, ptr addrspace(1) %in1, ptr addrspace(1) %out0) {
+; GFX10-LABEL: shuffle3546:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    global_load_dword v6, v[0:1], off
+; GFX10-NEXT:    global_load_dword v7, v[2:3], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_perm_b32 v0, v6, v7, 0x2000107
+; GFX10-NEXT:    global_store_dword v[4:5], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: shuffle3546:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v6, v[0:1], off
+; GFX9-NEXT:    global_load_dword v7, v[2:3], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x2000107
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v0, v6, v7, s4
+; GFX9-NEXT:    global_store_dword v[4:5], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %in1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 3, i32 5, i32 4, i32 6>
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0, align 4
+  ret void
+}
+
+
+define hidden void @shuffle7330ud2(ptr addrspace(1) %in0, ptr addrspace(1) %out0) {
+; GFX10-LABEL: shuffle7330ud2:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    global_load_dword v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_perm_b32 v0, v0, v0, 0x4070706
+; GFX10-NEXT:    global_store_dword v[2:3], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: shuffle7330ud2:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v0, v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x4070706
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:    global_store_dword v[2:3], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> undef, <4 x i32> <i32 7, i32 3, i32 3, i32 0>
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0, align 4
+  ret void
+}
+
+define hidden void @shuffle5341ud2(ptr addrspace(1) %in0, ptr addrspace(1) %out0) {
+; GFX10-LABEL: shuffle5341ud2:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    global_load_dword v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_perm_b32 v0, v0, v0, 0x5040706
+; GFX10-NEXT:    global_store_dword v[2:3], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: shuffle5341ud2:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v0, v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040706
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:    global_store_dword v[2:3], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> undef, <4 x i32> <i32 5, i32 3, i32 4, i32 1>
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0, align 4
+  ret void
+}
+
+define hidden void @shuffle6106ud2(ptr addrspace(1) %in0, ptr addrspace(1) %out0) {
+; GFX10-LABEL: shuffle6106ud2:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    global_load_dword v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_perm_b32 v0, v0, v0, 0x5040504
+; GFX10-NEXT:    global_store_dword v[2:3], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: shuffle6106ud2:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v0, v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040504
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:    global_store_dword v[2:3], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> undef, <4 x i32> <i32 6, i32 1, i32 0, i32 6>
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0, align 4
+  ret void
+}
+
+
+define hidden void @shuffle4327ud2(ptr addrspace(1) %in0, ptr addrspace(1) %out0) {
+; GFX10-LABEL: shuffle4327ud2:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    global_load_dword v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_perm_b32 v0, v0, v0, 0x7060706
+; GFX10-NEXT:    global_store_dword v[2:3], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: shuffle4327ud2:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v0, v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x7060706
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:    global_store_dword v[2:3], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> undef, <4 x i32> <i32 4, i32 3, i32 2, i32 7>
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0, align 4
+  ret void
+}
+
+define hidden void @shuffle3263ud2(ptr addrspace(1) %in0, ptr addrspace(1) %out0) {
+; GFX10-LABEL: shuffle3263ud2:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    global_load_dword v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_perm_b32 v0, v0, v0, 0x7060607
+; GFX10-NEXT:    global_store_dword v[2:3], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: shuffle3263ud2:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v0, v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x7060607
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:    global_store_dword v[2:3], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> undef, <4 x i32> <i32 3, i32 2, i32 6, i32 3>
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0, align 4
+  ret void
+}
+
+define hidden void @shuffle2763ud2(ptr addrspace(1) %in0, ptr addrspace(1) %out0) {
+; GFX10-LABEL: shuffle2763ud2:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    global_load_dword v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_perm_b32 v0, v0, v0, 0x7060706
+; GFX10-NEXT:    global_store_dword v[2:3], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: shuffle2763ud2:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v0, v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x7060706
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:    global_store_dword v[2:3], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> undef, <4 x i32> <i32 2, i32 7, i32 6, i32 3>
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0, align 4
+  ret void
+}
+
+define hidden void @shuffle1327ud2(ptr addrspace(1) %in0, ptr addrspace(1) %out0) {
+; GFX10-LABEL: shuffle1327ud2:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    global_load_dword v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_perm_b32 v0, v0, v0, 0x7060705
+; GFX10-NEXT:    global_store_dword v[2:3], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: shuffle1327ud2:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v0, v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x7060705
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:    global_store_dword v[2:3], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> undef, <4 x i32> <i32 1, i32 3, i32 2, i32 7>
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0, align 4
+  ret void
+}
+
+define hidden void @shuffle0605ud2(ptr addrspace(1) %in0, ptr addrspace(1) %out0) {
+; GFX10-LABEL: shuffle0605ud2:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    global_load_dword v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_perm_b32 v0, v0, v0, 0x5040504
+; GFX10-NEXT:    global_store_dword v[2:3], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: shuffle0605ud2:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v0, v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x5040504
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v0, v0, v0, s4
+; GFX9-NEXT:    global_store_dword v[2:3], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> undef, <4 x i32> <i32 0, i32 6, i32 0, i32 5>
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0, align 4
+  ret void
+}
+
+define hidden void @insertUsesOr(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0) {
+; GFX10-LABEL: insertUsesOr:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    global_load_dword v0, v[0:1], off
+; GFX10-NEXT:    v_lshlrev_b16 v1, 8, v4
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT:    global_store_dword v[5:6], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: insertUsesOr:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v0, v[0:1], off
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT:    global_store_dword v[5:6], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %in1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 4>
+  %vecins = insertelement <4 x i8> %shuffle0_0, i8 %elt, i32 1
+  store <4 x i8> %vecins, ptr addrspace(1) %out0
+  ret void
+}
+
+define hidden void @addUsesOr(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0) {
+; GFX10-LABEL: addUsesOr:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    global_load_dword v4, v[0:1], off
+; GFX10-NEXT:    global_load_dword v7, v[2:3], off
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 24, v4
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 24, v7
+; GFX10-NEXT:    v_lshrrev_b32_e32 v2, 16, v4
+; GFX10-NEXT:    v_lshrrev_b32_e32 v3, 16, v7
+; GFX10-NEXT:    v_add_nc_u16 v0, v0, v1
+; GFX10-NEXT:    v_lshrrev_b16 v1, 8, v7
+; GFX10-NEXT:    v_add_nc_u16 v2, v2, v3
+; GFX10-NEXT:    v_lshlrev_b16 v0, 8, v0
+; GFX10-NEXT:    v_add_nc_u16 v1, v4, v1
+; GFX10-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_lshlrev_b16 v1, 8, v1
+; GFX10-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT:    global_store_dword v[5:6], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: addUsesOr:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    global_load_dword v7, v[2:3], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_add_u16_sdwa v0, v4, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX9-NEXT:    v_add_u16_sdwa v1, v4, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX9-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_add_u16_sdwa v0, v4, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PRESERVE src0_sel:DWORD src1_sel:BYTE_1
+; GFX9-NEXT:    global_store_dword v[5:6], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %in1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> undef, <4 x i32> <i32 7, i32 0, i32 6, i32 3>
+  %added = add <4 x i8> %shuffle0_0, %vec1
+  store <4 x i8> %added, ptr addrspace(1) %out0
+  ret void
+}
+
+
+define amdgpu_kernel void @shuffle8i8(ptr addrspace(1) %in0, ptr addrspace(1) %in1, ptr addrspace(1) %out1) #0 {
+; GFX10-LABEL: shuffle8i8:
+; GFX10:       ; %bb.0: ; %bb
+; GFX10-NEXT:    s_clause 0x1
+; GFX10-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX10-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX10-NEXT:    v_mov_b32_e32 v4, 0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-NEXT:    s_load_dwordx2 s[8:9], s[6:7], 0x0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_lshr_b32 s1, s1, 8
+; GFX10-NEXT:    s_lshr_b32 s4, s9, 16
+; GFX10-NEXT:    v_lshlrev_b16 v0, 8, s9
+; GFX10-NEXT:    v_and_b32_e64 v1, 0xffffff00, s8
+; GFX10-NEXT:    v_lshlrev_b16 v2, 8, s4
+; GFX10-NEXT:    v_lshlrev_b16 v3, 8, s8
+; GFX10-NEXT:    s_lshr_b32 s4, s0, 16
+; GFX10-NEXT:    v_or_b32_sdwa v0, s1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v1, s4, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v2, s0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v3, s0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT:    global_store_dwordx2 v4, v[0:1], s[2:3]
+; GFX10-NEXT:    s_endpgm
+;
+; GFX9-LABEL: shuffle8i8:
+; GFX9:       ; %bb.0: ; %bb
+; GFX9-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x24
+; GFX9-NEXT:    s_load_dwordx2 s[2:3], s[0:1], 0x34
+; GFX9-NEXT:    v_mov_b32_e32 v0, 0xffffff00
+; GFX9-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX9-NEXT:    s_load_dwordx2 s[8:9], s[6:7], 0x0
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-NEXT:    s_lshr_b32 s1, s1, 8
+; GFX9-NEXT:    v_lshlrev_b16_e64 v1, 8, s9
+; GFX9-NEXT:    v_or_b32_sdwa v4, s1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    s_lshr_b32 s1, s9, 16
+; GFX9-NEXT:    s_lshr_b32 s4, s0, 16
+; GFX9-NEXT:    v_lshlrev_b16_e64 v3, 8, s8
+; GFX9-NEXT:    v_and_b32_e32 v0, s8, v0
+; GFX9-NEXT:    v_lshlrev_b16_e64 v1, 8, s1
+; GFX9-NEXT:    v_or_b32_sdwa v3, s0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v0, s4, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, s0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v0, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX9-NEXT:    s_endpgm
+bb:
+  %vec0 = load <8 x i8>, ptr addrspace(1) %in0
+  %vec1 = load <8 x i8>, ptr addrspace(1) %in1
+  %shuffle0 = shufflevector <8 x i8> %vec0, <8 x i8> %vec1, <8 x i32> <i32 1, i32 8, i32 5, i32 12, i32 0, i32 14, i32 2, i32 9>
+  store <8 x i8> %shuffle0, ptr addrspace(1) %out1
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
+declare i32 @llvm.amdgcn.workitem.id.y() nounwind readnone
+
+; Not combined to perm due to non-vectorized use, non-divergent
+define hidden void @add(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0) {
+; GFX10-LABEL: add:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    global_load_dword v4, v[0:1], off
+; GFX10-NEXT:    global_load_dword v7, v[2:3], off
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 24, v4
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_lshrrev_b16 v1, 8, v7
+; GFX10-NEXT:    v_lshrrev_b32_e32 v2, 24, v7
+; GFX10-NEXT:    v_lshrrev_b16 v3, 8, v4
+; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 16, v7
+; GFX10-NEXT:    v_add_nc_u16 v0, v0, v1
+; GFX10-NEXT:    v_add_nc_u16 v2, v7, v2
+; GFX10-NEXT:    v_add_nc_u16 v3, v3, v7
+; GFX10-NEXT:    v_add_nc_u16 v1, v1, v4
+; GFX10-NEXT:    v_lshlrev_b16 v0, 8, v0
+; GFX10-NEXT:    v_lshlrev_b16 v2, 8, v2
+; GFX10-NEXT:    v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT:    global_store_dword v[5:6], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: add:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v4, v[2:3], off
+; GFX9-NEXT:    global_load_dword v7, v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 8, v4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_add_u16_sdwa v1, v7, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_add_u16_sdwa v2, v4, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-NEXT:    v_add_u16_sdwa v3, v7, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_add_u16_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    global_store_dword v[5:6], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %in1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 1, i32 3, i32 5, i32 4>
+  %vecins = add <4 x i8> %shuffle0_0, %vec1
+  store <4 x i8> %vecins, ptr addrspace(1) %out0
+  ret void
+}
+
+; Not combined to perm due to non-vectorized use
+define hidden void @add_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0) {
+; GFX10-LABEL: add_div:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    global_load_dword v4, v[0:1], off
+; GFX10-NEXT:    global_load_dword v7, v[2:3], off
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 24, v4
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_lshrrev_b16 v1, 8, v7
+; GFX10-NEXT:    v_add_nc_u16 v0, v0, v1
+; GFX10-NEXT:    v_lshrrev_b16 v1, 8, v4
+; GFX10-NEXT:    v_lshlrev_b16 v0, 8, v0
+; GFX10-NEXT:    v_add_nc_u16 v1, v1, v7
+; GFX10-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT:    global_store_dword v[5:6], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: add_div:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    global_load_dword v7, v[2:3], off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_add_u16_sdwa v0, v4, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_1
+; GFX9-NEXT:    v_add_u16_sdwa v1, v4, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT:    global_store_dword v[5:6], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <4 x i8>, ptr addrspace(1) %in0, i32 %tid
+  %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %in1, i32 %tid
+  %vec0 = load <4 x i8>, ptr addrspace(1) %gep0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 4>
+  %vecins = add <4 x i8> %shuffle0_0, %vec1
+  store <4 x i8> %vecins, ptr addrspace(1) %out0
+  ret void
+}
+
+; Not combined to perm due to non-divergent use
+define hidden void @add_store(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0, ptr addrspace(1) %out1) {
+; GFX10-LABEL: add_store:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    global_load_dword v4, v[0:1], off
+; GFX10-NEXT:    global_load_dword v9, v[2:3], off
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 24, v4
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_lshrrev_b16 v1, 8, v9
+; GFX10-NEXT:    v_lshrrev_b16 v2, 8, v4
+; GFX10-NEXT:    v_add_nc_u16 v1, v0, v1
+; GFX10-NEXT:    v_add_nc_u16 v3, v2, v9
+; GFX10-NEXT:    v_lshlrev_b16 v0, 8, v0
+; GFX10-NEXT:    v_lshlrev_b16 v1, 8, v1
+; GFX10-NEXT:    v_or_b32_e32 v0, v2, v0
+; GFX10-NEXT:    v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX10-NEXT:    global_store_dword v[5:6], v1, off
+; GFX10-NEXT:    global_store_dword v[7:8], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: add_store:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    global_load_dword v9, v[2:3], off
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 8, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 24, v4
+; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_add_u16_sdwa v1, v1, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
+; GFX9-NEXT:    v_add_u16_e32 v3, v0, v9
+; GFX9-NEXT:    v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT:    global_store_dword v[5:6], v1, off
+; GFX9-NEXT:    global_store_dword v[7:8], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %vec0 = load <4 x i8>, ptr addrspace(1) %in0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %in1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 4>
+  %vecins = add <4 x i8> %shuffle0_0, %vec1
+  store <4 x i8> %vecins, ptr addrspace(1) %out0
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out1
+  ret void
+}
+
+; Not combined to perm due to 16 bit or
+define hidden void @add_store_div_16(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0, ptr addrspace(1) %out1) {
+; GFX10-LABEL: add_store_div_16:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    global_load_dword v4, v[0:1], off
+; GFX10-NEXT:    global_load_dword v9, v[2:3], off
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 24, v4
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_lshrrev_b16 v1, 8, v9
+; GFX10-NEXT:    v_lshrrev_b16 v2, 8, v4
+; GFX10-NEXT:    v_add_nc_u16 v1, v0, v1
+; GFX10-NEXT:    v_add_nc_u16 v3, v2, v9
+; GFX10-NEXT:    v_lshlrev_b16 v0, 8, v0
+; GFX10-NEXT:    v_lshlrev_b16 v1, 8, v1
+; GFX10-NEXT:    v_or_b32_e32 v0, v2, v0
+; GFX10-NEXT:    v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX10-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX10-NEXT:    global_store_dword v[5:6], v1, off
+; GFX10-NEXT:    global_store_dword v[7:8], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: add_store_div_16:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v9, v[0:1], off
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v0, v[0:1], off
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_lshrrev_b16_e32 v1, 8, v9
+; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 24, v9
+; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v2
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_add_u16_sdwa v2, v2, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
+; GFX9-NEXT:    v_add_u16_e32 v0, v1, v0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_e32 v1, v1, v3
+; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
+; GFX9-NEXT:    v_and_b32_e32 v1, 0xffff, v1
+; GFX9-NEXT:    global_store_dword v[5:6], v0, off
+; GFX9-NEXT:    global_store_dword v[7:8], v1, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <4 x i8>, ptr addrspace(1) %in0, i32 %tid
+  %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %in1, i32 %tid
+  %vec0 = load <4 x i8>, ptr addrspace(1) %gep0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> undef, <4 x i32> <i32 1, i32 3, i32 5, i32 4>
+  %vecins = add <4 x i8> %shuffle0_0, %vec1
+  store <4 x i8> %vecins, ptr addrspace(1) %out0
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out1
+  ret void
+}
+
+; Vectorized use, divergent, 32 bit or
+define hidden void @add_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0, ptr addrspace(1) %out1) {
+; GFX10-LABEL: add_store_div:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    global_load_dword v4, v[0:1], off
+; GFX10-NEXT:    global_load_dword v9, v[2:3], off
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 24, v4
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_lshrrev_b16 v1, 8, v9
+; GFX10-NEXT:    v_lshrrev_b32_e32 v2, 24, v9
+; GFX10-NEXT:    v_lshrrev_b16 v3, 8, v4
+; GFX10-NEXT:    v_lshrrev_b32_e32 v10, 16, v9
+; GFX10-NEXT:    v_add_nc_u16 v0, v0, v1
+; GFX10-NEXT:    v_add_nc_u16 v2, v9, v2
+; GFX10-NEXT:    v_add_nc_u16 v3, v3, v9
+; GFX10-NEXT:    v_add_nc_u16 v1, v1, v10
+; GFX10-NEXT:    v_lshlrev_b16 v0, 8, v0
+; GFX10-NEXT:    v_lshlrev_b16 v2, 8, v2
+; GFX10-NEXT:    v_or_b32_sdwa v0, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT:    v_perm_b32 v1, v4, v9, 0x10705
+; GFX10-NEXT:    global_store_dword v[5:6], v0, off
+; GFX10-NEXT:    global_store_dword v[7:8], v1, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: add_store_div:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v4, v[2:3], off
+; GFX9-NEXT:    global_load_dword v9, v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x10705
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 8, v4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v1, v9, v4, s4
+; GFX9-NEXT:    v_add_u16_sdwa v2, v9, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_add_u16_sdwa v3, v4, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-NEXT:    v_add_u16_sdwa v9, v9, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_add_u16_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    global_store_dword v[5:6], v0, off
+; GFX9-NEXT:    global_store_dword v[7:8], v1, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <4 x i8>, ptr addrspace(1) %in0, i32 %tid
+  %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %in1, i32 %tid
+  %vec0 = load <4 x i8>, ptr addrspace(1) %gep0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 1, i32 3, i32 5, i32 4>
+  %vecins = add <4 x i8> %shuffle0_0, %vec1
+  store <4 x i8> %vecins, ptr addrspace(1) %out0
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out1
+  ret void
+}
+
+define hidden void @and_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0, ptr addrspace(1) %out1) {
+; GFX10-LABEL: and_store_div:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    global_load_dword v4, v[0:1], off
+; GFX10-NEXT:    global_load_dword v9, v[2:3], off
+; GFX10-NEXT:    v_mov_b32_e32 v0, 2
+; GFX10-NEXT:    v_mov_b32_e32 v1, 1
+; GFX10-NEXT:    v_mov_b32_e32 v3, 0x102
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_and_b32_e32 v2, 0x100, v4
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_and_b32_sdwa v0, v9, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_and_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX10-NEXT:    v_and_b32_sdwa v1, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT:    v_perm_b32 v1, v4, v9, 0x5070006
+; GFX10-NEXT:    global_store_dword v[5:6], v0, off
+; GFX10-NEXT:    global_store_dword v[7:8], v1, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: and_store_div:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    global_load_dword v9, v[2:3], off
+; GFX9-NEXT:    v_mov_b32_e32 v0, 2
+; GFX9-NEXT:    v_mov_b32_e32 v1, 1
+; GFX9-NEXT:    s_movk_i32 s5, 0x102
+; GFX9-NEXT:    s_mov_b32 s4, 0x5070006
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_and_b32_e32 v2, 0x100, v4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_sdwa v0, v9, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v1, v4, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v2, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX9-NEXT:    v_and_b32_sdwa v1, v2, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    v_perm_b32 v3, v4, v9, s4
+; GFX9-NEXT:    global_store_dword v[5:6], v0, off
+; GFX9-NEXT:    global_store_dword v[7:8], v3, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <4 x i8>, ptr addrspace(1) %in0, i32 %tid
+  %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %in1, i32 %tid
+  %vec0 = load <4 x i8>, ptr addrspace(1) %gep0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 2, i32 4, i32 3, i32 1>
+  %vecins = and <4 x i8> %shuffle0_0, <i8 1, i8 2, i8 2, i8 1>
+  store <4 x i8> %vecins, ptr addrspace(1) %out0
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out1
+  ret void
+}
+
+define hidden void @ashr_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0, ptr addrspace(1) %out1) {
+; GFX10-LABEL: ashr_store_div:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    global_load_dword v9, v[0:1], off
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v2, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    global_load_dword v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_ashrrev_i32_e32 v1, 24, v9
+; GFX10-NEXT:    v_bfe_i32 v2, v9, 0, 8
+; GFX10-NEXT:    v_lshlrev_b16 v3, 6, v1
+; GFX10-NEXT:    v_lshlrev_b16 v2, 7, v2
+; GFX10-NEXT:    v_lshrrev_b16 v1, 1, v1
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_ashrrev_i16 v4, 10, v0
+; GFX10-NEXT:    v_perm_b32 v0, v9, v0, 0x4010707
+; GFX10-NEXT:    v_and_b32_e32 v3, 0xffffff00, v3
+; GFX10-NEXT:    v_and_b32_e32 v2, 0xffffff00, v2
+; GFX10-NEXT:    v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v2, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT:    global_store_dword v[5:6], v1, off
+; GFX10-NEXT:    global_store_dword v[7:8], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: ashr_store_div:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    global_load_dword v9, v[2:3], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x4010707
+; GFX9-NEXT:    v_mov_b32_e32 v0, 7
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 24, v4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v2, v4, v9, s4
+; GFX9-NEXT:    v_lshlrev_b16_sdwa v0, v0, sext(v4) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX9-NEXT:    v_lshlrev_b16_e32 v4, 6, v1
+; GFX9-NEXT:    v_ashrrev_i16_e32 v3, 10, v9
+; GFX9-NEXT:    v_lshrrev_b16_e32 v1, 1, v1
+; GFX9-NEXT:    v_and_b32_e32 v0, 0xffffff00, v0
+; GFX9-NEXT:    v_and_b32_e32 v4, 0xffffff00, v4
+; GFX9-NEXT:    v_or_b32_sdwa v0, v3, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    global_store_dword v[5:6], v0, off
+; GFX9-NEXT:    global_store_dword v[7:8], v2, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <4 x i8>, ptr addrspace(1) %in0, i32 %tid
+  %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %in1, i32 %tid
+  %vec0 = load <4 x i8>, ptr addrspace(1) %gep0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 3, i32 3, i32 5, i32 0>
+  %vecins = ashr <4 x i8> %shuffle0_0, <i8 1, i8 2, i8 2, i8 1>
+  store <4 x i8> %vecins, ptr addrspace(1) %out0
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out1
+  ret void
+}
+
+define hidden void @bc_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0, ptr addrspace(1) %out1) {
+; GFX10-LABEL: bc_store_div:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    global_load_dword v4, v[0:1], off
+; GFX10-NEXT:    global_load_dword v9, v[2:3], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_perm_b32 v0, v9, v4, 0x7060104
+; GFX10-NEXT:    global_store_dword v[7:8], v0, off
+; GFX10-NEXT:    global_store_dword v[5:6], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: bc_store_div:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    global_load_dword v9, v[2:3], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x7060104
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v0, v9, v4, s4
+; GFX9-NEXT:    global_store_dword v[7:8], v0, off
+; GFX9-NEXT:    global_store_dword v[5:6], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <4 x i8>, ptr addrspace(1) %in0, i32 %tid
+  %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %in1, i32 %tid
+  %vec0 = load <4 x i8>, ptr addrspace(1) %gep0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 4, i32 1, i32 6, i32 7>
+  %insvec = bitcast <4 x i8> %shuffle0_0 to i32
+  store i32 %insvec, ptr addrspace(1) %out1
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0
+  ret void
+}
+
+
+define hidden void @eve_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0, ptr addrspace(1) %out1, ptr addrspace(1) %out2) {
+; GFX10-LABEL: eve_store_div:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    global_load_dword v4, v[0:1], off
+; GFX10-NEXT:    global_load_dword v5, v[2:3], off
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 24, v4
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_perm_b32 v1, v5, v4, 0x1020305
+; GFX10-NEXT:    global_store_byte v[9:10], v0, off
+; GFX10-NEXT:    global_store_dword v[7:8], v1, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: eve_store_div:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    global_load_dword v5, v[2:3], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x1020305
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 24, v4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v0, v5, v4, s4
+; GFX9-NEXT:    global_store_byte v[9:10], v1, off
+; GFX9-NEXT:    global_store_dword v[7:8], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <4 x i8>, ptr addrspace(1) %in0, i32 %tid
+  %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %in1, i32 %tid
+  %vec0 = load <4 x i8>, ptr addrspace(1) %gep0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 5, i32 3, i32 2, i32 1>
+  %tmp = extractelement <4 x i8> %shuffle0_0, i32 1
+  store i8 %tmp, ptr addrspace(1) %out2
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out1
+  ret void
+}
+
+; Not combined to perm due to multi use of or operands (introduced by insert op)
+define hidden void @ive_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0, ptr addrspace(1) %out1) {
+; GFX10-LABEL: ive_store_div:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v9, 0x3ff, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v9, 2, v9
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v9
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v9
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    global_load_dword v9, v[0:1], off
+; GFX10-NEXT:    global_load_dword v10, v[2:3], off
+; GFX10-NEXT:    v_mov_b32_e32 v0, 16
+; GFX10-NEXT:    v_mov_b32_e32 v1, 0xff
+; GFX10-NEXT:    v_lshlrev_b16 v2, 8, v4
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v0, v0, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_and_b32_sdwa v1, v10, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v0, v9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_e32 v1, v1, v2
+; GFX10-NEXT:    v_lshlrev_b32_e32 v2, 16, v0
+; GFX10-NEXT:    v_alignbit_b32 v0, v0, v10, 16
+; GFX10-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT:    global_store_dword v[5:6], v1, off
+; GFX10-NEXT:    global_store_dword v[7:8], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: ive_store_div:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v9, 0x3ff, v31
+; GFX9-NEXT:    v_lshlrev_b32_e32 v9, 2, v9
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v9
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v9
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v9, v[0:1], off
+; GFX9-NEXT:    global_load_dword v10, v[2:3], off
+; GFX9-NEXT:    s_movk_i32 s4, 0xff
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 8, v4
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v9
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_sdwa v2, v10, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_e32 v0, v2, v0
+; GFX9-NEXT:    v_alignbit_b32 v2, v1, v10, 16
+; GFX9-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    global_store_dword v[5:6], v0, off
+; GFX9-NEXT:    global_store_dword v[7:8], v2, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <4 x i8>, ptr addrspace(1) %in0, i32 %tid
+  %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %in1, i32 %tid
+  %vec0 = load <4 x i8>, ptr addrspace(1) %gep0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 6, i32 7, i32 0, i32 2>
+  %vecins = insertelement <4 x i8> %shuffle0_0, i8 %elt, i32 1
+  store <4 x i8> %vecins, ptr addrspace(1) %out0
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out1
+  ret void
+}
+
+
+define hidden void @lhsr_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0, ptr addrspace(1) %out1) {
+; GFX10-LABEL: lhsr_store_div:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    global_load_dword v4, v[0:1], off
+; GFX10-NEXT:    global_load_dword v9, v[2:3], off
+; GFX10-NEXT:    v_mov_b32_e32 v0, 26
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_lshrrev_b16 v1, 1, v4
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v0, v0, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_lshrrev_b32_e32 v2, 25, v9
+; GFX10-NEXT:    v_lshrrev_b32_e32 v3, 26, v4
+; GFX10-NEXT:    v_and_b32_e32 v1, 0x7f00, v1
+; GFX10-NEXT:    v_or_b32_e32 v0, v2, v0
+; GFX10-NEXT:    v_or_b32_sdwa v1, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT:    v_perm_b32 v1, v9, v4, 0x1030707
+; GFX10-NEXT:    global_store_dword v[5:6], v0, off
+; GFX10-NEXT:    global_store_dword v[7:8], v1, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: lhsr_store_div:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    global_load_dword v9, v[2:3], off
+; GFX9-NEXT:    v_mov_b32_e32 v0, 26
+; GFX9-NEXT:    s_mov_b32 s4, 0x1030707
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_lshrrev_b16_e32 v3, 1, v4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_sdwa v0, v0, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 25, v9
+; GFX9-NEXT:    v_perm_b32 v1, v9, v4, s4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v4, 26, v4
+; GFX9-NEXT:    v_or_b32_e32 v0, v2, v0
+; GFX9-NEXT:    v_and_b32_e32 v2, 0x7f00, v3
+; GFX9-NEXT:    v_or_b32_sdwa v2, v4, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    global_store_dword v[5:6], v0, off
+; GFX9-NEXT:    global_store_dword v[7:8], v1, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <4 x i8>, ptr addrspace(1) %in0, i32 %tid
+  %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %in1, i32 %tid
+  %vec0 = load <4 x i8>, ptr addrspace(1) %gep0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 7, i32 7, i32 3, i32 1>
+  %vecins = lshr <4 x i8> %shuffle0_0, <i8 1, i8 2, i8 2, i8 1>
+  store <4 x i8> %vecins, ptr addrspace(1) %out0
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out1
+  ret void
+}
+
+
+define hidden void @mul_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0, ptr addrspace(1) %out1) {
+; GFX10-LABEL: mul_store_div:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    global_load_dword v4, v[0:1], off
+; GFX10-NEXT:    global_load_dword v9, v[2:3], off
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_lshrrev_b16 v0, 8, v4
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 24, v9
+; GFX10-NEXT:    v_lshrrev_b16 v2, 8, v9
+; GFX10-NEXT:    v_lshrrev_b32_e32 v3, 16, v9
+; GFX10-NEXT:    v_mul_lo_u16 v0, v0, v2
+; GFX10-NEXT:    v_mul_lo_u16 v1, v3, v1
+; GFX10-NEXT:    v_mul_lo_u16 v2, v4, v9
+; GFX10-NEXT:    v_mul_lo_u16 v3, v9, v3
+; GFX10-NEXT:    v_lshlrev_b16 v0, 8, v0
+; GFX10-NEXT:    v_lshlrev_b16 v1, 8, v1
+; GFX10-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v1, v3, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT:    v_perm_b32 v1, v4, v9, 0x2000504
+; GFX10-NEXT:    global_store_dword v[5:6], v0, off
+; GFX10-NEXT:    global_store_dword v[7:8], v1, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: mul_store_div:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v4, v[2:3], off
+; GFX9-NEXT:    global_load_dword v9, v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x2000504
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_mul_lo_u16_e32 v2, v9, v4
+; GFX9-NEXT:    v_mul_lo_u16_sdwa v3, v9, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1
+; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_mul_lo_u16_sdwa v3, v0, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3
+; GFX9-NEXT:    v_mul_lo_u16_e32 v0, v4, v0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    v_perm_b32 v1, v9, v4, s4
+; GFX9-NEXT:    global_store_dword v[5:6], v0, off
+; GFX9-NEXT:    global_store_dword v[7:8], v1, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <4 x i8>, ptr addrspace(1) %in0, i32 %tid
+  %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %in1, i32 %tid
+  %vec0 = load <4 x i8>, ptr addrspace(1) %gep0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 0, i32 1, i32 4, i32 6>
+  %vecins = mul <4 x i8> %shuffle0_0, %vec1
+  store <4 x i8> %vecins, ptr addrspace(1) %out0
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out1
+  ret void
+}
+
+
+define hidden void @or_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0, ptr addrspace(1) %out1) {
+; GFX10-LABEL: or_store_div:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    global_load_dword v4, v[2:3], off
+; GFX10-NEXT:    global_load_dword v9, v[0:1], off
+; GFX10-NEXT:    v_mov_b32_e32 v0, 16
+; GFX10-NEXT:    v_mov_b32_e32 v2, 0x102
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_lshlrev_b16 v1, 8, v4
+; GFX10-NEXT:    v_lshrrev_b32_sdwa v0, v0, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_or_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_e32 v1, 0x201, v1
+; GFX10-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT:    v_perm_b32 v1, v9, v4, 0x2010005
+; GFX10-NEXT:    global_store_dword v[5:6], v0, off
+; GFX10-NEXT:    global_store_dword v[7:8], v1, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: or_store_div:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v2, v[2:3], off
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v0, v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x2010005
+; GFX9-NEXT:    s_movk_i32 s5, 0x102
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 8, v2
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 8, v1
+; GFX9-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, s5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v4, v0, v2, s4
+; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_e32 v0, 0x201, v0
+; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    global_store_dword v[5:6], v0, off
+; GFX9-NEXT:    global_store_dword v[7:8], v4, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <4 x i8>, ptr addrspace(1) %in0, i32 %tid
+  %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %in1, i32 %tid
+  %vec0 = load <4 x i8>, ptr addrspace(1) %gep0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 1, i32 4, i32 5, i32 6>
+  %vecins = or <4 x i8> %shuffle0_0, <i8 1, i8 2, i8 2, i8 1>
+  store <4 x i8> %vecins, ptr addrspace(1) %out0
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out1
+  ret void
+}
+
+define hidden void @sdiv_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0, ptr addrspace(1) %out1) {
+; GFX10-LABEL: sdiv_store_div:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    global_load_dword v4, v[2:3], off
+; GFX10-NEXT:    global_load_dword v9, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_bfe_i32 v0, v4, 0, 8
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_ashrrev_i32_e32 v2, 24, v9
+; GFX10-NEXT:    v_bfe_i32 v3, v4, 8, 8
+; GFX10-NEXT:    v_bfe_i32 v1, v9, 16, 8
+; GFX10-NEXT:    v_bfe_i32 v10, v4, 16, 8
+; GFX10-NEXT:    v_cvt_f32_i32_e32 v13, v0
+; GFX10-NEXT:    v_ashrrev_i32_e32 v11, 24, v4
+; GFX10-NEXT:    v_xor_b32_e32 v15, v2, v3
+; GFX10-NEXT:    v_cvt_f32_i32_e32 v3, v3
+; GFX10-NEXT:    v_xor_b32_e32 v12, v1, v0
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v16, v13
+; GFX10-NEXT:    v_cvt_f32_i32_e32 v14, v1
+; GFX10-NEXT:    v_xor_b32_e32 v1, v1, v10
+; GFX10-NEXT:    v_cvt_f32_i32_e32 v10, v10
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v17, v3
+; GFX10-NEXT:    v_xor_b32_e32 v0, v0, v11
+; GFX10-NEXT:    v_cvt_f32_i32_e32 v11, v11
+; GFX10-NEXT:    v_cvt_f32_i32_e32 v2, v2
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v18, v10
+; GFX10-NEXT:    v_ashrrev_i32_e32 v12, 30, v12
+; GFX10-NEXT:    v_mul_f32_e32 v16, v14, v16
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v19, v11
+; GFX10-NEXT:    v_ashrrev_i32_e32 v15, 30, v15
+; GFX10-NEXT:    v_ashrrev_i32_e32 v1, 30, v1
+; GFX10-NEXT:    v_mul_f32_e32 v17, v2, v17
+; GFX10-NEXT:    v_trunc_f32_e32 v16, v16
+; GFX10-NEXT:    v_or_b32_e32 v12, 1, v12
+; GFX10-NEXT:    v_or_b32_e32 v15, 1, v15
+; GFX10-NEXT:    v_mul_f32_e32 v18, v14, v18
+; GFX10-NEXT:    v_trunc_f32_e32 v17, v17
+; GFX10-NEXT:    v_mad_f32 v20, -v16, v13, v14
+; GFX10-NEXT:    v_mul_f32_e32 v19, v13, v19
+; GFX10-NEXT:    v_ashrrev_i32_e32 v0, 30, v0
+; GFX10-NEXT:    v_trunc_f32_e32 v18, v18
+; GFX10-NEXT:    v_mad_f32 v2, -v17, v3, v2
+; GFX10-NEXT:    v_cmp_ge_f32_e64 vcc_lo, |v20|, |v13|
+; GFX10-NEXT:    v_trunc_f32_e32 v19, v19
+; GFX10-NEXT:    v_or_b32_e32 v1, 1, v1
+; GFX10-NEXT:    v_mad_f32 v14, -v18, v10, v14
+; GFX10-NEXT:    v_or_b32_e32 v0, 1, v0
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, 0, v12, vcc_lo
+; GFX10-NEXT:    v_cmp_ge_f32_e64 vcc_lo, |v2|, |v3|
+; GFX10-NEXT:    v_mad_f32 v21, -v19, v11, v13
+; GFX10-NEXT:    v_cvt_i32_f32_e32 v16, v16
+; GFX10-NEXT:    v_cvt_i32_f32_e32 v17, v17
+; GFX10-NEXT:    v_cvt_i32_f32_e32 v18, v18
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0, v15, vcc_lo
+; GFX10-NEXT:    v_cmp_ge_f32_e64 vcc_lo, |v14|, |v10|
+; GFX10-NEXT:    v_cvt_i32_f32_e32 v19, v19
+; GFX10-NEXT:    v_add_nc_u32_e32 v3, v16, v12
+; GFX10-NEXT:    v_add_nc_u32_sdwa v2, v17, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_ge_f32_e64 vcc_lo, |v21|, |v11|
+; GFX10-NEXT:    v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_add_nc_u32_e32 v1, v18, v1
+; GFX10-NEXT:    v_cndmask_b32_e32 v0, 0, v0, vcc_lo
+; GFX10-NEXT:    v_add_nc_u32_sdwa v0, v19, v0 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v0, v1, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_perm_b32 v1, v9, v4, 0x60706
+; GFX10-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT:    global_store_dword v[5:6], v0, off
+; GFX10-NEXT:    global_store_dword v[7:8], v1, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: sdiv_store_div:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v4, v[2:3], off
+; GFX9-NEXT:    global_load_dword v9, v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x60706
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_bfe_i32 v1, v4, 0, 8
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v0, v9, v4, s4
+; GFX9-NEXT:    v_bfe_i32 v2, v9, 16, 8
+; GFX9-NEXT:    v_ashrrev_i32_e32 v3, 24, v9
+; GFX9-NEXT:    v_bfe_i32 v9, v4, 8, 8
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v12, v1
+; GFX9-NEXT:    v_bfe_i32 v10, v4, 16, 8
+; GFX9-NEXT:    v_ashrrev_i32_e32 v4, 24, v4
+; GFX9-NEXT:    v_xor_b32_e32 v14, v3, v9
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v9, v9
+; GFX9-NEXT:    v_xor_b32_e32 v11, v2, v1
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v13, v2
+; GFX9-NEXT:    v_xor_b32_e32 v2, v2, v10
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v10, v10
+; GFX9-NEXT:    v_xor_b32_e32 v1, v1, v4
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v4, v4
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v15, v12
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v3, v3
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v16, v9
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v17, v10
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v18, v4
+; GFX9-NEXT:    v_mul_f32_e32 v15, v13, v15
+; GFX9-NEXT:    v_mul_f32_e32 v16, v3, v16
+; GFX9-NEXT:    v_trunc_f32_e32 v15, v15
+; GFX9-NEXT:    v_ashrrev_i32_e32 v11, 30, v11
+; GFX9-NEXT:    v_mul_f32_e32 v17, v13, v17
+; GFX9-NEXT:    v_mul_f32_e32 v18, v12, v18
+; GFX9-NEXT:    v_trunc_f32_e32 v16, v16
+; GFX9-NEXT:    v_mad_f32 v19, -v15, v12, v13
+; GFX9-NEXT:    v_ashrrev_i32_e32 v14, 30, v14
+; GFX9-NEXT:    v_or_b32_e32 v11, 1, v11
+; GFX9-NEXT:    v_trunc_f32_e32 v17, v17
+; GFX9-NEXT:    v_trunc_f32_e32 v18, v18
+; GFX9-NEXT:    v_mad_f32 v3, -v16, v9, v3
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v19|, |v12|
+; GFX9-NEXT:    v_ashrrev_i32_e32 v2, 30, v2
+; GFX9-NEXT:    v_or_b32_e32 v14, 1, v14
+; GFX9-NEXT:    v_cvt_i32_f32_e32 v15, v15
+; GFX9-NEXT:    v_cvt_i32_f32_e32 v16, v16
+; GFX9-NEXT:    v_mad_f32 v13, -v17, v10, v13
+; GFX9-NEXT:    v_cvt_i32_f32_e32 v17, v17
+; GFX9-NEXT:    v_mad_f32 v20, -v18, v4, v12
+; GFX9-NEXT:    v_cvt_i32_f32_e32 v18, v18
+; GFX9-NEXT:    v_cndmask_b32_e32 v11, 0, v11, vcc
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v3|, |v9|
+; GFX9-NEXT:    v_ashrrev_i32_e32 v1, 30, v1
+; GFX9-NEXT:    v_or_b32_e32 v2, 1, v2
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, 0, v14, vcc
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v13|, |v10|
+; GFX9-NEXT:    v_or_b32_e32 v1, 1, v1
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v20|, |v4|
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, 0, v1, vcc
+; GFX9-NEXT:    v_add_u32_e32 v4, v15, v11
+; GFX9-NEXT:    v_add_u32_sdwa v3, v16, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_add_u32_e32 v2, v17, v2
+; GFX9-NEXT:    v_add_u32_sdwa v1, v18, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    global_store_dword v[5:6], v1, off
+; GFX9-NEXT:    global_store_dword v[7:8], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <4 x i8>, ptr addrspace(1) %in0, i32 %tid
+  %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %in1, i32 %tid
+  %vec0 = load <4 x i8>, ptr addrspace(1) %gep0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 2, i32 3, i32 2, i32 4>
+  %vecins = sdiv <4 x i8> %shuffle0_0, %vec1
+  store <4 x i8> %vecins, ptr addrspace(1) %out0
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out1
+  ret void
+}
+
+
+define hidden void @sext_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0, ptr addrspace(1) %out1) {
+; GFX10-LABEL: sext_store_div:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    global_load_dword v4, v[2:3], off
+; GFX10-NEXT:    global_load_dword v9, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 16, v4
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 16, v9
+; GFX10-NEXT:    v_ashrrev_i16 v2, 8, v4
+; GFX10-NEXT:    v_ashrrev_i16 v0, 8, v0
+; GFX10-NEXT:    v_ashrrev_i16 v3, 8, v1
+; GFX10-NEXT:    v_perm_b32 v1, v0, v2, 0x5040100
+; GFX10-NEXT:    v_perm_b32 v0, v3, v3, 0x5040100
+; GFX10-NEXT:    v_perm_b32 v2, v9, v4, 0x3010707
+; GFX10-NEXT:    global_store_dwordx2 v[7:8], v[0:1], off
+; GFX10-NEXT:    global_store_dword v[5:6], v2, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: sext_store_div:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    global_load_dword v9, v[2:3], off
+; GFX9-NEXT:    v_mov_b32_e32 v0, 8
+; GFX9-NEXT:    s_mov_b32 s5, 0x5040100
+; GFX9-NEXT:    s_mov_b32 s4, 0x3010707
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_ashrrev_i16_e32 v1, 8, v9
+; GFX9-NEXT:    v_ashrrev_i16_sdwa v3, v0, v9 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_ashrrev_i16_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT:    v_perm_b32 v1, v3, v1, s5
+; GFX9-NEXT:    v_perm_b32 v0, v0, v0, s5
+; GFX9-NEXT:    v_perm_b32 v2, v4, v9, s4
+; GFX9-NEXT:    global_store_dwordx2 v[7:8], v[0:1], off
+; GFX9-NEXT:    global_store_dword v[5:6], v2, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <4 x i8>, ptr addrspace(1) %in0, i32 %tid
+  %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %in1, i32 %tid
+  %vec0 = load <4 x i8>, ptr addrspace(1) %gep0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 3, i32 3, i32 5, i32 7>
+  %insvec = sext <4 x i8> %shuffle0_0 to <4 x i16>
+  store <4 x i16> %insvec, ptr addrspace(1) %out1
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0
+  ret void
+}
+
+
+define hidden void @shl_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0, ptr addrspace(1) %out1) {
+; GFX10-LABEL: shl_store_div:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    global_load_dword v4, v[0:1], off
+; GFX10-NEXT:    global_load_dword v9, v[2:3], off
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_lshlrev_b16 v0, 2, v4
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_lshlrev_b16 v1, 1, v9
+; GFX10-NEXT:    v_and_b32_e32 v2, 0xfffffc00, v0
+; GFX10-NEXT:    v_and_b32_e32 v3, 0xfe, v1
+; GFX10-NEXT:    v_and_b32_e32 v1, 0xfffffe00, v1
+; GFX10-NEXT:    v_and_b32_e32 v0, 0xfc, v0
+; GFX10-NEXT:    v_or_b32_e32 v2, v3, v2
+; GFX10-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_perm_b32 v1, v9, v4, 0x5000104
+; GFX10-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT:    global_store_dword v[5:6], v0, off
+; GFX10-NEXT:    global_store_dword v[7:8], v1, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: shl_store_div:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    global_load_dword v9, v[2:3], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x5000104
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v1, 2, v4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 1, v9
+; GFX9-NEXT:    v_perm_b32 v0, v9, v4, s4
+; GFX9-NEXT:    v_and_b32_e32 v3, 0xfffffc00, v1
+; GFX9-NEXT:    v_and_b32_e32 v4, 0xfe, v2
+; GFX9-NEXT:    v_and_b32_e32 v2, 0xfffffe00, v2
+; GFX9-NEXT:    v_and_b32_e32 v1, 0xfc, v1
+; GFX9-NEXT:    v_or_b32_e32 v3, v4, v3
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    global_store_dword v[5:6], v1, off
+; GFX9-NEXT:    global_store_dword v[7:8], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <4 x i8>, ptr addrspace(1) %in0, i32 %tid
+  %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %in1, i32 %tid
+  %vec0 = load <4 x i8>, ptr addrspace(1) %gep0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 4, i32 1, i32 0, i32 5>
+  %vecins = shl <4 x i8> %shuffle0_0, <i8 1, i8 2, i8 2, i8 1>
+  store <4 x i8> %vecins, ptr addrspace(1) %out0
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out1
+  ret void
+}
+
+
+define hidden void @sitofp_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0, ptr addrspace(1) %out1) {
+; GFX10-LABEL: sitofp_store_div:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    global_load_dword v4, v[2:3], off
+; GFX10-NEXT:    global_load_dword v9, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 16, v4
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 16, v9
+; GFX10-NEXT:    v_ashrrev_i16 v2, 8, v9
+; GFX10-NEXT:    v_ashrrev_i16 v3, 8, v4
+; GFX10-NEXT:    v_perm_b32 v4, v4, v9, 0x6010205
+; GFX10-NEXT:    v_bfe_i32 v10, v0, 0, 8
+; GFX10-NEXT:    v_bfe_i32 v1, v1, 0, 8
+; GFX10-NEXT:    v_cvt_f32_i32_sdwa v2, sext(v2) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX10-NEXT:    v_cvt_f32_i32_sdwa v0, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX10-NEXT:    v_cvt_f32_i32_sdwa v3, sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX10-NEXT:    v_cvt_f32_i32_sdwa v1, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX10-NEXT:    global_store_dwordx4 v[7:8], v[0:3], off
+; GFX10-NEXT:    global_store_dword v[5:6], v4, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: sitofp_store_div:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v9, v[0:1], off
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x6010205
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v9
+; GFX9-NEXT:    v_ashrrev_i16_e32 v1, 8, v9
+; GFX9-NEXT:    v_bfe_i32 v10, v0, 0, 8
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v2, 16, v4
+; GFX9-NEXT:    v_ashrrev_i16_e32 v3, 8, v4
+; GFX9-NEXT:    v_bfe_i32 v11, v2, 0, 8
+; GFX9-NEXT:    v_cvt_f32_i32_sdwa v2, sext(v1) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX9-NEXT:    v_cvt_f32_i32_sdwa v0, sext(v3) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX9-NEXT:    v_cvt_f32_i32_sdwa v3, sext(v11) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX9-NEXT:    v_cvt_f32_i32_sdwa v1, sext(v10) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0
+; GFX9-NEXT:    v_perm_b32 v4, v4, v9, s4
+; GFX9-NEXT:    global_store_dwordx4 v[7:8], v[0:3], off
+; GFX9-NEXT:    global_store_dword v[5:6], v4, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <4 x i8>, ptr addrspace(1) %in0, i32 %tid
+  %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %in1, i32 %tid
+  %vec0 = load <4 x i8>, ptr addrspace(1) %gep0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 5, i32 2, i32 1, i32 6>
+  %insvec = sitofp <4 x i8> %shuffle0_0 to <4 x float>
+  store <4 x float> %insvec, ptr addrspace(1) %out1
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0
+  ret void
+}
+
+
+define hidden void @srem_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0, ptr addrspace(1) %out1) {
+; GFX10-LABEL: srem_store_div:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    global_load_dword v4, v[2:3], off
+; GFX10-NEXT:    global_load_dword v9, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_bfe_i32 v1, v4, 0, 8
+; GFX10-NEXT:    v_bfe_i32 v2, v4, 16, 8
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_ashrrev_i32_e32 v10, 24, v9
+; GFX10-NEXT:    v_bfe_i32 v11, v4, 8, 8
+; GFX10-NEXT:    v_ashrrev_i32_e32 v12, 24, v4
+; GFX10-NEXT:    v_bfe_i32 v13, v9, 16, 8
+; GFX10-NEXT:    v_xor_b32_e32 v14, v2, v1
+; GFX10-NEXT:    v_cvt_f32_i32_e32 v1, v1
+; GFX10-NEXT:    v_xor_b32_e32 v16, v10, v11
+; GFX10-NEXT:    v_cvt_f32_i32_e32 v11, v11
+; GFX10-NEXT:    v_cvt_f32_i32_e32 v15, v2
+; GFX10-NEXT:    v_cvt_f32_i32_e32 v10, v10
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v18, v1
+; GFX10-NEXT:    v_cvt_f32_i32_e32 v17, v12
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v19, v11
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v20, v15
+; GFX10-NEXT:    v_xor_b32_e32 v2, v12, v2
+; GFX10-NEXT:    v_xor_b32_e32 v12, v13, v12
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v21, v17
+; GFX10-NEXT:    v_ashrrev_i32_e32 v14, 30, v14
+; GFX10-NEXT:    v_cvt_f32_i32_e32 v13, v13
+; GFX10-NEXT:    v_ashrrev_i32_e32 v16, 30, v16
+; GFX10-NEXT:    v_mul_f32_e32 v18, v15, v18
+; GFX10-NEXT:    v_ashrrev_i32_e32 v2, 30, v2
+; GFX10-NEXT:    v_mul_f32_e32 v19, v10, v19
+; GFX10-NEXT:    v_mul_f32_e32 v20, v17, v20
+; GFX10-NEXT:    v_or_b32_e32 v14, 1, v14
+; GFX10-NEXT:    v_trunc_f32_e32 v18, v18
+; GFX10-NEXT:    v_mul_f32_e32 v21, v13, v21
+; GFX10-NEXT:    v_trunc_f32_e32 v19, v19
+; GFX10-NEXT:    v_trunc_f32_e32 v20, v20
+; GFX10-NEXT:    v_or_b32_e32 v16, 1, v16
+; GFX10-NEXT:    v_mad_f32 v22, -v18, v1, v15
+; GFX10-NEXT:    v_trunc_f32_e32 v21, v21
+; GFX10-NEXT:    v_mad_f32 v10, -v19, v11, v10
+; GFX10-NEXT:    v_mad_f32 v23, -v20, v15, v17
+; GFX10-NEXT:    v_ashrrev_i32_e32 v12, 30, v12
+; GFX10-NEXT:    v_cmp_ge_f32_e64 vcc_lo, |v22|, |v1|
+; GFX10-NEXT:    v_or_b32_e32 v2, 1, v2
+; GFX10-NEXT:    v_mad_f32 v13, -v21, v17, v13
+; GFX10-NEXT:    v_cvt_i32_f32_e32 v18, v18
+; GFX10-NEXT:    v_or_b32_e32 v12, 1, v12
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, 0, v14, vcc_lo
+; GFX10-NEXT:    v_cmp_ge_f32_e64 vcc_lo, |v10|, |v11|
+; GFX10-NEXT:    v_cvt_i32_f32_e32 v19, v19
+; GFX10-NEXT:    v_cvt_i32_f32_e32 v20, v20
+; GFX10-NEXT:    v_cvt_i32_f32_e32 v21, v21
+; GFX10-NEXT:    v_lshrrev_b32_e32 v0, 16, v4
+; GFX10-NEXT:    v_cndmask_b32_e32 v10, 0, v16, vcc_lo
+; GFX10-NEXT:    v_cmp_ge_f32_e64 vcc_lo, |v23|, |v15|
+; GFX10-NEXT:    v_lshrrev_b32_e32 v3, 8, v4
+; GFX10-NEXT:    v_add_nc_u32_e32 v1, v18, v1
+; GFX10-NEXT:    v_add_nc_u32_e32 v10, v19, v10
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, 0, v2, vcc_lo
+; GFX10-NEXT:    v_cmp_ge_f32_e64 vcc_lo, |v13|, |v17|
+; GFX10-NEXT:    v_mul_lo_u32 v1, v1, v4
+; GFX10-NEXT:    v_mul_lo_u32 v3, v10, v3
+; GFX10-NEXT:    v_add_nc_u32_e32 v2, v20, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, 0, v12, vcc_lo
+; GFX10-NEXT:    v_lshrrev_b32_e32 v12, 24, v4
+; GFX10-NEXT:    v_mul_lo_u32 v2, v2, v0
+; GFX10-NEXT:    v_add_nc_u32_e32 v11, v21, v11
+; GFX10-NEXT:    v_sub_nc_u32_e32 v0, v0, v1
+; GFX10-NEXT:    v_sub_nc_u32_sdwa v1, v9, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX10-NEXT:    v_mul_lo_u32 v10, v11, v12
+; GFX10-NEXT:    v_sub_nc_u32_e32 v2, v12, v2
+; GFX10-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_sub_nc_u32_sdwa v3, v9, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v1, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT:    v_perm_b32 v1, v4, v9, 0x2070306
+; GFX10-NEXT:    global_store_dword v[5:6], v0, off
+; GFX10-NEXT:    global_store_dword v[7:8], v1, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: srem_store_div:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v4, v[2:3], off
+; GFX9-NEXT:    global_load_dword v9, v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x2070306
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_bfe_i32 v2, v4, 0, 8
+; GFX9-NEXT:    v_bfe_i32 v3, v4, 16, 8
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_ashrrev_i32_e32 v11, 24, v9
+; GFX9-NEXT:    v_bfe_i32 v12, v4, 8, 8
+; GFX9-NEXT:    v_xor_b32_e32 v16, v3, v2
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v2, v2
+; GFX9-NEXT:    v_ashrrev_i32_e32 v13, 24, v4
+; GFX9-NEXT:    v_xor_b32_e32 v18, v11, v12
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v12, v12
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v17, v3
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v19, v13
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v20, v2
+; GFX9-NEXT:    v_bfe_i32 v15, v9, 16, 8
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v11, v11
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v21, v12
+; GFX9-NEXT:    v_xor_b32_e32 v3, v13, v3
+; GFX9-NEXT:    v_xor_b32_e32 v13, v15, v13
+; GFX9-NEXT:    v_cvt_f32_i32_e32 v15, v15
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v22, v17
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v23, v19
+; GFX9-NEXT:    v_mul_f32_e32 v20, v17, v20
+; GFX9-NEXT:    v_mul_f32_e32 v21, v11, v21
+; GFX9-NEXT:    v_trunc_f32_e32 v20, v20
+; GFX9-NEXT:    v_ashrrev_i32_e32 v16, 30, v16
+; GFX9-NEXT:    v_mul_f32_e32 v22, v19, v22
+; GFX9-NEXT:    v_mul_f32_e32 v23, v15, v23
+; GFX9-NEXT:    v_trunc_f32_e32 v21, v21
+; GFX9-NEXT:    v_mad_f32 v24, -v20, v2, v17
+; GFX9-NEXT:    v_ashrrev_i32_e32 v18, 30, v18
+; GFX9-NEXT:    v_or_b32_e32 v16, 1, v16
+; GFX9-NEXT:    v_trunc_f32_e32 v22, v22
+; GFX9-NEXT:    v_trunc_f32_e32 v23, v23
+; GFX9-NEXT:    v_mad_f32 v11, -v21, v12, v11
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v24|, |v2|
+; GFX9-NEXT:    v_ashrrev_i32_e32 v3, 30, v3
+; GFX9-NEXT:    v_or_b32_e32 v18, 1, v18
+; GFX9-NEXT:    v_cvt_i32_f32_e32 v20, v20
+; GFX9-NEXT:    v_cvt_i32_f32_e32 v21, v21
+; GFX9-NEXT:    v_mad_f32 v25, -v22, v17, v19
+; GFX9-NEXT:    v_cvt_i32_f32_e32 v22, v22
+; GFX9-NEXT:    v_mad_f32 v15, -v23, v19, v15
+; GFX9-NEXT:    v_cvt_i32_f32_e32 v23, v23
+; GFX9-NEXT:    v_cndmask_b32_e32 v2, 0, v16, vcc
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v11|, |v12|
+; GFX9-NEXT:    v_ashrrev_i32_e32 v13, 30, v13
+; GFX9-NEXT:    v_or_b32_e32 v3, 1, v3
+; GFX9-NEXT:    v_cndmask_b32_e32 v11, 0, v18, vcc
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v25|, |v17|
+; GFX9-NEXT:    v_or_b32_e32 v13, 1, v13
+; GFX9-NEXT:    v_cndmask_b32_e32 v3, 0, v3, vcc
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v15|, |v19|
+; GFX9-NEXT:    v_cndmask_b32_e32 v12, 0, v13, vcc
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 8, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v14, 24, v4
+; GFX9-NEXT:    v_add_u32_e32 v2, v20, v2
+; GFX9-NEXT:    v_add_u32_e32 v11, v21, v11
+; GFX9-NEXT:    v_add_u32_e32 v3, v22, v3
+; GFX9-NEXT:    v_add_u32_e32 v12, v23, v12
+; GFX9-NEXT:    v_perm_b32 v1, v4, v9, s4
+; GFX9-NEXT:    v_mul_lo_u32 v2, v2, v4
+; GFX9-NEXT:    v_mul_lo_u32 v4, v11, v10
+; GFX9-NEXT:    v_mul_lo_u32 v3, v3, v0
+; GFX9-NEXT:    v_mul_lo_u32 v10, v12, v14
+; GFX9-NEXT:    v_sub_u32_e32 v0, v0, v2
+; GFX9-NEXT:    v_sub_u32_sdwa v2, v9, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_sub_u32_e32 v3, v14, v3
+; GFX9-NEXT:    v_sub_u32_sdwa v4, v9, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    global_store_dword v[5:6], v0, off
+; GFX9-NEXT:    global_store_dword v[7:8], v1, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <4 x i8>, ptr addrspace(1) %in0, i32 %tid
+  %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %in1, i32 %tid
+  %vec0 = load <4 x i8>, ptr addrspace(1) %gep0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 6, i32 3, i32 7, i32 2>
+  %vecins = srem <4 x i8> %shuffle0_0, %vec1
+  store <4 x i8> %vecins, ptr addrspace(1) %out0
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out1
+  ret void
+}
+
+
+define hidden void @sub_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0, ptr addrspace(1) %out1) {
+; GFX10-LABEL: sub_store_div:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    global_load_dword v2, v[2:3], off
+; GFX10-NEXT:    global_load_dword v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX10-NEXT:    v_lshrrev_b16 v3, 8, v2
+; GFX10-NEXT:    v_lshrrev_b32_e32 v4, 24, v2
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_sub_nc_u16 v3, v0, v3
+; GFX10-NEXT:    v_sub_nc_u16 v9, v1, v4
+; GFX10-NEXT:    v_sub_nc_u16 v10, v4, v2
+; GFX10-NEXT:    v_sub_nc_u16 v1, v4, v1
+; GFX10-NEXT:    v_perm_b32 v0, v2, v0, 0x6070007
+; GFX10-NEXT:    v_lshlrev_b16 v3, 8, v3
+; GFX10-NEXT:    v_lshlrev_b16 v4, 8, v9
+; GFX10-NEXT:    v_or_b32_sdwa v3, v10, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v1, v1, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT:    global_store_dword v[5:6], v1, off
+; GFX10-NEXT:    global_store_dword v[7:8], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: sub_store_div:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    global_load_dword v2, v[2:3], off
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    global_load_dword v0, v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x6070007
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_lshrrev_b32_e32 v1, 16, v2
+; GFX9-NEXT:    v_lshrrev_b32_e32 v3, 24, v2
+; GFX9-NEXT:    v_sub_u16_sdwa v9, v1, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v4, v2, v0, s4
+; GFX9-NEXT:    v_sub_u16_sdwa v0, v0, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
+; GFX9-NEXT:    v_sub_u16_e32 v2, v3, v2
+; GFX9-NEXT:    v_sub_u16_e32 v1, v3, v1
+; GFX9-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    global_store_dword v[5:6], v0, off
+; GFX9-NEXT:    global_store_dword v[7:8], v4, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <4 x i8>, ptr addrspace(1) %in0, i32 %tid
+  %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %in1, i32 %tid
+  %vec0 = load <4 x i8>, ptr addrspace(1) %gep0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 7, i32 0, i32 7, i32 6>
+  %vecins = sub <4 x i8> %shuffle0_0, %vec1
+  store <4 x i8> %vecins, ptr addrspace(1) %out0
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out1
+  ret void
+}
+
+
+define hidden void @sv_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0, ptr addrspace(1) %out1, ptr addrspace(1) %out2) {
+; GFX10-LABEL: sv_store_div:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    global_load_dword v4, v[0:1], off
+; GFX10-NEXT:    global_load_dword v5, v[2:3], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_perm_b32 v0, v4, v5, 0x50705
+; GFX10-NEXT:    global_store_dword v[7:8], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: sv_store_div:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    global_load_dword v5, v[2:3], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x50705
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v0, v4, v5, s4
+; GFX9-NEXT:    global_store_dword v[7:8], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <4 x i8>, ptr addrspace(1) %in0, i32 %tid
+  %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %in1, i32 %tid
+  %vec0 = load <4 x i8>, ptr addrspace(1) %gep0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 1, i32 3, i32 1, i32 4>
+  %insvec = shufflevector <4 x i8> %shuffle0_0, <4 x i8> %vec1, <4 x i32> <i32 6, i32 3, i32 7, i32 0>
+  store <4 x i8> %insvec, ptr addrspace(1) %out1
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out1
+  ret void
+}
+
+
+define hidden void @trunc_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0, ptr addrspace(1) %out1) {
+; GFX10-LABEL: trunc_store_div:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    global_load_dword v4, v[0:1], off
+; GFX10-NEXT:    global_load_dword v9, v[2:3], off
+; GFX10-NEXT:    v_mov_b32_e32 v0, 1
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_and_b32_sdwa v1, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_and_b32_sdwa v0, v9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX10-NEXT:    v_lshlrev_b16 v1, 1, v1
+; GFX10-NEXT:    v_lshlrev_b16 v2, 2, v0
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    v_lshlrev_b16 v1, 3, v4
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX10-NEXT:    v_or_b32_e32 v0, v0, v1
+; GFX10-NEXT:    v_perm_b32 v1, v9, v4, 0x50205
+; GFX10-NEXT:    v_and_b32_e32 v0, 15, v0
+; GFX10-NEXT:    global_store_byte v[7:8], v0, off
+; GFX10-NEXT:    global_store_dword v[5:6], v1, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: trunc_store_div:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    global_load_dword v9, v[2:3], off
+; GFX9-NEXT:    v_mov_b32_e32 v0, 1
+; GFX9-NEXT:    s_mov_b32 s4, 0x50205
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_lshlrev_b16_e32 v3, 3, v4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_sdwa v2, v9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
+; GFX9-NEXT:    v_and_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_lshlrev_b16_e32 v0, 1, v0
+; GFX9-NEXT:    v_perm_b32 v1, v9, v4, s4
+; GFX9-NEXT:    v_lshlrev_b16_e32 v4, 2, v2
+; GFX9-NEXT:    v_or_b32_e32 v0, v2, v0
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v4
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v3
+; GFX9-NEXT:    v_and_b32_e32 v0, 15, v0
+; GFX9-NEXT:    global_store_byte v[7:8], v0, off
+; GFX9-NEXT:    global_store_dword v[5:6], v1, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <4 x i8>, ptr addrspace(1) %in0, i32 %tid
+  %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %in1, i32 %tid
+  %vec0 = load <4 x i8>, ptr addrspace(1) %gep0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 5, i32 2, i32 5, i32 0>
+  %insvec = trunc <4 x i8> %shuffle0_0 to <4 x i1>
+  store <4 x i1> %insvec, ptr addrspace(1) %out1
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0
+  ret void
+}
+
+define hidden void @udiv(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0, ptr addrspace(1) %out1) {
+; GFX10-LABEL: udiv:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    global_load_dword v2, v[2:3], off
+; GFX10-NEXT:    global_load_dword v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v1, v2
+; GFX10-NEXT:    v_cvt_f32_ubyte1_e32 v3, v2
+; GFX10-NEXT:    v_cvt_f32_ubyte3_e32 v9, v2
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_cvt_f32_ubyte3_e32 v14, v0
+; GFX10-NEXT:    v_cvt_f32_ubyte2_e32 v4, v2
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v10, v1
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v11, v3
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v13, v9
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v15, v0
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v12, v4
+; GFX10-NEXT:    v_perm_b32 v0, v0, v2, 0x40207
+; GFX10-NEXT:    v_mul_f32_e32 v10, v14, v10
+; GFX10-NEXT:    v_mul_f32_e32 v11, v4, v11
+; GFX10-NEXT:    v_mul_f32_e32 v13, v1, v13
+; GFX10-NEXT:    v_mul_f32_e32 v12, v15, v12
+; GFX10-NEXT:    v_trunc_f32_e32 v10, v10
+; GFX10-NEXT:    v_trunc_f32_e32 v11, v11
+; GFX10-NEXT:    v_trunc_f32_e32 v13, v13
+; GFX10-NEXT:    v_trunc_f32_e32 v12, v12
+; GFX10-NEXT:    v_mad_f32 v14, -v10, v1, v14
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v10, v10
+; GFX10-NEXT:    v_mad_f32 v16, -v11, v3, v4
+; GFX10-NEXT:    v_mad_f32 v17, -v13, v9, v1
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v11, v11
+; GFX10-NEXT:    v_cmp_ge_f32_e64 vcc_lo, |v14|, v1
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v13, v13
+; GFX10-NEXT:    v_mad_f32 v15, -v12, v4, v15
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v12, v12
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v10, vcc_lo
+; GFX10-NEXT:    v_cmp_ge_f32_e64 vcc_lo, |v16|, v3
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v11, vcc_lo
+; GFX10-NEXT:    v_cmp_ge_f32_e64 vcc_lo, |v17|, v9
+; GFX10-NEXT:    v_lshlrev_b16 v3, 8, v3
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, 0, v13, vcc_lo
+; GFX10-NEXT:    v_cmp_ge_f32_e64 vcc_lo, |v15|, v4
+; GFX10-NEXT:    v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_lshlrev_b16 v9, 8, v9
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, 0, v12, vcc_lo
+; GFX10-NEXT:    v_or_b32_sdwa v3, v4, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT:    global_store_dword v[5:6], v1, off
+; GFX10-NEXT:    global_store_dword v[7:8], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: udiv:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v4, v[2:3], off
+; GFX9-NEXT:    global_load_dword v9, v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x40207
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v2, v4
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v11, v2
+; GFX9-NEXT:    v_cvt_f32_ubyte1_e32 v3, v4
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v12, v3
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cvt_f32_ubyte3_e32 v1, v9
+; GFX9-NEXT:    v_cvt_f32_ubyte2_e32 v10, v4
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v13, v10
+; GFX9-NEXT:    v_mul_f32_e32 v11, v1, v11
+; GFX9-NEXT:    v_perm_b32 v0, v9, v4, s4
+; GFX9-NEXT:    v_cvt_f32_ubyte3_e32 v4, v4
+; GFX9-NEXT:    v_trunc_f32_e32 v11, v11
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v14, v4
+; GFX9-NEXT:    v_mul_f32_e32 v12, v10, v12
+; GFX9-NEXT:    v_mad_f32 v1, -v11, v2, v1
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v11, v11
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v9, v9
+; GFX9-NEXT:    v_trunc_f32_e32 v12, v12
+; GFX9-NEXT:    v_mul_f32_e32 v13, v9, v13
+; GFX9-NEXT:    v_mad_f32 v15, -v12, v3, v10
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v12, v12
+; GFX9-NEXT:    v_trunc_f32_e32 v13, v13
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v1|, v2
+; GFX9-NEXT:    v_mul_f32_e32 v14, v2, v14
+; GFX9-NEXT:    v_mad_f32 v9, -v13, v10, v9
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v13, v13
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v11, vcc
+; GFX9-NEXT:    v_trunc_f32_e32 v14, v14
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v15|, v3
+; GFX9-NEXT:    v_mad_f32 v16, -v14, v4, v2
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v14, v14
+; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, 0, v12, vcc
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v9|, v10
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v13, vcc
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v16|, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v4, vcc, 0, v14, vcc
+; GFX9-NEXT:    v_lshlrev_b16_e32 v2, 8, v2
+; GFX9-NEXT:    v_lshlrev_b16_e32 v4, 8, v4
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v2, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    global_store_dword v[5:6], v1, off
+; GFX9-NEXT:    global_store_dword v[7:8], v0, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <4 x i8>, ptr addrspace(1) %in0, i32 %tid
+  %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %in1, i32 %tid
+  %vec0 = load <4 x i8>, ptr addrspace(1) %gep0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 3, i32 6, i32 0, i32 4>
+  %vecins = udiv <4 x i8> %shuffle0_0, %vec1
+  store <4 x i8> %vecins, ptr addrspace(1) %out0
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out1
+  ret void
+}
+
+
+define hidden void @uitofp_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0, ptr addrspace(1) %out1) {
+; GFX10-LABEL: uitofp_store_div:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    global_load_dword v4, v[2:3], off
+; GFX10-NEXT:    global_load_dword v9, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_cvt_f32_ubyte1_e32 v3, v4
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_cvt_f32_ubyte2_e32 v2, v9
+; GFX10-NEXT:    v_cvt_f32_ubyte1_e32 v1, v9
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v0, v4
+; GFX10-NEXT:    v_perm_b32 v4, v4, v9, 0x5020104
+; GFX10-NEXT:    global_store_dwordx4 v[7:8], v[0:3], off
+; GFX10-NEXT:    global_store_dword v[5:6], v4, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: uitofp_store_div:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    global_load_dword v9, v[2:3], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x5020104
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_cvt_f32_ubyte2_e32 v2, v4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cvt_f32_ubyte1_e32 v3, v9
+; GFX9-NEXT:    v_cvt_f32_ubyte1_e32 v1, v4
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v0, v9
+; GFX9-NEXT:    v_perm_b32 v10, v9, v4, s4
+; GFX9-NEXT:    global_store_dwordx4 v[7:8], v[0:3], off
+; GFX9-NEXT:    global_store_dword v[5:6], v10, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <4 x i8>, ptr addrspace(1) %in0, i32 %tid
+  %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %in1, i32 %tid
+  %vec0 = load <4 x i8>, ptr addrspace(1) %gep0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 4, i32 1, i32 2, i32 5>
+  %insvec = uitofp <4 x i8> %shuffle0_0 to <4 x float>
+  store <4 x float> %insvec, ptr addrspace(1) %out1
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0
+  ret void
+}
+
+
+define hidden void @urem_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0, ptr addrspace(1) %out1) {
+; GFX10-LABEL: urem_store_div:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    global_load_dword v2, v[2:3], off
+; GFX10-NEXT:    global_load_dword v0, v[0:1], off
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_cvt_f32_ubyte0_e32 v1, v2
+; GFX10-NEXT:    v_cvt_f32_ubyte1_e32 v3, v2
+; GFX10-NEXT:    v_cvt_f32_ubyte2_e32 v4, v2
+; GFX10-NEXT:    v_cvt_f32_ubyte3_e32 v9, v2
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_cvt_f32_ubyte2_e32 v15, v0
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v10, v1
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v11, v3
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v12, v4
+; GFX10-NEXT:    v_rcp_iflag_f32_e32 v13, v9
+; GFX10-NEXT:    v_lshrrev_b32_e32 v14, 16, v2
+; GFX10-NEXT:    v_lshrrev_b32_e32 v16, 8, v2
+; GFX10-NEXT:    v_lshrrev_b32_e32 v17, 24, v2
+; GFX10-NEXT:    v_mul_f32_e32 v10, v3, v10
+; GFX10-NEXT:    v_mul_f32_e32 v11, v3, v11
+; GFX10-NEXT:    v_mul_f32_e32 v12, v3, v12
+; GFX10-NEXT:    v_mul_f32_e32 v13, v15, v13
+; GFX10-NEXT:    v_trunc_f32_e32 v10, v10
+; GFX10-NEXT:    v_trunc_f32_e32 v11, v11
+; GFX10-NEXT:    v_trunc_f32_e32 v12, v12
+; GFX10-NEXT:    v_trunc_f32_e32 v13, v13
+; GFX10-NEXT:    v_mad_f32 v18, -v10, v1, v3
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v10, v10
+; GFX10-NEXT:    v_mad_f32 v19, -v11, v3, v3
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v11, v11
+; GFX10-NEXT:    v_mad_f32 v20, -v12, v4, v3
+; GFX10-NEXT:    v_cmp_ge_f32_e64 vcc_lo, |v18|, v1
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v12, v12
+; GFX10-NEXT:    v_mad_f32 v15, -v13, v9, v15
+; GFX10-NEXT:    v_cvt_u32_f32_e32 v13, v13
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v10, vcc_lo
+; GFX10-NEXT:    v_cmp_ge_f32_e64 vcc_lo, |v19|, v3
+; GFX10-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v11, vcc_lo
+; GFX10-NEXT:    v_cmp_ge_f32_e64 vcc_lo, |v20|, v4
+; GFX10-NEXT:    v_mul_lo_u32 v3, v3, v16
+; GFX10-NEXT:    v_sub_nc_u32_e32 v1, v16, v1
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, 0, v12, vcc_lo
+; GFX10-NEXT:    v_cmp_ge_f32_e64 vcc_lo, |v15|, v9
+; GFX10-NEXT:    v_mul_lo_u32 v4, v4, v14
+; GFX10-NEXT:    v_sub_nc_u32_sdwa v3, v16, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, 0, v13, vcc_lo
+; GFX10-NEXT:    v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_mul_lo_u32 v9, v9, v17
+; GFX10-NEXT:    v_sub_nc_u32_e32 v4, v16, v4
+; GFX10-NEXT:    v_sub_nc_u32_sdwa v9, v0, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT:    v_perm_b32 v0, v2, v0, 0x2050505
+; GFX10-NEXT:    v_or_b32_sdwa v3, v4, v9 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT:    global_store_dword v[5:6], v1, off
+; GFX10-NEXT:    global_store_dword v[7:8], v0, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: urem_store_div:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v4, v[2:3], off
+; GFX9-NEXT:    global_load_dword v9, v[0:1], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x2050505
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_cvt_f32_ubyte0_e32 v2, v4
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v15, v2
+; GFX9-NEXT:    v_cvt_f32_ubyte1_e32 v3, v4
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v16, v3
+; GFX9-NEXT:    v_cvt_f32_ubyte2_e32 v11, v4
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v17, v11
+; GFX9-NEXT:    v_mul_f32_e32 v15, v3, v15
+; GFX9-NEXT:    v_cvt_f32_ubyte3_e32 v14, v4
+; GFX9-NEXT:    v_trunc_f32_e32 v15, v15
+; GFX9-NEXT:    v_rcp_iflag_f32_e32 v18, v14
+; GFX9-NEXT:    v_mul_f32_e32 v16, v3, v16
+; GFX9-NEXT:    v_mad_f32 v19, -v15, v2, v3
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v15, v15
+; GFX9-NEXT:    v_trunc_f32_e32 v16, v16
+; GFX9-NEXT:    v_mul_f32_e32 v17, v3, v17
+; GFX9-NEXT:    v_mad_f32 v20, -v16, v3, v3
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v16, v16
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_cvt_f32_ubyte2_e32 v13, v9
+; GFX9-NEXT:    v_trunc_f32_e32 v17, v17
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v19|, v2
+; GFX9-NEXT:    v_mul_f32_e32 v18, v13, v18
+; GFX9-NEXT:    v_mad_f32 v21, -v17, v11, v3
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v17, v17
+; GFX9-NEXT:    v_addc_co_u32_e32 v2, vcc, 0, v15, vcc
+; GFX9-NEXT:    v_trunc_f32_e32 v18, v18
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v20|, v3
+; GFX9-NEXT:    v_mad_f32 v13, -v18, v14, v13
+; GFX9-NEXT:    v_cvt_u32_f32_e32 v18, v18
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v16, vcc
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v21|, v11
+; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, 0, v17, vcc
+; GFX9-NEXT:    v_cmp_ge_f32_e64 vcc, |v13|, v14
+; GFX9-NEXT:    v_lshrrev_b32_e32 v0, 16, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v10, 8, v4
+; GFX9-NEXT:    v_lshrrev_b32_e32 v12, 24, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v13, vcc, 0, v18, vcc
+; GFX9-NEXT:    v_perm_b32 v1, v4, v9, s4
+; GFX9-NEXT:    v_mul_lo_u32 v2, v2, v4
+; GFX9-NEXT:    v_mul_lo_u32 v3, v3, v10
+; GFX9-NEXT:    v_mul_lo_u32 v0, v11, v0
+; GFX9-NEXT:    v_mul_lo_u32 v4, v13, v12
+; GFX9-NEXT:    v_sub_u32_e32 v2, v10, v2
+; GFX9-NEXT:    v_sub_u32_sdwa v3, v10, v3 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-NEXT:    v_sub_u32_e32 v0, v10, v0
+; GFX9-NEXT:    v_sub_u32_sdwa v4, v9, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    global_store_dword v[5:6], v0, off
+; GFX9-NEXT:    global_store_dword v[7:8], v1, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <4 x i8>, ptr addrspace(1) %in0, i32 %tid
+  %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %in1, i32 %tid
+  %vec0 = load <4 x i8>, ptr addrspace(1) %gep0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 5, i32 5, i32 5, i32 2>
+  %vecins = urem <4 x i8> %shuffle0_0, %vec1
+  store <4 x i8> %vecins, ptr addrspace(1) %out0
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out1
+  ret void
+}
+
+
+define hidden void @xor_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0, ptr addrspace(1) %out1) {
+; GFX10-LABEL: xor_store_div:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    global_load_dword v4, v[0:1], off
+; GFX10-NEXT:    global_load_dword v9, v[2:3], off
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0xffffff00
+; GFX10-NEXT:    v_mov_b32_e32 v1, 1
+; GFX10-NEXT:    v_mov_b32_e32 v2, 2
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_and_b32_sdwa v0, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_and_b32_e32 v3, 0xffffff00, v9
+; GFX10-NEXT:    v_xor_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX10-NEXT:    v_xor_b32_sdwa v2, v9, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT:    v_xor_b32_e32 v0, 0x200, v0
+; GFX10-NEXT:    v_xor_b32_e32 v3, 0x100, v3
+; GFX10-NEXT:    v_or_b32_e32 v0, v1, v0
+; GFX10-NEXT:    v_or_b32_sdwa v1, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX10-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX10-NEXT:    v_perm_b32 v1, v9, v4, 0x5060307
+; GFX10-NEXT:    global_store_dword v[5:6], v0, off
+; GFX10-NEXT:    global_store_dword v[7:8], v1, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: xor_store_div:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    global_load_dword v9, v[2:3], off
+; GFX9-NEXT:    s_movk_i32 s4, 0xff00
+; GFX9-NEXT:    v_mov_b32_e32 v0, 1
+; GFX9-NEXT:    v_mov_b32_e32 v1, 2
+; GFX9-NEXT:    s_mov_b32 s5, 0x5060307
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_and_b32_sdwa v2, v4, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v3, 0xffffff00, v9
+; GFX9-NEXT:    v_xor_b32_sdwa v0, v9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD
+; GFX9-NEXT:    v_xor_b32_sdwa v1, v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_xor_b32_e32 v2, 0x200, v2
+; GFX9-NEXT:    v_xor_b32_e32 v3, 0x100, v3
+; GFX9-NEXT:    v_or_b32_e32 v0, v0, v2
+; GFX9-NEXT:    v_or_b32_sdwa v1, v1, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-NEXT:    v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
+; GFX9-NEXT:    v_perm_b32 v4, v9, v4, s5
+; GFX9-NEXT:    global_store_dword v[5:6], v0, off
+; GFX9-NEXT:    global_store_dword v[7:8], v4, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <4 x i8>, ptr addrspace(1) %in0, i32 %tid
+  %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %in1, i32 %tid
+  %vec0 = load <4 x i8>, ptr addrspace(1) %gep0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 7, i32 3, i32 6, i32 5>
+  %vecins = xor <4 x i8> %shuffle0_0, <i8 1, i8 2, i8 2, i8 1>
+  store <4 x i8> %vecins, ptr addrspace(1) %out0
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out1
+  ret void
+}
+
+
+define hidden void @zext_store_div(ptr addrspace(1) %in0, ptr addrspace(1) %in1, i8 %elt, ptr addrspace(1) %out0, ptr addrspace(1) %out1) {
+; GFX10-LABEL: zext_store_div:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX10-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v2, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
+; GFX10-NEXT:    global_load_dword v4, v[0:1], off
+; GFX10-NEXT:    global_load_dword v9, v[2:3], off
+; GFX10-NEXT:    v_mov_b32_e32 v0, 0xff
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    v_lshrrev_b16 v1, 8, v4
+; GFX10-NEXT:    v_and_b32_e32 v2, 0xff, v4
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_and_b32_e32 v3, 0xff, v9
+; GFX10-NEXT:    v_and_b32_sdwa v10, v4, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-NEXT:    v_perm_b32 v0, v1, v2, 0x5040100
+; GFX10-NEXT:    v_perm_b32 v2, v4, v9, 0x60504
+; GFX10-NEXT:    v_perm_b32 v1, v3, v10, 0x5040100
+; GFX10-NEXT:    global_store_dwordx2 v[7:8], v[0:1], off
+; GFX10-NEXT:    global_store_dword v[5:6], v2, off
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: zext_store_div:
+; GFX9:       ; %bb.0:
+; GFX9-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT:    v_and_b32_e32 v4, 0x3ff, v31
+; GFX9-NEXT:    v_lshlrev_b32_e32 v4, 2, v4
+; GFX9-NEXT:    v_add_co_u32_e32 v0, vcc, v0, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
+; GFX9-NEXT:    v_add_co_u32_e32 v2, vcc, v2, v4
+; GFX9-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT:    global_load_dword v4, v[0:1], off
+; GFX9-NEXT:    global_load_dword v9, v[2:3], off
+; GFX9-NEXT:    s_mov_b32 s4, 0x60504
+; GFX9-NEXT:    s_movk_i32 s5, 0xff
+; GFX9-NEXT:    s_mov_b32 s6, 0x5040100
+; GFX9-NEXT:    s_waitcnt vmcnt(1)
+; GFX9-NEXT:    v_lshrrev_b16_e32 v0, 8, v4
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    v_perm_b32 v2, v4, v9, s4
+; GFX9-NEXT:    v_and_b32_e32 v1, 0xff, v4
+; GFX9-NEXT:    v_and_b32_e32 v3, 0xff, v9
+; GFX9-NEXT:    v_and_b32_sdwa v4, v4, s5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NEXT:    v_perm_b32 v0, v0, v1, s6
+; GFX9-NEXT:    v_perm_b32 v1, v3, v4, s6
+; GFX9-NEXT:    global_store_dwordx2 v[7:8], v[0:1], off
+; GFX9-NEXT:    global_store_dword v[5:6], v2, off
+; GFX9-NEXT:    s_waitcnt vmcnt(0)
+; GFX9-NEXT:    s_setpc_b64 s[30:31]
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %gep0 = getelementptr <4 x i8>, ptr addrspace(1) %in0, i32 %tid
+  %gep1 = getelementptr <4 x i8>, ptr addrspace(1) %in1, i32 %tid
+  %vec0 = load <4 x i8>, ptr addrspace(1) %gep0, align 4
+  %vec1 = load <4 x i8>, ptr addrspace(1) %gep1, align 4
+  %shuffle0_0 = shufflevector <4 x i8> %vec0, <4 x i8> %vec1, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+  %insvec = zext <4 x i8> %shuffle0_0 to <4 x i16>
+  store <4 x i16> %insvec, ptr addrspace(1) %out1
+  store <4 x i8> %shuffle0_0, ptr addrspace(1) %out0
+  ret void
+}

From dae5cd73cb38b7176f10ffa527ae4d083fdc0efe Mon Sep 17 00:00:00 2001
From: Florian Hahn <flo@fhahn.com>
Date: Mon, 19 Jun 2023 17:57:05 +0100
Subject: [PATCH 062/130] Recommit "[LSR] Consider post-inc form when creating
 extends/truncates."

This reverts the revert commit 1797ab36efc9c90c921cd725831f8c3f6a7125a2.

The recommitted version now checks the PostIncLoopSets for all fixups
and returns nullptr if the result doesn't match for all fixups.
---
 .../Transforms/Scalar/LoopStrengthReduce.cpp  | 39 +++++++++++++++---
 .../CodeGen/AMDGPU/GlobalISel/llvm.memmove.ll | 34 +++++++--------
 .../X86/postinc-iv-used-by-urem-and-udiv.ll   | 41 +++++++++----------
 3 files changed, 72 insertions(+), 42 deletions(-)

diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index f82f4a4ec55f0..a863eab7c604c 100644
--- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -4138,6 +4138,29 @@ void LSRInstance::GenerateScales(LSRUse &LU, unsigned LUIdx, Formula Base) {
   }
 }
 
+/// Extend/Truncate \p Expr to \p ToTy considering post-inc uses in \p Loops.
+/// For all PostIncLoopSets in \p Loops, first de-normalize \p Expr, then
+/// perform the extension/truncate and normalize again, as the normalized form
+/// can result in folds that are not valid in the post-inc use contexts. The
+/// expressions for all PostIncLoopSets must match, otherwise return nullptr.
+static const SCEV *
+getAnyExtendConsideringPostIncUses(ArrayRef<PostIncLoopSet> Loops,
+                                   const SCEV *Expr, Type *ToTy,
+                                   ScalarEvolution &SE) {
+  const SCEV *Result = nullptr;
+  for (auto &L : Loops) {
+    auto *DenormExpr = denormalizeForPostIncUse(Expr, L, SE);
+    const SCEV *NewDenormExpr = SE.getAnyExtendExpr(DenormExpr, ToTy);
+    const SCEV *New = normalizeForPostIncUse(NewDenormExpr, L, SE);
+    if (Result && New != Result)
+      return nullptr;
+    Result = New;
+  }
+
+  assert(Result && "failed to create expression");
+  return Result;
+}
+
 /// Generate reuse formulae from different IV types.
 void LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) {
   // Don't bother truncating symbolic values.
@@ -4157,6 +4180,10 @@ void LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) {
              [](const SCEV *S) { return S->getType()->isPointerTy(); }))
     return;
 
+  SmallVector<PostIncLoopSet> Loops;
+  for (auto &LF : LU.Fixups)
+    Loops.push_back(LF.PostIncLoops);
+
   for (Type *SrcTy : Types) {
     if (SrcTy != DstTy && TTI.isTruncateFree(SrcTy, DstTy)) {
       Formula F = Base;
@@ -4166,15 +4193,17 @@ void LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) {
       // initial node (maybe due to depth limitations), but it can do them while
       // taking ext.
       if (F.ScaledReg) {
-        const SCEV *NewScaledReg = SE.getAnyExtendExpr(F.ScaledReg, SrcTy);
-        if (NewScaledReg->isZero())
-         continue;
+        const SCEV *NewScaledReg =
+            getAnyExtendConsideringPostIncUses(Loops, F.ScaledReg, SrcTy, SE);
+        if (!NewScaledReg || NewScaledReg->isZero())
+          continue;
         F.ScaledReg = NewScaledReg;
       }
       bool HasZeroBaseReg = false;
       for (const SCEV *&BaseReg : F.BaseRegs) {
-        const SCEV *NewBaseReg = SE.getAnyExtendExpr(BaseReg, SrcTy);
-        if (NewBaseReg->isZero()) {
+        const SCEV *NewBaseReg =
+            getAnyExtendConsideringPostIncUses(Loops, BaseReg, SrcTy, SE);
+        if (!NewBaseReg || NewBaseReg->isZero()) {
           HasZeroBaseReg = true;
           break;
         }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memmove.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memmove.ll
index 466147cac3439..0ec4f64b38a1b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memmove.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.memmove.ll
@@ -31,29 +31,31 @@ define amdgpu_cs void @memmove_p1i8(ptr addrspace(1) %dst, ptr addrspace(1) %src
 ; LOOP-NEXT:    s_waitcnt vmcnt(0)
 ; LOOP-NEXT:    buffer_store_byte v8, v[6:7], s[0:3], 0 addr64
 ; LOOP-NEXT:    s_cbranch_vccnz .LBB0_2
-; LOOP-NEXT:  .LBB0_3: ; %Flow14
+; LOOP-NEXT:  .LBB0_3: ; %Flow17
 ; LOOP-NEXT:    s_andn2_saveexec_b64 s[0:1], s[4:5]
 ; LOOP-NEXT:    s_cbranch_execz .LBB0_6
 ; LOOP-NEXT:  ; %bb.4: ; %copy_backwards
-; LOOP-NEXT:    s_mov_b64 s[4:5], 3
-; LOOP-NEXT:    s_mov_b32 s2, 0
-; LOOP-NEXT:    s_mov_b32 s3, 0xf000
-; LOOP-NEXT:    s_mov_b64 s[0:1], 0
-; LOOP-NEXT:    v_mov_b32_e32 v4, s4
-; LOOP-NEXT:    v_mov_b32_e32 v5, s5
+; LOOP-NEXT:    s_mov_b32 s0, -4
+; LOOP-NEXT:    v_add_i32_e32 v0, vcc, 3, v0
+; LOOP-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; LOOP-NEXT:    v_add_i32_e32 v2, vcc, 3, v2
+; LOOP-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
+; LOOP-NEXT:    s_mov_b32 s6, 0
+; LOOP-NEXT:    s_mov_b32 s7, 0xf000
+; LOOP-NEXT:    s_mov_b64 s[4:5], 0
+; LOOP-NEXT:    v_mov_b32_e32 v4, s0
 ; LOOP-NEXT:  .LBB0_5: ; %copy_backwards_loop
 ; LOOP-NEXT:    ; =>This Inner Loop Header: Depth=1
-; LOOP-NEXT:    v_add_i32_e32 v6, vcc, v2, v4
-; LOOP-NEXT:    v_addc_u32_e32 v7, vcc, v3, v5, vcc
 ; LOOP-NEXT:    s_waitcnt expcnt(0)
-; LOOP-NEXT:    buffer_load_ubyte v8, v[6:7], s[0:3], 0 addr64
-; LOOP-NEXT:    v_add_i32_e32 v6, vcc, v0, v4
-; LOOP-NEXT:    v_addc_u32_e32 v7, vcc, v1, v5, vcc
-; LOOP-NEXT:    v_add_i32_e32 v4, vcc, -1, v4
-; LOOP-NEXT:    v_addc_u32_e32 v5, vcc, -1, v5, vcc
-; LOOP-NEXT:    v_cmp_eq_u32_e32 vcc, -1, v4
+; LOOP-NEXT:    buffer_load_ubyte v5, v[2:3], s[4:7], 0 addr64
+; LOOP-NEXT:    v_add_i32_e32 v4, vcc, 1, v4
+; LOOP-NEXT:    s_and_b64 vcc, vcc, exec
 ; LOOP-NEXT:    s_waitcnt vmcnt(0)
-; LOOP-NEXT:    buffer_store_byte v8, v[6:7], s[0:3], 0 addr64
+; LOOP-NEXT:    buffer_store_byte v5, v[0:1], s[4:7], 0 addr64
+; LOOP-NEXT:    v_add_i32_e64 v0, s[0:1], -1, v0
+; LOOP-NEXT:    v_addc_u32_e64 v1, s[0:1], -1, v1, s[0:1]
+; LOOP-NEXT:    v_add_i32_e64 v2, s[0:1], -1, v2
+; LOOP-NEXT:    v_addc_u32_e64 v3, s[0:1], -1, v3, s[0:1]
 ; LOOP-NEXT:    s_cbranch_vccz .LBB0_5
 ; LOOP-NEXT:  .LBB0_6: ; %memmove_done
 ; LOOP-NEXT:    s_endpgm
diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/postinc-iv-used-by-urem-and-udiv.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/postinc-iv-used-by-urem-and-udiv.ll
index 8735bd3036c9d..373dd4bacd77e 100644
--- a/llvm/test/Transforms/LoopStrengthReduce/X86/postinc-iv-used-by-urem-and-udiv.ll
+++ b/llvm/test/Transforms/LoopStrengthReduce/X86/postinc-iv-used-by-urem-and-udiv.ll
@@ -12,18 +12,19 @@ define i32 @test_pr38847() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i64 [ [[LSR_IV_NEXT:%.*]], [[LOOP]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[LSR_IV1:%.*]] = phi i32 [ [[LSR_IV_NEXT2:%.*]], [[LOOP]] ], [ 1, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i64 [ [[LSR_IV_NEXT:%.*]], [[LOOP]] ], [ 1, [[ENTRY]] ]
+; CHECK-NEXT:    [[LSR_IV_NEXT2]] = add nsw i32 [[LSR_IV1]], -1
+; CHECK-NEXT:    [[LSR:%.*]] = trunc i32 [[LSR_IV_NEXT2]] to i8
 ; CHECK-NEXT:    call void @use(i64 [[LSR_IV]])
 ; CHECK-NEXT:    [[LSR_IV_NEXT]] = add nsw i64 [[LSR_IV]], -1
-; CHECK-NEXT:    [[TMP1:%.*]] = trunc i64 [[LSR_IV_NEXT]] to i8
-; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 [[TMP1]], -1
+; CHECK-NEXT:    [[CMP2:%.*]] = icmp sgt i8 [[LSR]], -1
 ; CHECK-NEXT:    br i1 [[CMP2]], label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
-; CHECK-NEXT:    [[TMP0:%.*]] = udiv i64 [[LSR_IV_NEXT]], 9
-; CHECK-NEXT:    [[TMP1:%.*]] = mul i64 [[TMP0]], 4294967287
-; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], [[LSR_IV_NEXT]]
-; CHECK-NEXT:    [[TMP:%.*]] = trunc i64 [[TMP2]] to i32
-; CHECK-NEXT:    ret i32 [[TMP]]
+; CHECK-NEXT:    [[TMP0:%.*]] = udiv i32 [[LSR_IV_NEXT2]], 9
+; CHECK-NEXT:    [[TMP1:%.*]] = mul nuw i32 [[TMP0]], 9
+; CHECK-NEXT:    [[TMP2:%.*]] = sub i32 [[LSR_IV_NEXT2]], [[TMP1]]
+; CHECK-NEXT:    ret i32 [[TMP2]]
 ;
 entry:
   br label %loop
@@ -47,12 +48,12 @@ define i64 @test_pr58039() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i64 [ [[LSR_IV_NEXT:%.*]], [[LOOP]] ], [ 83, [[ENTRY:%.*]] ]
+; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i64 [ [[LSR_IV_NEXT:%.*]], [[LOOP]] ], [ -4294967213, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ 0, [[ENTRY]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
 ; CHECK-NEXT:    [[TMP2:%.*]] = trunc i64 [[IV]] to i32
 ; CHECK-NEXT:    call void @use.i32(i32 [[TMP2]])
 ; CHECK-NEXT:    [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
-; CHECK-NEXT:    [[LSR_IV_NEXT]] = add nuw nsw i64 [[LSR_IV]], 4294967295
+; CHECK-NEXT:    [[LSR_IV_NEXT]] = add nsw i64 [[LSR_IV]], 4294967295
 ; CHECK-NEXT:    br i1 false, label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    [[TMP0:%.*]] = udiv i64 [[LSR_IV_NEXT]], 12
@@ -93,25 +94,23 @@ define i32 @test_pr62852() {
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    br label [[LOOP:%.*]]
 ; CHECK:       loop:
-; CHECK-NEXT:    [[LSR_IV4:%.*]] = phi i64 [ [[LSR_IV_NEXT5:%.*]], [[LOOP]] ], [ -1, [[ENTRY:%.*]] ]
-; CHECK-NEXT:    [[LSR_IV1:%.*]] = phi i64 [ [[LSR_IV_NEXT2:%.*]], [[LOOP]] ], [ 1, [[ENTRY]] ]
+; CHECK-NEXT:    [[LSR_IV1:%.*]] = phi i64 [ [[LSR_IV_NEXT2:%.*]], [[LOOP]] ], [ -1, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[LSR_IV:%.*]] = phi i64 [ [[LSR_IV_NEXT:%.*]], [[LOOP]] ], [ 2, [[ENTRY]] ]
-; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[LSR_IV4]], 1
+; CHECK-NEXT:    [[IV_1:%.*]] = phi i32 [ 1, [[ENTRY]] ], [ [[DEC_1:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = add i64 [[LSR_IV1]], 1
+; CHECK-NEXT:    [[DEC_1]] = add nsw i32 [[IV_1]], -1
 ; CHECK-NEXT:    call void @use(i64 [[TMP0]])
 ; CHECK-NEXT:    [[LSR_IV_NEXT]] = add nsw i64 [[LSR_IV]], -1
 ; CHECK-NEXT:    [[TMP:%.*]] = trunc i64 [[LSR_IV_NEXT]] to i32
-; CHECK-NEXT:    [[LSR_IV_NEXT2]] = add nsw i64 [[LSR_IV1]], -1
-; CHECK-NEXT:    [[LSR_IV_NEXT5]] = add nsw i64 [[LSR_IV4]], 1
+; CHECK-NEXT:    [[LSR_IV_NEXT2]] = add nsw i64 [[LSR_IV1]], 1
 ; CHECK-NEXT:    [[CMP6_1:%.*]] = icmp sgt i32 [[TMP]], 0
 ; CHECK-NEXT:    br i1 [[CMP6_1]], label [[LOOP]], label [[EXIT:%.*]]
 ; CHECK:       exit:
 ; CHECK-NEXT:    call void @use(i64 [[LSR_IV_NEXT]])
-; CHECK-NEXT:    call void @use(i64 [[LSR_IV_NEXT5]])
-; CHECK-NEXT:    [[TMP1:%.*]] = udiv i64 [[LSR_IV_NEXT2]], 53
-; CHECK-NEXT:    [[TMP2:%.*]] = mul i64 [[TMP1]], 4294967243
-; CHECK-NEXT:    [[TMP3:%.*]] = add i64 [[TMP2]], [[LSR_IV_NEXT]]
-; CHECK-NEXT:    [[TMP4:%.*]] = add i64 [[TMP3]], -1
-; CHECK-NEXT:    [[TMP3:%.*]] = trunc i64 [[TMP4]] to i32
+; CHECK-NEXT:    call void @use(i64 [[LSR_IV_NEXT2]])
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv i32 [[DEC_1]], 53
+; CHECK-NEXT:    [[TMP2:%.*]] = mul nuw i32 [[TMP1]], 53
+; CHECK-NEXT:    [[TMP3:%.*]] = sub i32 [[DEC_1]], [[TMP2]]
 ; CHECK-NEXT:    ret i32 [[TMP3]]
 ;
 entry:

From aeb99dc48a58f872465e818d0eda7d9c3f221e06 Mon Sep 17 00:00:00 2001
From: Vladislav Dzhidzhoev <vdzhidzhoev@accesssoftek.com>
Date: Mon, 19 Jun 2023 19:16:13 +0200
Subject: [PATCH 063/130] Revert "[DebugMetadata][DwarfDebug] Support
 function-local types in lexical block scopes (4/7)"

This reverts commit 66511b401042f28c74d2ded3aac76d19a53bd7c4.
llvm/test/DebugInfo/Generic/local-type-as-template-parameter.ll is
broken.
---
 .../CodeGen/debug-info-codeview-unnamed.c     |  16 +-
 clang/test/CodeGen/debug-info-unused-types.c  |  14 +-
 .../test/CodeGen/debug-info-unused-types.cpp  |  12 +-
 clang/test/CodeGenCXX/debug-info-access.cpp   |   2 +-
 .../CodeGenCXX/debug-info-anon-union-vars.cpp |  12 +-
 .../debug-info-codeview-unnamed.cpp           | 110 ++---
 .../debug-info-gline-tables-only-codeview.cpp |   4 +-
 clang/test/CodeGenCXX/debug-lambda-this.cpp   |   4 +-
 llvm/include/llvm/IR/DIBuilder.h              |   6 +-
 llvm/lib/Bitcode/Reader/MetadataLoader.cpp    |  87 ++--
 .../CodeGen/AsmPrinter/DwarfCompileUnit.cpp   |  60 +--
 .../lib/CodeGen/AsmPrinter/DwarfCompileUnit.h |  16 +-
 llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp    |  13 +-
 llvm/lib/IR/DIBuilder.cpp                     |  34 +-
 llvm/lib/IR/Verifier.cpp                      |   6 +-
 llvm/test/Bitcode/upgrade-cu-locals.ll        |  68 ++-
 llvm/test/Bitcode/upgrade-cu-locals.ll.bc     | Bin 2780 -> 2688 bytes
 .../DebugInfo/Generic/inlined-local-type.ll   | 128 ------
 .../Generic/lexical-block-retained-types.ll   |  55 ---
 .../DebugInfo/Generic/lexical-block-types.ll  | 425 ------------------
 .../local-type-as-template-parameter.ll       | 160 -------
 .../Generic/verifier-invalid-disubprogram.ll  |   2 +-
 llvm/test/DebugInfo/X86/set.ll                |   4 +-
 23 files changed, 187 insertions(+), 1051 deletions(-)
 delete mode 100644 llvm/test/DebugInfo/Generic/inlined-local-type.ll
 delete mode 100644 llvm/test/DebugInfo/Generic/lexical-block-retained-types.ll
 delete mode 100644 llvm/test/DebugInfo/Generic/lexical-block-types.ll
 delete mode 100644 llvm/test/DebugInfo/Generic/local-type-as-template-parameter.ll

diff --git a/clang/test/CodeGen/debug-info-codeview-unnamed.c b/clang/test/CodeGen/debug-info-codeview-unnamed.c
index 16ffb3682236f..bd2a7543e56b2 100644
--- a/clang/test/CodeGen/debug-info-codeview-unnamed.c
+++ b/clang/test/CodeGen/debug-info-codeview-unnamed.c
@@ -8,23 +8,23 @@ int main(int argc, char* argv[], char* arge[]) {
   //
   struct { int bar; } one = {42};
   //
-  // LINUX:      [[TYPE_OF_ONE:![0-9]+]] = distinct !DICompositeType(
+  // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "one"
+  // LINUX-SAME:     type: [[TYPE_OF_ONE:![0-9]+]]
+  // LINUX-SAME:     )
+  // LINUX:      [[TYPE_OF_ONE]] = distinct !DICompositeType(
   // LINUX-SAME:     tag: DW_TAG_structure_type
   // LINUX-NOT:      name:
   // LINUX-NOT:      identifier:
   // LINUX-SAME:     )
-  // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "one"
-  // LINUX-SAME:     type: [[TYPE_OF_ONE]]
-  // LINUX-SAME:     )
   //
-  // MSVC:       [[TYPE_OF_ONE:![0-9]+]] = distinct !DICompositeType
+  // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "one"
+  // MSVC-SAME:      type: [[TYPE_OF_ONE:![0-9]+]]
+  // MSVC-SAME:      )
+  // MSVC:       [[TYPE_OF_ONE]] = distinct !DICompositeType
   // MSVC-SAME:      tag: DW_TAG_structure_type
   // MSVC-NOT:       name:
   // MSVC-NOT:       identifier:
   // MSVC-SAME:      )
-  // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "one"
-  // MSVC-SAME:      type: [[TYPE_OF_ONE]]
-  // MSVC-SAME:      )
 
   return 0;
 }
diff --git a/clang/test/CodeGen/debug-info-unused-types.c b/clang/test/CodeGen/debug-info-unused-types.c
index 5bcc71f710a5c..3e9f7b07658e3 100644
--- a/clang/test/CodeGen/debug-info-unused-types.c
+++ b/clang/test/CodeGen/debug-info-unused-types.c
@@ -18,13 +18,13 @@ void quux(void) {
 // CHECK: !DICompileUnit{{.+}}retainedTypes: [[RETTYPES:![0-9]+]]
 // CHECK: [[TYPE0:![0-9]+]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "bar"
 // CHECK: [[TYPE1:![0-9]+]] = !DIEnumerator(name: "BAR"
-// CHECK: [[RETTYPES]] = !{[[TYPE2:![0-9]+]], [[TYPE3:![0-9]+]], [[TYPE0]], [[TYPE4:![0-9]+]], {{![0-9]+}}, [[TYPE5:![0-9]+]], [[TYPE6:![0-9]+]], [[TYPE8:![0-9]+]]}
-// CHECK: [[TYPE2]] = !DIDerivedType(tag: DW_TAG_typedef, name: "my_int"
-// CHECK: [[TYPE3]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "foo"
-// CHECK: [[TYPE4]] = distinct !DICompositeType(tag: DW_TAG_union_type, name: "baz"
-// CHECK: [[TYPE5]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "y"
-// CHECK: [[TYPE6]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "z"
-// CHECK: [[TYPE7:![0-9]+]] = !DIEnumerator(name: "Z"
+// CHECK: [[TYPE2:![0-9]+]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "z"
+// CHECK: [[TYPE3:![0-9]+]] = !DIEnumerator(name: "Z"
+// CHECK: [[RETTYPES]] = !{[[TYPE4:![0-9]+]], [[TYPE5:![0-9]+]], [[TYPE0]], [[TYPE6:![0-9]+]], {{![0-9]+}}, [[TYPE7:![0-9]+]], [[TYPE2]], [[TYPE8:![0-9]+]]}
+// CHECK: [[TYPE4]] = !DIDerivedType(tag: DW_TAG_typedef, name: "my_int"
+// CHECK: [[TYPE5]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "foo"
+// CHECK: [[TYPE6]] = distinct !DICompositeType(tag: DW_TAG_union_type, name: "baz"
+// CHECK: [[TYPE7]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "y"
 // CHECK: [[TYPE8]] = distinct !DICompositeType(tag: DW_TAG_union_type, name: "w"
 
 // Check that debug info is not emitted for the typedef, struct, enum, and
diff --git a/clang/test/CodeGen/debug-info-unused-types.cpp b/clang/test/CodeGen/debug-info-unused-types.cpp
index 715fada47eab8..023cac159faa4 100644
--- a/clang/test/CodeGen/debug-info-unused-types.cpp
+++ b/clang/test/CodeGen/debug-info-unused-types.cpp
@@ -13,12 +13,12 @@ void quux() {
 // CHECK: !DICompileUnit{{.+}}retainedTypes: [[RETTYPES:![0-9]+]]
 // CHECK: [[TYPE0:![0-9]+]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "baz"
 // CHECK: [[TYPE1:![0-9]+]] = !DIEnumerator(name: "BAZ"
-// CHECK: [[RETTYPES]] = !{[[TYPE2:![0-9]+]], [[TYPE3:![0-9]+]], [[TYPE0]], {{![0-9]+}}, [[TYPE4:![0-9]+]], [[TYPE5:![0-9]+]]}
-// CHECK: [[TYPE2]] = !DIDerivedType(tag: DW_TAG_typedef, name: "foo"
-// CHECK: [[TYPE3]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "bar"
-// CHECK: [[TYPE4]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "y"
-// CHECK: [[TYPE5]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "z"
-// CHECK: [[TYPE6:![0-9]+]] = !DIEnumerator(name: "Z"
+// CHECK: [[TYPE2:![0-9]+]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "z"
+// CHECK: [[TYPE3:![0-9]+]] = !DIEnumerator(name: "Z"
+// CHECK: [[RETTYPES]] = !{[[TYPE4:![0-9]+]], [[TYPE5:![0-9]+]], [[TYPE0]], {{![0-9]+}}, [[TYPE6:![0-9]+]], [[TYPE2]]}
+// CHECK: [[TYPE4]] = !DIDerivedType(tag: DW_TAG_typedef, name: "foo"
+// CHECK: [[TYPE5]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "bar"
+// CHECK: [[TYPE6]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "y"
 
 // NODBG-NOT: !DI{{CompositeType|Enumerator|DerivedType}}
 
diff --git a/clang/test/CodeGenCXX/debug-info-access.cpp b/clang/test/CodeGenCXX/debug-info-access.cpp
index 7c0bf8ccb0384..9f2c044843d0f 100644
--- a/clang/test/CodeGenCXX/debug-info-access.cpp
+++ b/clang/test/CodeGenCXX/debug-info-access.cpp
@@ -18,9 +18,9 @@ class B : public A {
   static int public_static;
 
 protected:
-  // CHECK-DAG: !DIDerivedType(tag: DW_TAG_typedef, name: "prot_using",{{.*}} line: [[@LINE+3]],{{.*}} flags: DIFlagProtected)
   // CHECK-DAG: !DIDerivedType(tag: DW_TAG_typedef, name: "prot_typedef",{{.*}} line: [[@LINE+1]],{{.*}} flags: DIFlagProtected)
   typedef int prot_typedef;
+  // CHECK-DAG: !DIDerivedType(tag: DW_TAG_typedef, name: "prot_using",{{.*}} line: [[@LINE+1]],{{.*}} flags: DIFlagProtected)
   using prot_using = prot_typedef;
   prot_using prot_member;
 
diff --git a/clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp b/clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp
index c91cf83c0405f..61b3c7c0526c8 100644
--- a/clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp
+++ b/clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp
@@ -51,13 +51,13 @@ void instantiate(int x) {
 // CHECK: !DIGlobalVariable(name: "b",{{.*}} file: [[FILE]], line: 6,{{.*}} isLocal: true, isDefinition: true
 // CHECK: !DIGlobalVariable(name: "result", {{.*}} isLocal: false, isDefinition: true
 // CHECK: !DIGlobalVariable(name: "value", {{.*}} isLocal: false, isDefinition: true
-// CHECK: ![[UNION:[0-9]+]] = distinct !DICompositeType(tag: DW_TAG_union_type,
-// CHECK-NOT: name:
-// CHECK: elements
-// CHECK: !DIDerivedType(tag: DW_TAG_member, name: "i", scope: ![[UNION]],
-// CHECK: !DIDerivedType(tag: DW_TAG_member, name: "c", scope: ![[UNION]],
 // CHECK: !DILocalVariable(name: "i", {{.*}}, flags: DIFlagArtificial
 // CHECK: !DILocalVariable(name: "c", {{.*}}, flags: DIFlagArtificial
 // CHECK: !DILocalVariable(
 // CHECK-NOT: name:
-// CHECK: type: ![[UNION]]
+// CHECK: type: ![[UNION:[0-9]+]]
+// CHECK: ![[UNION]] = distinct !DICompositeType(tag: DW_TAG_union_type,
+// CHECK-NOT: name:
+// CHECK: elements
+// CHECK: !DIDerivedType(tag: DW_TAG_member, name: "i", scope: ![[UNION]],
+// CHECK: !DIDerivedType(tag: DW_TAG_member, name: "c", scope: ![[UNION]],
diff --git a/clang/test/CodeGenCXX/debug-info-codeview-unnamed.cpp b/clang/test/CodeGenCXX/debug-info-codeview-unnamed.cpp
index 9602ac1b02497..b4c79936ab33e 100644
--- a/clang/test/CodeGenCXX/debug-info-codeview-unnamed.cpp
+++ b/clang/test/CodeGenCXX/debug-info-codeview-unnamed.cpp
@@ -3,60 +3,6 @@
 
 int main(int argc, char* argv[], char* arge[]) {
   //
-  // LINUX:      [[TYPE_OF_ONE:![0-9]+]] = distinct !DICompositeType(
-  // LINUX-SAME:     tag: DW_TAG_structure_type
-  // LINUX-NOT:      name:
-  // LINUX-NOT:      identifier:
-  // LINUX-SAME:     )
-  //
-  // MSVC:       [[TYPE_OF_ONE:![0-9]+]] = distinct !DICompositeType
-  // MSVC-SAME:      tag: DW_TAG_structure_type
-  // MSVC-SAME:      name: "<unnamed-type-one>"
-  // MSVC-SAME:      identifier: ".?AU<unnamed-type-one>@?1??main@@9@"
-  // MSVC-SAME:      )
-
-
-  //
-  // LINUX:      [[TYPE_OF_TWO:![0-9]+]] = distinct !DICompositeType(
-  // LINUX-SAME:     tag: DW_TAG_structure_type
-  // LINUX-NOT:      name:
-  // LINUX-NOT:      identifier:
-  // LINUX-SAME:     )
-  //
-  // MSVC:       [[TYPE_OF_TWO:![0-9]+]] = distinct !DICompositeType
-  // MSVC-SAME:      tag: DW_TAG_structure_type
-  // MSVC-SAME:      name: "<unnamed-type-two>"
-  // MSVC-SAME:      identifier: ".?AU<unnamed-type-two>@?2??main@@9@"
-  // MSVC-SAME:      )
-
-
-  //
-  // LINUX:      [[TYPE_OF_THREE:![0-9]+]] = distinct !DICompositeType(
-  // LINUX-SAME:     tag: DW_TAG_structure_type
-  // LINUX-SAME:     name: "named"
-  // LINUX-NOT:      identifier:
-  // LINUX-SAME:     )
-  //
-  // MSVC:       [[TYPE_OF_THREE:![0-9]+]] = distinct !DICompositeType
-  // MSVC-SAME:      tag: DW_TAG_structure_type
-  // MSVC-SAME:      name: "named"
-  // MSVC-SAME:      identifier: ".?AUnamed@?1??main@@9@"
-  // MSVC-SAME:      )
-
-  //
-  // LINUX:      [[TYPE_OF_FOUR:![0-9]+]] = distinct !DICompositeType(
-  // LINUX-SAME:     tag: DW_TAG_class_type
-  // LINUX-NOT:      name:
-  // LINUX-NOT:      identifier:
-  // LINUX-SAME:     )
-  //
-  // MSVC:       [[TYPE_OF_FOUR:![0-9]+]] = distinct !DICompositeType
-  // MSVC-SAME:      tag: DW_TAG_class_type
-  // MSVC-SAME:      name: "<lambda_0>"
-  // MSVC-SAME:      identifier: ".?AV<lambda_0>@?0??main@@9@"
-  // MSVC-SAME:      )
-
-
   // In CodeView, the LF_MFUNCTION entry for "bar()" refers to the forward
   // reference of the unnamed struct. Visual Studio requires a unique
   // identifier to match the LF_STRUCTURE forward reference to the definition.
@@ -64,11 +10,21 @@ int main(int argc, char* argv[], char* arge[]) {
   struct { void bar() {} } one;
   //
   // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "one"
-  // LINUX-SAME:     type: [[TYPE_OF_ONE]]
+  // LINUX-SAME:     type: [[TYPE_OF_ONE:![0-9]+]]
+  // LINUX-SAME:     )
+  // LINUX:      [[TYPE_OF_ONE]] = distinct !DICompositeType(
+  // LINUX-SAME:     tag: DW_TAG_structure_type
+  // LINUX-NOT:      name:
+  // LINUX-NOT:      identifier:
   // LINUX-SAME:     )
   //
   // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "one"
-  // MSVC-SAME:      type: [[TYPE_OF_ONE]]
+  // MSVC-SAME:      type: [[TYPE_OF_ONE:![0-9]+]]
+  // MSVC-SAME:      )
+  // MSVC:       [[TYPE_OF_ONE]] = distinct !DICompositeType
+  // MSVC-SAME:      tag: DW_TAG_structure_type
+  // MSVC-SAME:      name: "<unnamed-type-one>"
+  // MSVC-SAME:      identifier: ".?AU<unnamed-type-one>@?1??main@@9@"
   // MSVC-SAME:      )
 
 
@@ -80,11 +36,21 @@ int main(int argc, char* argv[], char* arge[]) {
   int decltype(two)::*ptr2unnamed = &decltype(two)::bar;
   //
   // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "two"
-  // LINUX-SAME:     type: [[TYPE_OF_TWO]]
+  // LINUX-SAME:     type: [[TYPE_OF_TWO:![0-9]+]]
+  // LINUX-SAME:     )
+  // LINUX:      [[TYPE_OF_TWO]] = distinct !DICompositeType(
+  // LINUX-SAME:     tag: DW_TAG_structure_type
+  // LINUX-NOT:      name:
+  // LINUX-NOT:      identifier:
   // LINUX-SAME:     )
   //
   // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "two"
-  // MSVC-SAME:      type: [[TYPE_OF_TWO]]
+  // MSVC-SAME:      type: [[TYPE_OF_TWO:![0-9]+]]
+  // MSVC-SAME:      )
+  // MSVC:       [[TYPE_OF_TWO]] = distinct !DICompositeType
+  // MSVC-SAME:      tag: DW_TAG_structure_type
+  // MSVC-SAME:      name: "<unnamed-type-two>"
+  // MSVC-SAME:      identifier: ".?AU<unnamed-type-two>@?2??main@@9@"
   // MSVC-SAME:      )
 
 
@@ -95,11 +61,21 @@ int main(int argc, char* argv[], char* arge[]) {
   struct named { int bar; int named::* p2mem; } three = { 42, &named::bar };
   //
   // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "three"
-  // LINUX-SAME:     type: [[TYPE_OF_THREE]]
+  // LINUX-SAME:     type: [[TYPE_OF_THREE:![0-9]+]]
+  // LINUX-SAME:     )
+  // LINUX:      [[TYPE_OF_THREE]] = distinct !DICompositeType(
+  // LINUX-SAME:     tag: DW_TAG_structure_type
+  // LINUX-SAME:     name: "named"
+  // LINUX-NOT:      identifier:
   // LINUX-SAME:     )
   //
   // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "three"
-  // MSVC-SAME:      type: [[TYPE_OF_THREE]]
+  // MSVC-SAME:      type: [[TYPE_OF_THREE:![0-9]+]]
+  // MSVC-SAME:      )
+  // MSVC:       [[TYPE_OF_THREE]] = distinct !DICompositeType
+  // MSVC-SAME:      tag: DW_TAG_structure_type
+  // MSVC-SAME:      name: "named"
+  // MSVC-SAME:      identifier: ".?AUnamed@?1??main@@9@"
   // MSVC-SAME:      )
 
 
@@ -111,11 +87,21 @@ int main(int argc, char* argv[], char* arge[]) {
   auto four = [argc](int i) -> int { return argc == i ? 1 : 0; };
   //
   // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "four"
-  // LINUX-SAME:     type: [[TYPE_OF_FOUR]]
+  // LINUX-SAME:     type: [[TYPE_OF_FOUR:![0-9]+]]
+  // LINUX-SAME:     )
+  // LINUX:      [[TYPE_OF_FOUR]] = distinct !DICompositeType(
+  // LINUX-SAME:     tag: DW_TAG_class_type
+  // LINUX-NOT:      name:
+  // LINUX-NOT:      identifier:
   // LINUX-SAME:     )
   //
   // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "four"
-  // MSVC-SAME:      type: [[TYPE_OF_FOUR]]
+  // MSVC-SAME:      type: [[TYPE_OF_FOUR:![0-9]+]]
+  // MSVC-SAME:      )
+  // MSVC:       [[TYPE_OF_FOUR]] = distinct !DICompositeType
+  // MSVC-SAME:      tag: DW_TAG_class_type
+  // MSVC-SAME:      name: "<lambda_0>"
+  // MSVC-SAME:      identifier: ".?AV<lambda_0>@?0??main@@9@"
   // MSVC-SAME:      )
 
   return 0;
diff --git a/clang/test/CodeGenCXX/debug-info-gline-tables-only-codeview.cpp b/clang/test/CodeGenCXX/debug-info-gline-tables-only-codeview.cpp
index 122e4aa62ea7d..6b9c9a243decd 100644
--- a/clang/test/CodeGenCXX/debug-info-gline-tables-only-codeview.cpp
+++ b/clang/test/CodeGenCXX/debug-info-gline-tables-only-codeview.cpp
@@ -51,9 +51,9 @@ void test() {
   // CHECK-SAME:                              name: "<lambda_2_1>",
   c.lambda_params();
 
-  // CHECK: ![[LAMBDA1:[0-9]+]] = !DICompositeType(tag: DW_TAG_class_type,
+  // CHECK: !DISubprogram(name: "operator()", scope: ![[LAMBDA1:[0-9]+]],
+  // CHECK: ![[LAMBDA1]] = !DICompositeType(tag: DW_TAG_class_type,
   // CHECK-SAME:                            name: "<lambda_1>",
   // CHECK-SAME:                            flags: DIFlagFwdDecl
-  // CHECK: !DISubprogram(name: "operator()", scope: ![[LAMBDA1]],
   c.lambda2();
 }
diff --git a/clang/test/CodeGenCXX/debug-lambda-this.cpp b/clang/test/CodeGenCXX/debug-lambda-this.cpp
index 3d659e7bfd004..eecbac6520ac9 100644
--- a/clang/test/CodeGenCXX/debug-lambda-this.cpp
+++ b/clang/test/CodeGenCXX/debug-lambda-this.cpp
@@ -13,10 +13,10 @@ int D::d(int x) {
 }
 
 // CHECK: ![[D:[0-9]+]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "D",
+// CHECK: ![[POINTER:.*]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: ![[D]], size: 64)
 // CHECK: !DIDerivedType(tag: DW_TAG_member, name: "this",
 // CHECK-SAME:           line: 11
-// CHECK-SAME:           baseType: ![[POINTER:[0-9]+]]
+// CHECK-SAME:           baseType: ![[POINTER]]
 // CHECK-SAME:           size: 64
 // CHECK-NOT:            offset: 0
 // CHECK-SAME:           ){{$}}
-// CHECK: ![[POINTER]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: ![[D]], size: 64)
diff --git a/llvm/include/llvm/IR/DIBuilder.h b/llvm/include/llvm/IR/DIBuilder.h
index 0aa180aec6d8a..ecd6dd7b0a4f8 100644
--- a/llvm/include/llvm/IR/DIBuilder.h
+++ b/llvm/include/llvm/IR/DIBuilder.h
@@ -49,7 +49,7 @@ namespace llvm {
     Function *LabelFn;       ///< llvm.dbg.label
     Function *AssignFn;      ///< llvm.dbg.assign
 
-    SmallVector<TrackingMDNodeRef, 4> EnumTypes;
+    SmallVector<TrackingMDNodeRef, 4> AllEnumTypes;
     /// Track the RetainTypes, since they can be updated later on.
     SmallVector<TrackingMDNodeRef, 4> AllRetainTypes;
     SmallVector<DISubprogram *, 4> AllSubprograms;
@@ -64,8 +64,8 @@ namespace llvm {
     SmallVector<TrackingMDNodeRef, 4> UnresolvedNodes;
     bool AllowUnresolvedNodes;
 
-    /// Each subprogram's preserved local variables, labels, imported entities,
-    /// and types.
+    /// Each subprogram's preserved local variables, labels and imported
+    /// entities.
     ///
     /// Do not use a std::vector.  Some versions of libc++ apparently copy
     /// instead of move on grow operations, and TrackingMDRef is expensive to
diff --git a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
index 321eae6fcb3b5..0a9a80688a419 100644
--- a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
+++ b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
@@ -548,8 +548,6 @@ class MetadataLoader::MetadataLoaderImpl {
 
   /// Move local imports from DICompileUnit's 'imports' field to
   /// DISubprogram's retainedNodes.
-  /// Move fucntion-local enums from DICompileUnit's enums
-  /// to DISubprogram's retainedNodes.
   void upgradeCULocals() {
     if (NamedMDNode *CUNodes = TheModule.getNamedMetadata("llvm.dbg.cu")) {
       for (unsigned I = 0, E = CUNodes->getNumOperands(); I != E; ++I) {
@@ -557,71 +555,48 @@ class MetadataLoader::MetadataLoaderImpl {
         if (!CU)
           continue;
 
-        SetVector<Metadata *> MetadataToRemove;
-        // Collect imported entities to be moved.
-        if (CU->getRawImportedEntities()) {
+        if (auto *RawImported = CU->getRawImportedEntities()) {
+          // Collect a set of imported entities to be moved.
+          SetVector<Metadata *> EntitiesToRemove;
           for (Metadata *Op : CU->getImportedEntities()->operands()) {
             auto *IE = cast<DIImportedEntity>(Op);
-            if (dyn_cast_or_null<DILocalScope>(IE->getScope())) {
-              MetadataToRemove.insert(IE);
+            if (auto *S = dyn_cast_or_null<DILocalScope>(IE->getScope())) {
+              EntitiesToRemove.insert(IE);
             }
           }
-        }
-        // Collect enums to be moved.
-        if (CU->getRawEnumTypes()) {
-          for (Metadata *Op : CU->getEnumTypes()->operands()) {
-            auto *Enum = cast<DICompositeType>(Op);
-            if (dyn_cast_or_null<DILocalScope>(Enum->getScope())) {
-              MetadataToRemove.insert(Enum);
-            }
-          }
-        }
 
-        if (!MetadataToRemove.empty()) {
-          // Make a new list of CU's 'imports'.
-          SmallVector<Metadata *> NewImports;
-          if (CU->getRawImportedEntities())
-            for (Metadata *Op : CU->getImportedEntities()->operands())
-              if (!MetadataToRemove.contains(Op))
+          if (!EntitiesToRemove.empty()) {
+            // Make a new list of CU's 'imports'.
+            SmallVector<Metadata *> NewImports;
+            for (Metadata *Op : CU->getImportedEntities()->operands()) {
+              if (!EntitiesToRemove.contains(cast<DIImportedEntity>(Op))) {
                 NewImports.push_back(Op);
+              }
+            }
 
-          // Make a new list of CU's 'enums'.
-          SmallVector<Metadata *> NewEnums;
-          if (CU->getRawEnumTypes())
-            for (Metadata *Op : CU->getEnumTypes()->operands())
-              if (!MetadataToRemove.contains(Op))
-                NewEnums.push_back(Op);
-
-          // Find DISubprogram corresponding to each entity.
-          std::map<DISubprogram *, SmallVector<Metadata *>> SPToEntities;
-          for (auto *I : MetadataToRemove) {
-            DILocalScope *Scope = nullptr;
-            if (auto *Entity = dyn_cast<DIImportedEntity>(I))
-              Scope = cast<DILocalScope>(Entity->getScope());
-            else if (auto *Enum = dyn_cast<DICompositeType>(I))
-              Scope = cast<DILocalScope>(Enum->getScope());
-
-            if (auto *SP = findEnclosingSubprogram(Scope)) {
-              SPToEntities[SP].push_back(I);
+            // Find DISubprogram corresponding to each entity.
+            std::map<DISubprogram *, SmallVector<Metadata *>> SPToEntities;
+            for (auto *I : EntitiesToRemove) {
+              auto *Entity = cast<DIImportedEntity>(I);
+              if (auto *SP = findEnclosingSubprogram(
+                      cast<DILocalScope>(Entity->getScope()))) {
+                SPToEntities[SP].push_back(Entity);
+              }
             }
-          }
 
-          // Update DISubprograms' retainedNodes.
-          for (auto I = SPToEntities.begin(); I != SPToEntities.end(); ++I) {
-            auto *SP = I->first;
-            auto RetainedNodes = SP->getRetainedNodes();
-            SmallVector<Metadata *> MDs(RetainedNodes.begin(),
-                                        RetainedNodes.end());
-            MDs.append(I->second);
-            SP->replaceRetainedNodes(MDNode::get(Context, MDs));
-          }
+            // Update DISubprograms' retainedNodes.
+            for (auto I = SPToEntities.begin(); I != SPToEntities.end(); ++I) {
+              auto *SP = I->first;
+              auto RetainedNodes = SP->getRetainedNodes();
+              SmallVector<Metadata *> MDs(RetainedNodes.begin(),
+                                          RetainedNodes.end());
+              MDs.append(I->second);
+              SP->replaceRetainedNodes(MDNode::get(Context, MDs));
+            }
 
-          // Remove entities with local scope from CU.
-          if (CU->getRawImportedEntities())
+            // Remove entities with local scope from CU.
             CU->replaceImportedEntities(MDTuple::get(Context, NewImports));
-          // Remove enums with local scope from CU.
-          if (CU->getRawEnumTypes())
-            CU->replaceEnumTypes(MDTuple::get(Context, NewEnums));
+          }
         }
       }
     }
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
index bf8bdbb7da49f..58ed21379d29b 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
@@ -598,9 +598,10 @@ void DwarfCompileUnit::constructScopeDIE(LexicalScope *Scope,
     return;
 
   // Emit lexical blocks.
-  DIE *ScopeDIE = getOrCreateLexicalBlockDIE(Scope, ParentScopeDIE);
+  DIE *ScopeDIE = constructLexicalScopeDIE(Scope);
   assert(ScopeDIE && "Scope DIE should not be null.");
 
+  ParentScopeDIE.addChild(ScopeDIE);
   createAndAddScopeChildren(Scope, *ScopeDIE);
 }
 
@@ -720,39 +721,24 @@ DIE *DwarfCompileUnit::constructInlinedScopeDIE(LexicalScope *Scope,
   return ScopeDIE;
 }
 
-DIE *DwarfCompileUnit::getOrCreateLexicalBlockDIE(LexicalScope *Scope,
-                                                  DIE &ParentScopeDIE) {
+// Construct new DW_TAG_lexical_block for this scope and attach
+// DW_AT_low_pc/DW_AT_high_pc labels.
+DIE *DwarfCompileUnit::constructLexicalScopeDIE(LexicalScope *Scope) {
   if (DD->isLexicalScopeDIENull(Scope))
     return nullptr;
   const auto *DS = Scope->getScopeNode();
-  DIE *ScopeDIE = nullptr;
-
-  // FIXME: We may have a concrete DIE for this scope already created.
-  // This may happen when we emit local variables for an abstract tree of
-  // an inlined function: if a local variable has a templated type with
-  // a function-local type as a template parameter. See PR55680 for details
-  // (see also llvm/test/DebugInfo/Generic/local-type-as-template-parameter.ll).
-  if (!Scope->isAbstractScope() && !Scope->getInlinedAt()) {
-    if (auto It = LexicalBlockDIEs.find(DS); It != LexicalBlockDIEs.end()) {
-      ScopeDIE = It->second;
-      assert(!ScopeDIE->findAttribute(dwarf::DW_AT_low_pc) &&
-             !ScopeDIE->findAttribute(dwarf::DW_AT_ranges));
-      assert(ScopeDIE->getParent() == &ParentScopeDIE);
-    }
-  }
-  if (!ScopeDIE) {
-    ScopeDIE = DIE::get(DIEValueAllocator, dwarf::DW_TAG_lexical_block);
-    ParentScopeDIE.addChild(ScopeDIE);
-
-    if (Scope->isAbstractScope()) {
-      assert(!getAbstractScopeDIEs().count(DS) &&
-             "Abstract DIE for this scope exists!");
-      getAbstractScopeDIEs()[DS] = ScopeDIE;
-      return ScopeDIE;
-    }
 
-    if (!Scope->getInlinedAt())
-      LexicalBlockDIEs[DS] = ScopeDIE;
+  auto ScopeDIE = DIE::get(DIEValueAllocator, dwarf::DW_TAG_lexical_block);
+  if (Scope->isAbstractScope()) {
+    assert(!getAbstractScopeDIEs().count(DS) &&
+           "Abstract DIE for this scope exists!");
+    getAbstractScopeDIEs()[DS] = ScopeDIE;
+    return ScopeDIE;
+  }
+  if (!Scope->getInlinedAt()) {
+    assert(!LexicalBlockDIEs.count(DS) &&
+           "Concrete out-of-line DIE for this scope exists!");
+    LexicalBlockDIEs[DS] = ScopeDIE;
   }
 
   attachRangesOrLowHighPC(*ScopeDIE, Scope->getRanges());
@@ -1695,21 +1681,15 @@ void DwarfCompileUnit::createBaseTypeDIEs() {
   }
 }
 
-DIE *DwarfCompileUnit::getLocalContextDIE(const DILexicalBlock *LB) {
+DIE *DwarfCompileUnit::getLexicalBlockDIE(const DILexicalBlock *LB) {
   // Assume if there is an abstract tree all the DIEs are already emitted.
   bool isAbstract = getAbstractScopeDIEs().count(LB->getSubprogram());
   if (isAbstract && getAbstractScopeDIEs().count(LB))
     return getAbstractScopeDIEs()[LB];
   assert(!isAbstract && "Missed lexical block DIE in abstract tree!");
 
-  // Check if we have a concrete DIE.
-  if (auto It = LexicalBlockDIEs.find(LB); It != LexicalBlockDIEs.end())
-    return It->second;
-
-  // If nothing available found, we cannot just create a new lexical block,
-  // because it isn't known where to put it into the DIE tree.
-  // So, we may only try to find the most close avaiable parent DIE.
-  return getOrCreateContextDIE(LB->getScope()->getNonLexicalBlockFileScope());
+  // Return a concrete DIE if it exists or nullptr otherwise.
+  return LexicalBlockDIEs.lookup(LB);
 }
 
 DIE *DwarfCompileUnit::getOrCreateContextDIE(const DIScope *Context) {
@@ -1717,7 +1697,7 @@ DIE *DwarfCompileUnit::getOrCreateContextDIE(const DIScope *Context) {
     if (auto *LFScope = dyn_cast<DILexicalBlockFile>(Context))
       Context = LFScope->getNonLexicalBlockFileScope();
     if (auto *LScope = dyn_cast<DILexicalBlock>(Context))
-      return getLocalContextDIE(LScope);
+      return getLexicalBlockDIE(LScope);
 
     // Otherwise the context must be a DISubprogram.
     auto *SPScope = cast<DISubprogram>(Context);
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
index 5662087bec322..6ef73ebd4f7f2 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
@@ -210,9 +210,14 @@ class DwarfCompileUnit final : public DwarfUnit {
   /// DIE to represent this concrete inlined copy of the function.
   DIE *constructInlinedScopeDIE(LexicalScope *Scope, DIE &ParentScopeDIE);
 
-  /// Get if available or create a new DW_TAG_lexical_block for the given
-  /// LexicalScope and attach DW_AT_low_pc/DW_AT_high_pc labels.
-  DIE *getOrCreateLexicalBlockDIE(LexicalScope *Scope, DIE &ParentDIE);
+  /// Construct new DW_TAG_lexical_block for this scope and
+  /// attach DW_AT_low_pc/DW_AT_high_pc labels.
+  DIE *constructLexicalScopeDIE(LexicalScope *Scope);
+
+  /// Get a DIE for the given DILexicalBlock.
+  /// Note that this function assumes that the DIE has been already created
+  /// and it's an error, if it hasn't.
+  DIE *getLexicalBlockDIE(const DILexicalBlock *LB);
 
   /// constructVariableDIE - Construct a DIE for the given DbgVariable.
   DIE *constructVariableDIE(DbgVariable &DV, bool Abstract = false);
@@ -229,11 +234,6 @@ class DwarfCompileUnit final : public DwarfUnit {
   /// This instance of 'getOrCreateContextDIE()' can handle DILocalScope.
   DIE *getOrCreateContextDIE(const DIScope *Ty) override;
 
-  /// Get DW_TAG_lexical_block for the given DILexicalBlock if available,
-  /// or the most close parent DIE, if no correspoding DW_TAG_lexical_block
-  /// exists.
-  DIE *getLocalContextDIE(const DILexicalBlock *LB);
-
   /// Construct a DIE for this subprogram scope.
   DIE &constructSubprogramScopeDIE(const DISubprogram *Sub,
                                    LexicalScope *Scope);
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index 557291c97265f..b8502b455f587 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -1229,13 +1229,12 @@ void DwarfDebug::beginModule(Module *M) {
         CU.getOrCreateGlobalVariableDIE(GV, sortGlobalExprs(GVMap[GV]));
     }
 
-    for (auto *Ty : CUNode->getEnumTypes()) {
-      assert(!isa_and_nonnull<DILocalScope>(Ty->getScope()) &&
-             "Unexpected function-local entity in 'enums' CU field.");
+    for (auto *Ty : CUNode->getEnumTypes())
       CU.getOrCreateTypeDIE(cast<DIType>(Ty));
-    }
 
     for (auto *Ty : CUNode->getRetainedTypes()) {
+      // The retained types array by design contains pointers to
+      // MDNodes rather than DIRefs. Unique them here.
       if (DIType *RT = dyn_cast<DIType>(Ty))
         // There is no point in force-emitting a forward declaration.
         CU.getOrCreateTypeDIE(RT);
@@ -1429,13 +1428,9 @@ void DwarfDebug::endModule() {
              "Unexpected function-local entity in 'imports' CU field.");
       CU->getOrCreateImportedEntityDIE(IE);
     }
-
-    // Emit function-local entities.
     for (const auto *D : CU->getDeferredLocalDecls()) {
       if (auto *IE = dyn_cast<DIImportedEntity>(D))
         CU->getOrCreateImportedEntityDIE(IE);
-      else if (auto *Ty = dyn_cast<DIType>(D))
-        CU->getOrCreateTypeDIE(Ty);
       else
         llvm_unreachable("Unexpected local retained node!");
     }
@@ -1533,8 +1528,6 @@ static const DILocalScope *getRetainedNodeScope(const MDNode *N) {
     S = L->getScope();
   else if (const auto *IE = dyn_cast<DIImportedEntity>(N))
     S = IE->getScope();
-  else if (const auto *T = dyn_cast<DIType>(N))
-    S = T->getScope();
   else
     llvm_unreachable("Unexpected retained node!");
 
diff --git a/llvm/lib/IR/DIBuilder.cpp b/llvm/lib/IR/DIBuilder.cpp
index 242ca44794262..ae1a1a37d7c74 100644
--- a/llvm/lib/IR/DIBuilder.cpp
+++ b/llvm/lib/IR/DIBuilder.cpp
@@ -29,7 +29,7 @@ DIBuilder::DIBuilder(Module &m, bool AllowUnresolvedNodes, DICompileUnit *CU)
       AllowUnresolvedNodes(AllowUnresolvedNodes) {
   if (CUNode) {
     if (const auto &ETs = CUNode->getEnumTypes())
-      EnumTypes.assign(ETs.begin(), ETs.end());
+      AllEnumTypes.assign(ETs.begin(), ETs.end());
     if (const auto &RTs = CUNode->getRetainedTypes())
       AllRetainTypes.assign(RTs.begin(), RTs.end());
     if (const auto &GVs = CUNode->getGlobalVariables())
@@ -66,10 +66,10 @@ void DIBuilder::finalize() {
     return;
   }
 
-  if (!EnumTypes.empty())
+  if (!AllEnumTypes.empty())
     CUNode->replaceEnumTypes(MDTuple::get(
-        VMContext, SmallVector<Metadata *, 16>(EnumTypes.begin(),
-                                               EnumTypes.end())));
+        VMContext, SmallVector<Metadata *, 16>(AllEnumTypes.begin(),
+                                               AllEnumTypes.end())));
 
   SmallVector<Metadata *, 16> RetainValues;
   // Declarations and definitions of the same type may be retained. Some
@@ -334,13 +334,10 @@ DIDerivedType *DIBuilder::createTypedef(DIType *Ty, StringRef Name,
                                         DIScope *Context, uint32_t AlignInBits,
                                         DINode::DIFlags Flags,
                                         DINodeArray Annotations) {
-  auto *T =
-      DIDerivedType::get(VMContext, dwarf::DW_TAG_typedef, Name, File, LineNo,
-                         getNonCompileUnitScope(Context), Ty, 0, AlignInBits, 0,
-                         std::nullopt, Flags, nullptr, Annotations);
-  if (isa_and_nonnull<DILocalScope>(Context))
-    getSubprogramNodesTrackingVector(Context).emplace_back(T);
-  return T;
+  return DIDerivedType::get(VMContext, dwarf::DW_TAG_typedef, Name, File,
+                            LineNo, getNonCompileUnitScope(Context), Ty, 0,
+                            AlignInBits, 0, std::nullopt, Flags, nullptr,
+                            Annotations);
 }
 
 DIDerivedType *DIBuilder::createFriend(DIType *Ty, DIType *FriendTy) {
@@ -488,8 +485,6 @@ DICompositeType *DIBuilder::createClassType(
       OffsetInBits, Flags, Elements, 0, VTableHolder,
       cast_or_null<MDTuple>(TemplateParams), UniqueIdentifier);
   trackIfUnresolved(R);
-  if (isa_and_nonnull<DILocalScope>(Context))
-    getSubprogramNodesTrackingVector(Context).emplace_back(R);
   return R;
 }
 
@@ -503,8 +498,6 @@ DICompositeType *DIBuilder::createStructType(
       getNonCompileUnitScope(Context), DerivedFrom, SizeInBits, AlignInBits, 0,
       Flags, Elements, RunTimeLang, VTableHolder, nullptr, UniqueIdentifier);
   trackIfUnresolved(R);
-  if (isa_and_nonnull<DILocalScope>(Context))
-    getSubprogramNodesTrackingVector(Context).emplace_back(R);
   return R;
 }
 
@@ -517,8 +510,6 @@ DICompositeType *DIBuilder::createUnionType(
       getNonCompileUnitScope(Scope), nullptr, SizeInBits, AlignInBits, 0, Flags,
       Elements, RunTimeLang, nullptr, nullptr, UniqueIdentifier);
   trackIfUnresolved(R);
-  if (isa_and_nonnull<DILocalScope>(Scope))
-    getSubprogramNodesTrackingVector(Scope).emplace_back(R);
   return R;
 }
 
@@ -551,10 +542,7 @@ DICompositeType *DIBuilder::createEnumerationType(
       getNonCompileUnitScope(Scope), UnderlyingType, SizeInBits, AlignInBits, 0,
       IsScoped ? DINode::FlagEnumClass : DINode::FlagZero, Elements, 0, nullptr,
       nullptr, UniqueIdentifier);
-  if (isa_and_nonnull<DILocalScope>(Scope))
-    getSubprogramNodesTrackingVector(Scope).emplace_back(CTy);
-  else
-    EnumTypes.emplace_back(CTy);
+  AllEnumTypes.emplace_back(CTy);
   trackIfUnresolved(CTy);
   return CTy;
 }
@@ -652,8 +640,6 @@ DIBuilder::createForwardDecl(unsigned Tag, StringRef Name, DIScope *Scope,
       SizeInBits, AlignInBits, 0, DINode::FlagFwdDecl, nullptr, RuntimeLang,
       nullptr, nullptr, UniqueIdentifier);
   trackIfUnresolved(RetTy);
-  if (isa_and_nonnull<DILocalScope>(Scope))
-    getSubprogramNodesTrackingVector(Scope).emplace_back(RetTy);
   return RetTy;
 }
 
@@ -670,8 +656,6 @@ DICompositeType *DIBuilder::createReplaceableCompositeType(
           nullptr, Annotations)
           .release();
   trackIfUnresolved(RetTy);
-  if (isa_and_nonnull<DILocalScope>(Scope))
-    getSubprogramNodesTrackingVector(Scope).emplace_back(RetTy);
   return RetTy;
 }
 
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 8f929dc7f48ab..293d443d754a4 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -1384,9 +1384,9 @@ void Verifier::visitDISubprogram(const DISubprogram &N) {
     CheckDI(Node, "invalid retained nodes list", &N, RawNode);
     for (Metadata *Op : Node->operands()) {
       CheckDI(Op && (isa<DILocalVariable>(Op) || isa<DILabel>(Op) ||
-                     isa<DIImportedEntity>(Op) || isa<DIType>(Op)),
-              "invalid retained nodes, expected DILocalVariable, DILabel, "
-              "DIImportedEntity or DIType",
+                     isa<DIImportedEntity>(Op)),
+              "invalid retained nodes, expected DILocalVariable, DILabel or "
+              "DIImportedEntity",
               &N, Node, Op);
     }
   }
diff --git a/llvm/test/Bitcode/upgrade-cu-locals.ll b/llvm/test/Bitcode/upgrade-cu-locals.ll
index 7052ab379bea4..9a590f0fc0774 100644
--- a/llvm/test/Bitcode/upgrade-cu-locals.ll
+++ b/llvm/test/Bitcode/upgrade-cu-locals.ll
@@ -1,4 +1,4 @@
-; Test moving of local imports/enums from DICompileUnit to DISubprogram's 'retainedNodes'
+; Test moving of local imports from DICompileUnit's 'imports' to DISubprogram's 'retainedNodes'
 ;
 ; RUN: llvm-dis -o - %s.bc | FileCheck %s
 
@@ -31,36 +31,31 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo
 ; CHECK: !4 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t4"
 
 ; CHECK: !5 = !{}
-; CHECK: !6 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !7, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !8, imports: !5, nameTableKind: GNU)
-; CHECK: !7 = !DIFile(filename: "b.cpp"
-; CHECK: !8 = !{!9}
-; CHECK: !9 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum2", scope: !6, file: !7, line: 4, size: 8, align: 8, elements: !5)
-
-; CHECK: !16 = distinct !DISubprogram(name: "main", scope: !7, file: !7, line: 2, type: !17, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !20)
-; CHECK: !20 = !{!21}
-; CHECK: !21 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !22, entity: !25,
-; CHECK: !22 = !DILexicalBlock(scope: !23, file: !7, line: 7, column: 35)
-; CHECK: !23 = !DILexicalBlock(scope: !24, file: !7, line: 7, column: 35)
-; CHECK: !24 = !DILexicalBlock(scope: !16, file: !7, line: 7, column: 35)
-; CHECK: !25 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t5", scope: !22,
-
-; CHECK: !27 = distinct !DISubprogram(name: "f1", linkageName: "_Z2f1v", scope: !1, file: !1, line: 3, type: !28, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !30)
-; CHECK: !30 = !{!31, !34, !36}
-; CHECK: !31 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !27, entity: !32,
-; CHECK: !32 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t1",
-; CHECK: !34 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !27, entity: !35,
-; CHECK: !35 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t2",
-; CHECK: !36 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !27, entity: !37,
-; CHECK: !37 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t3",
-
-; CHECK: !42 = distinct !DISubprogram(name: "main2", scope: !7, file: !7, line: 10, type: !17, scopeLine: 10, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !43)
-; CHECK: !43 = !{!44, !46, !48, !49}
-; CHECK: !44 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !42, entity: !45,
-; CHECK: !45 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t6"
-; CHECK: !46 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !42, entity: !47,
-; CHECK: !47 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t7",
-; CHECK: !48 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum", scope: !42, file: !7, line: 3, size: 8, align: 8, elements: !5)
-; CHECK: !49 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum3", scope: !42, file: !7, line: 5, size: 8, align: 8, elements: !5)
+; CHECK: !6 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !7, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, imports: !5, nameTableKind: GNU)
+
+; CHECK: !14 = distinct !DISubprogram(name: "main", scope: !7, file: !7, line: 2, type: !15, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !18)
+; CHECK: !18 = !{!19}
+; CHECK: !19 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !20, entity: !23,
+; CHECK: !20 = !DILexicalBlock(scope: !21, file: !7, line: 7, column: 35)
+; CHECK: !21 = !DILexicalBlock(scope: !22, file: !7, line: 7, column: 35)
+; CHECK: !22 = !DILexicalBlock(scope: !14, file: !7, line: 7, column: 35)
+; CHECK: !23 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t5", scope: !20,
+
+; CHECK: !25 = distinct !DISubprogram(name: "f1", linkageName: "_Z2f1v", scope: !1, file: !1, line: 3, type: !26, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !28)
+; CHECK: !28 = !{!29, !32, !34}
+; CHECK: !29 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !25, entity: !30,
+; CHECK: !30 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t1",
+; CHECK: !32 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !25, entity: !33,
+; CHECK: !33 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t2",
+; CHECK: !34 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !25, entity: !35,
+; CHECK: !35 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t3",
+
+; CHECK: !40 = distinct !DISubprogram(name: "main2", scope: !7, file: !7, line: 10, type: !15, scopeLine: 10, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !41)
+; CHECK: !41 = !{!42, !44}
+; CHECK: !42 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !40, entity: !43,
+; CHECK: !43 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t6"
+; CHECK: !44 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !40, entity: !45,
+; CHECK: !45 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t7",
 
 
 !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, imports: !2, nameTableKind: GNU)
@@ -87,7 +82,7 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo
 ; Leave t4 in CU
 !14 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !0, entity: !15, file: !1, line: 3)
 !15 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t4", scope: !0, file: !1, line: 1, size: 8, flags: DIFlagTypePassByValue, elements: !7, identifier: "_ZTSN2ns2t4E")
-!16 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !17, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, imports: !18, enums: !50, nameTableKind: GNU)
+!16 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !17, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, imports: !18, nameTableKind: GNU)
 !17 = !DIFile(filename: "b.cpp", directory: "/")
 !18 = !{!19, !28, !31}
 
@@ -121,12 +116,3 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo
 !41 = distinct !DILocation(line: 3, column: 3, scope: !23)
 !42 = !DILocation(line: 3, column: 41, scope: !4, inlinedAt: !41)
 !43 = !DILocation(line: 4, column: 1, scope: !23)
-
-!50 = !{!51, !52, !53}
-; Move to main2
-!51 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum", scope: !29, file: !17, line: 3, size: 8, align: 8, elements: !7)
-; Leave in b.cpp's CU
-!52 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum2", scope: !16, file: !17, line: 4, size: 8, align: 8, elements: !7)
-; Move to main2
-!53 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum3", scope: !29, file: !17, line: 5, size: 8, align: 8, elements: !7)
-
diff --git a/llvm/test/Bitcode/upgrade-cu-locals.ll.bc b/llvm/test/Bitcode/upgrade-cu-locals.ll.bc
index f73d05668af5b16f9ee03e400e2877faab339a0d..9d0ea8fd9a3704ea1fb3a92250500addc75f0587 100644
GIT binary patch
delta 1011
zcmYjPT}&fY6h1R;?+nAR8K7ukpbT3~NOmQ+|Ckt^#_fv6bXy??4aQDuk;YOAbd5%0
zrv<}iQ#P<|%$oR6v-sc!%?9*sAEv1*sKCpvhU}79AB@J-1png8D&A>B@aE3>`OdxH
z`DW%Ebf_IG{nX^@Bi~TU>Tgesr^SEYyYtha=j~yWY!HBg<7MPOE-e79rviCYP}RO~
z{gy^qhXoMu+C9!tM`Yd7K=*rCbF3X;n<h+~OH`|YIw-tvv~}+*HOenH<t0qpa4y8=
zHS>ZdEU$I902B_4N`Kn>%1fgo0Y`Gqk@ou2o|DYupHkvUM?9G>Oq(ubP_5n(a4sm=
zSie0NwAk3&#7VXnE9Y+cR78YGZ0=kSKu|xb4S$DwriFb<1z>Yhhyjpv9A9-m_myaT
zL(g*3@Li&q)A9VPEq8uL*&bj)mDsk61ggK}NN7uR?PWy%O-xFsCjs;lE_N)IMV>_D
zKJ<VL#yUlNktuhbP{;%YAPW0`^xgcQLag7?M(?gN<`w4afdQb^OCrcbyRM_~7ovJk
zNHh~b24E!gv^6YS+3S+7BYC33XFb+2(9pq;LKfhUKA1%l7^Y!o3AQ_nWw=G*URruh
zMutPA&9iE$Y~eK#oI*-qW-;T0g%$|!2;gj`Q;a5?4q3o2P@nm%-O&sQe%1rxYjsbU
z=Cxy{^=ua}TlJP3D30MR*uj5hdmoe^XJpi20TX^o;0UY4X?z)1s<H2RgJn{|6$#z<
zR;n`*f?W)!5|HZNntA7khsjPT59ziZB&B9$Lu%dtZpAW|tfCTdC6Y?b=`i<!I2Ng*
zNfkNTu9e33_iOjYx4+mM|MHjft$U@G=auN@>vPZ8&dgmG{g!t_+n1JiKi&Q4*X8vq
zpX}VuKUzy9i#hYNWpjYqoMKH>Ktt-x<rKPch!#4@&scBPKP}i01Gz9h`{B{i(UTAU
zA-4KAF_pZ}UMzRsq;W(hvxgQP(I6jBp>;^$Xi)Hp1+A%zQVaK+&J4<A;u~v5YD|{n
zlyG{H{@geu1nPyCNvpFYYBLlg1s69s5SfVLsK@JZx`(jG?sT~nm&;-I+LfWlT#gBk
cSBb^kZo4~HxXhgw;!~;V+4S6CBGKsbANlGKqW}N^

delta 1080
zcmYk5e`p(J7{{NxT<<RFwOop6V|rQMH7e;k%bOqVBKYItmQkU&t8>^v2T79^Yj3eX
zCa4ST^$4!hY-CFq`$I=`D*iELuKy_e=R!ISm`cGRL56hTA0ibxhWMA%_q{6H%X`oB
z-1ohE-p}*!zJ}+nr}PA4Tzv8e=ETMO&yU}{_3PDJt*yNt=um-!|A&k}H9J|6zN*x)
zz`pltPnw1RR8~Q?N^cBEwg8N+ilr9PDjEQ4=T<*p+z#fi@<&_{iTssC`dw0W77=EH
z*c_9SV-gA2gRx8VvGQFm-JQ(6b&5U_YNDso)49lW?sWONHl>_yJF*18yXh_f*wo~p
zm<*emHz?jF4h4_b4>b}uctVLpliLG87yzu1pRs>l{*5sRvWb=j0!DRwX=wDE!o5S^
zOf<4d(z>W#%y#_LCnWZe`V1s?y-g$s)sK#yRhD8G6zL2mOMNo{bO~2Hg=K}-PyV<3
zq#>kxqe*`rGx=)KWD%^aF+h`3j$?WG#v>f#G%w}o32^!&9A2suj-yWr%l!{I44%|a
zL4<$~0r4859CHsjn1Ia@0;sw`ra%slionW?#uo{nWGL`pFMPtkj$s~}XJN(nVGSA#
z4zOq@#XFE-No&eR!r!V%R>}e1C})ip;CBXV%(E=PadK-*f}<Bi)~=kQK=3&0zR$P?
zFab!z?O}~vR!8z0DHMT0x@`|!-w+nJg$D{euBG5HOG<>AheRU;84oDiwJ=Y7|E^bU
zPf#aaZ~liW_I^`?Z3Zt9R(*qaScJ9SUwv_3Q<kDq+B#R0>fX~?T-W%k-X<>8UgL=j
zd}ekQW|i;f7Vq_tsi(X3-JsT;L+v-o1XsyBp4;T7(%n-D;dAR-CS<UohPl7N`l(K4
zr0Ys`qSe~^Wumbb`*GsCKMuR^K<6vz%<9a-OTmGKYrRRW?O0>w{Q6hxpZ$6M`h_o>
zw~J42>AA|JXZ&N&#J$xHv5k3on>Xg(ukv3b;h>Ln<a)b(XM#LwT?+4icV}nkh0TA6
zo&B4bdNC2MEc==)o>a+U<_nRGCQVoQRnT!plNqI)a{8Ff@)_sAV5(!c!vJ$=_ozlX
z<-P*D&c#bd?4=WAs{s;qIV!309p_*u{7g7Dd~zy26wOR#W081lI2)ci5Q#<(#PJXw
bjz`1MY<74kUf$<AES)45abZx`X{vt#JnkA(

diff --git a/llvm/test/DebugInfo/Generic/inlined-local-type.ll b/llvm/test/DebugInfo/Generic/inlined-local-type.ll
deleted file mode 100644
index a6f7d294de928..0000000000000
--- a/llvm/test/DebugInfo/Generic/inlined-local-type.ll
+++ /dev/null
@@ -1,128 +0,0 @@
-; RUN: %llc_dwarf -O0 -filetype=obj < %s | llvm-dwarfdump -debug-info - | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s
-; REQUIRES: object-emission
-
-; inline __attribute__((always_inline))
-; int removed() { struct A {int i;}; struct A a; return a.i++; }
-;
-; __attribute__((always_inline))
-; int not_removed() { struct B {int i;}; struct B b; return b.i++; }
-;
-; int foo() { return removed() + not_removed(); }}
-
-; Ensure that function-local types have the correct subprogram parent even if
-; those subprograms are inlined.
-
-; CHECK: DW_TAG_compile_unit
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_AT_abstract_origin	({{0x.*}} "not_removed")
-; CHECK:     DW_TAG_variable
-; CHECK:     NULL
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_AT_name	("removed")
-; CHECK: [[A:0x.*]]: DW_TAG_structure_type
-; CHECK:       DW_AT_name	("A")
-; CHECK:       DW_TAG_member
-; CHECK:       NULL
-; CHECK:     DW_TAG_variable
-; CHECK:       DW_AT_type	([[A]] "A")
-; CHECK:     NULL
-; CHECK:   DW_TAG_base_type
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_AT_name	("not_removed")
-; CHECK: [[B:0x.*]]: DW_TAG_structure_type
-; CHECK:       DW_AT_name	("B")
-; CHECK:       DW_TAG_member
-; CHECK:       NULL
-; CHECK:     DW_TAG_variable
-; CHECK:       DW_AT_type	([[B]] "B")
-; CHECK:     NULL
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_TAG_inlined_subroutine
-; CHECK:       DW_TAG_variable
-; CHECK:       NULL
-; CHECK:     DW_TAG_inlined_subroutine
-; CHECK:       DW_TAG_variable
-; CHECK:       NULL
-; CHECK:     NULL
-; CHECK:   NULL
-
-%struct.B = type { i32 }
-%struct.A = type { i32 }
-
-define dso_local i32 @not_removed() !dbg !12 {
-  %1 = alloca %struct.B, align 4
-  call void @llvm.dbg.declare(metadata %struct.B* %1, metadata !18, metadata !DIExpression()), !dbg !22
-  %2 = getelementptr inbounds %struct.B, %struct.B* %1, i32 0, i32 0, !dbg !23
-  %3 = load i32, i32* %2, align 4, !dbg !24
-  %4 = add nsw i32 %3, 1, !dbg !24
-  store i32 %4, i32* %2, align 4, !dbg !24
-  ret i32 %3, !dbg !25
-}
-
-declare void @llvm.dbg.declare(metadata, metadata, metadata)
-
-define dso_local i32 @foo() !dbg !26 {
-  %1 = alloca %struct.A, align 4
-  %2 = alloca %struct.B, align 4
-  call void @llvm.dbg.declare(metadata %struct.A* %1, metadata !27, metadata !DIExpression()), !dbg !32
-  %3 = getelementptr inbounds %struct.A, %struct.A* %1, i32 0, i32 0, !dbg !34
-  %4 = load i32, i32* %3, align 4, !dbg !35
-  %5 = add nsw i32 %4, 1, !dbg !35
-  store i32 %5, i32* %3, align 4, !dbg !35
-  call void @llvm.dbg.declare(metadata %struct.B* %2, metadata !18, metadata !DIExpression()), !dbg !36
-  %6 = getelementptr inbounds %struct.B, %struct.B* %2, i32 0, i32 0, !dbg !38
-  %7 = load i32, i32* %6, align 4, !dbg !39
-  %8 = add nsw i32 %7, 1, !dbg !39
-  store i32 %8, i32* %6, align 4, !dbg !39
-  %9 = add nsw i32 %4, %7, !dbg !40
-  ret i32 %9, !dbg !41
-}
-
-!llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!2, !3, !4, !5, !6, !7, !8, !9, !10}
-!llvm.ident = !{!11}
-
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
-!1 = !DIFile(filename: "inlined-local-type.cpp", directory: "")
-!2 = !{i32 7, !"Dwarf Version", i32 4}
-!3 = !{i32 2, !"Debug Info Version", i32 3}
-!4 = !{i32 1, !"wchar_size", i32 4}
-!5 = !{i32 1, !"branch-target-enforcement", i32 0}
-!6 = !{i32 1, !"sign-return-address", i32 0}
-!7 = !{i32 1, !"sign-return-address-all", i32 0}
-!8 = !{i32 1, !"sign-return-address-with-bkey", i32 0}
-!9 = !{i32 7, !"uwtable", i32 1}
-!10 = !{i32 7, !"frame-pointer", i32 1}
-!11 = !{!"clang version 14.0.0"}
-!12 = distinct !DISubprogram(name: "not_removed", scope: !13, file: !13, line: 5, type: !14, scopeLine: 5, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !43)
-!13 = !DIFile(filename: "inlined-local-type.cpp", directory: "")
-!14 = !DISubroutineType(types: !15)
-!15 = !{!16}
-!16 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-!17 = !{}
-!18 = !DILocalVariable(name: "b", scope: !12, file: !13, line: 5, type: !19)
-!19 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "B", scope: !12, file: !13, line: 5, size: 32, elements: !20)
-!20 = !{!21}
-!21 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !19, file: !13, line: 5, baseType: !16, size: 32)
-!22 = !DILocation(line: 5, column: 49, scope: !12)
-!23 = !DILocation(line: 5, column: 61, scope: !12)
-!24 = !DILocation(line: 5, column: 62, scope: !12)
-!25 = !DILocation(line: 5, column: 52, scope: !12)
-!26 = distinct !DISubprogram(name: "foo", scope: !13, file: !13, line: 7, type: !14, scopeLine: 7, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !17)
-!27 = !DILocalVariable(name: "a", scope: !28, file: !13, line: 2, type: !29)
-!28 = distinct !DISubprogram(name: "removed", scope: !13, file: !13, line: 2, type: !14, scopeLine: 2, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !42)
-!29 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A", scope: !28, file: !13, line: 2, size: 32, elements: !30)
-!30 = !{!31}
-!31 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !29, file: !13, line: 2, baseType: !16, size: 32)
-!32 = !DILocation(line: 2, column: 45, scope: !28, inlinedAt: !33)
-!33 = distinct !DILocation(line: 7, column: 20, scope: !26)
-!34 = !DILocation(line: 2, column: 57, scope: !28, inlinedAt: !33)
-!35 = !DILocation(line: 2, column: 58, scope: !28, inlinedAt: !33)
-!36 = !DILocation(line: 5, column: 49, scope: !12, inlinedAt: !37)
-!37 = distinct !DILocation(line: 7, column: 32, scope: !26)
-!38 = !DILocation(line: 5, column: 61, scope: !12, inlinedAt: !37)
-!39 = !DILocation(line: 5, column: 62, scope: !12, inlinedAt: !37)
-!40 = !DILocation(line: 7, column: 30, scope: !26)
-!41 = !DILocation(line: 7, column: 13, scope: !26)
-!42 = !{!29}
-!43 = !{!19}
diff --git a/llvm/test/DebugInfo/Generic/lexical-block-retained-types.ll b/llvm/test/DebugInfo/Generic/lexical-block-retained-types.ll
deleted file mode 100644
index 64de1323c98c6..0000000000000
--- a/llvm/test/DebugInfo/Generic/lexical-block-retained-types.ll
+++ /dev/null
@@ -1,55 +0,0 @@
-; RUN: %llc_dwarf -O0 -filetype=obj < %s | llvm-dwarfdump - | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s
-
-; Test that retained unused (unreferenced) types emission.
-
-; Compiled from
-; $ clang -cc1 -debug-info-kind=unused-types test.cpp -emit-llvm
-
-; void test_unused() {
-;   struct Y {};
-;   {
-;     struct X {};
-;   }
-; }
-
-; CHECK: DW_TAG_compile_unit
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_AT_name	("test_unused")
-; CHECK:     DW_TAG_structure_type
-; CHECK:       DW_AT_name	("Y")
-
-; FIXME: here should be DW_TAG_lexical_block as a parent of structure 'X'.
-; But it's not possible to reliably emit a lexical block for which a LexicalScope
-; wasn't created, so we just fallback to the most close parent DIE
-; (see DwarfCompileUnit::getOrCreateLexicalBlockDIE() for details).
-
-; CHECK:     DW_TAG_structure_type
-; CHECK:       DW_AT_name	("X")
-; CHECK:     NULL
-; CHECK:   NULL
-
-define dso_local void @_Z11test_unusedv() !dbg !5 {
-entry:
-  ret void, !dbg !16
-}
-
-!llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!13, !14}
-!llvm.ident = !{!15}
-
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 15.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, retainedTypes: !2, splitDebugInlining: false, nameTableKind: None)
-!1 = !DIFile(filename: "<stdin>", directory: "/")
-!2 = !{!3, !10}
-!3 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Y", scope: !5, file: !4, line: 2, size: 8, flags: DIFlagTypePassByValue, elements: !8)
-!4 = !DIFile(filename: "test.cpp", directory: "/")
-!5 = distinct !DISubprogram(name: "test_unused", linkageName: "_Z11test_unusedv", scope: !4, file: !4, line: 1, type: !6, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !9)
-!6 = !DISubroutineType(types: !7)
-!7 = !{null}
-!8 = !{}
-!9 = !{!3, !10}
-!10 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "X", scope: !11, file: !4, line: 4, size: 8, flags: DIFlagTypePassByValue, elements: !8)
-!11 = distinct !DILexicalBlock(scope: !5, file: !4, line: 3, column: 3)
-!13 = !{i32 2, !"Debug Info Version", i32 3}
-!14 = !{i32 1, !"wchar_size", i32 4}
-!15 = !{!"clang version 15.0.0"}
-!16 = !DILocation(line: 6, column: 1, scope: !5)
diff --git a/llvm/test/DebugInfo/Generic/lexical-block-types.ll b/llvm/test/DebugInfo/Generic/lexical-block-types.ll
deleted file mode 100644
index aef59984351a7..0000000000000
--- a/llvm/test/DebugInfo/Generic/lexical-block-types.ll
+++ /dev/null
@@ -1,425 +0,0 @@
-; RUN: %llc_dwarf -O0 -filetype=obj < %s | llvm-dwarfdump -debug-info - | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s
-; REQUIRES: object-emission
-
-; inline __attribute__((always_inline))
-; void removed() {
-;   struct A1 { int i; };
-;   typedef int Int1;
-;   {
-;     struct I1 { Int1 j; };
-;     struct C1 { typedef char Char1; Char1 c; };
-;     A1 a1; a1.i++;
-;     {
-;       I1 i1; i1.j++;
-;       C1 c1; c1.c++;
-;     }
-;   }
-; }
-;
-; __attribute__((always_inline))
-; void not_removed() {
-;   struct A2 { int i; };
-;   typedef int Int2;
-;   {
-;     struct I2 { Int2 j; };
-;     struct C2 { typedef char Char2; Char2 c; };
-;     A2 a2; a2.i++;
-;     {
-;       I2 i2; i2.j++;
-;       C2 c2; c2.c++;
-;     }
-;   }
-; }
-;
-; void foo() {
-;   struct A3 { int i; };
-;   typedef int Int3;
-;   {
-;     struct I3 { Int3 j; };
-;     {
-;       struct C3 { typedef char Char3; Char3 c; };
-;       A3 a3; a3.i++;
-;       {
-;         I3 i3; i3.j++;
-;         C3 c3; c3.c++;
-;       }
-;     }
-;   }
-;   removed();
-;   not_removed();
-; }
-;
-; CHECK: DW_TAG_compile_unit
-
-; Out-of-line definition of `not_removed()` shouldn't contain any debug info for types.
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_AT_abstract_origin	{{.*}} "_Z11not_removedv"
-; CHECK:     DW_TAG_lexical_block
-; CHECK:       DW_TAG_variable
-; CHECK:         DW_AT_abstract_origin	{{.*}} "a2"
-; CHECK:       DW_TAG_lexical_block
-; CHECK:         DW_TAG_variable
-; CHECK:           DW_AT_abstract_origin	{{.*}} "i2"
-; CHECK:         DW_TAG_variable
-; CHECK:           DW_AT_abstract_origin	{{.*}} "c2"
-; CHECK:         NULL
-; CHECK:       NULL
-; CHECK:     NULL
-
-; Abstract definition of `removed()`.
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_AT_name	("removed")
-; CHECK:     DW_AT_inline	(DW_INL_inlined)
-
-; I1 and C1 defined in the first lexical block, typedef Char1 is a child of C1.
-; CHECK:     DW_TAG_lexical_block
-; CHECK:       DW_TAG_variable
-; CHECK:         DW_AT_name	("a1")
-; CHECK:         DW_AT_type	{{.*}} "A1"
-; CHECK:       DW_TAG_lexical_block
-; CHECK:         DW_TAG_variable
-; CHECK:           DW_AT_type	{{.*}} "I1"
-; CHECK:         DW_TAG_variable
-; CHECK:           DW_AT_type	{{.*}} "C1"
-; CHECK:         NULL
-; CHECK:       DW_TAG_structure_type
-; CHECK:         DW_AT_name	("I1")
-; CHECK:         DW_TAG_member
-; CHECK:           DW_AT_type	{{.*}} "Int1"
-; CHECK:         NULL
-; CHECK:       DW_TAG_structure_type
-; CHECK:         DW_AT_name	("C1")
-; CHECK:         DW_TAG_member
-; CHECK:           DW_AT_type	{{.*}} "Char1"
-; CHECK:         DW_TAG_typedef
-; CHECK:           DW_AT_name	("Char1")
-; CHECK:         NULL
-; CHECK:       NULL
-
-; A1 and typedef Int1 defined in the subprogram scope.
-; CHECK:     DW_TAG_structure_type
-; CHECK:       DW_AT_name	("A1")
-; CHECK:       DW_TAG_member
-; CHECK:       NULL
-; CHECK:     DW_TAG_typedef
-; CHECK:       DW_AT_name	("Int1")
-; CHECK:     NULL
-
-; CHECK:   DW_TAG_base_type
-; CHECK:   DW_TAG_base_type
-
-; Abstract definition of `not_removed()`.
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_AT_name	("not_removed")
-; CHECK:     DW_AT_inline	(DW_INL_inlined)
-
-; I2 and C2 defined in the first lexical block, typedef Char2 is a child of C2.
-; CHECK:     DW_TAG_lexical_block
-; CHECK:       DW_TAG_variable
-; CHECK:         DW_AT_name	("a2")
-; CHECK:         DW_AT_type	{{.*}} "A2"
-; CHECK:       DW_TAG_lexical_block
-; CHECK:         DW_TAG_variable
-; CHECK:           DW_AT_name	("i2")
-; CHECK:           DW_AT_type	{{.*}} "I2"
-; CHECK:         DW_TAG_variable
-; CHECK:           DW_AT_name	("c2")
-; CHECK:           DW_AT_type	{{.*}} "C2"
-; CHECK:         NULL
-; CHECK:       DW_TAG_structure_type
-; CHECK:         DW_AT_name	("I2")
-; CHECK:         DW_TAG_member
-; CHECK:           DW_AT_type	{{.*}} "Int2"
-; CHECK:         NULL
-; CHECK:       DW_TAG_structure_type
-; CHECK:         DW_AT_name	("C2")
-; CHECK:         DW_TAG_member
-; CHECK:           DW_AT_type	{{.*}} "Char2"
-; CHECK:         DW_TAG_typedef
-; CHECK:           DW_AT_name	("Char2")
-; CHECK:         NULL
-; CHECK:       NULL
-
-; A2 and typedef Int2 defined in subprogram scope.
-; CHECK:     DW_TAG_structure_type
-; CHECK:       DW_AT_name	("A2")
-; CHECK:       DW_TAG_member
-; CHECK:       NULL
-; CHECK:     DW_TAG_typedef
-; CHECK:       DW_AT_name	("Int2")
-; CHECK:     NULL
-
-; Definition of `foo()`.
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_AT_name	("foo")
-
-; CHECK:     DW_TAG_lexical_block
-; CHECK:       DW_TAG_lexical_block
-; CHECK:         DW_TAG_variable
-; CHECK:           DW_AT_name	("a3")
-; CHECK:           DW_AT_type	{{.*}} "A3"
-; CHECK:         DW_TAG_lexical_block
-; CHECK:           DW_TAG_variable
-; CHECK:             DW_AT_name	("i3")
-; CHECK:             DW_AT_type	{{.*}} "I3"
-; CHECK:           DW_TAG_variable
-; CHECK:             DW_AT_name	("c3")
-; CHECK:             DW_AT_type	{{.*}} "C3"
-; CHECK:           NULL
-
-; C3 has the inner lexical block scope, typedef Char3 is a child of C3.
-; CHECK:         DW_TAG_structure_type
-; CHECK:           DW_AT_name	("C3")
-; CHECK:           DW_TAG_member
-; CHECK:             DW_AT_type	{{.*}} "Char3"
-; CHECK:           DW_TAG_typedef
-; CHECK:             DW_AT_name	("Char3")
-; CHECK:           NULL
-; CHECK:         NULL
-
-; I3 has the outer lexical block scope.
-; CHECK:       DW_TAG_structure_type
-; CHECK:         DW_AT_name	("I3")
-; CHECK:         DW_TAG_member
-; CHECK:           DW_AT_type	{{.*}} "Int3"
-; CHECK:         NULL
-; CHECK:       NULL
-
-; CHECK:     DW_TAG_inlined_subroutine
-; CHECK:       DW_AT_abstract_origin	{{.*}} "_Z7removedv"
-; CHECK:       DW_TAG_lexical_block
-; CHECK:         DW_TAG_variable
-; CHECK:         DW_TAG_lexical_block
-; CHECK:           DW_TAG_variable
-; CHECK:           DW_TAG_variable
-; CHECK:           NULL
-; CHECK:         NULL
-; CHECK:       NULL
-
-; CHECK:     DW_TAG_inlined_subroutine
-; CHECK:       DW_AT_abstract_origin	{{.*}} "_Z11not_removedv"
-; CHECK:       DW_TAG_lexical_block
-; CHECK:         DW_TAG_variable
-; CHECK:         DW_TAG_lexical_block
-; CHECK:           DW_TAG_variable
-; CHECK:           DW_TAG_variable
-; CHECK:           NULL
-; CHECK:         NULL
-; CHECK:       NULL
-
-; A3 and Int3 defined within the subprogam scope.
-; CHECK:     DW_TAG_structure_type
-; CHECK:       DW_AT_name	("A3")
-; CHECK:       DW_TAG_member
-; CHECK:       NULL
-; CHECK:     DW_TAG_typedef
-; CHECK:       DW_AT_name	("Int3")
-; CHECK:     NULL
-; CHECK:   NULL
-
-%struct.A2 = type { i32 }
-%struct.I2 = type { i32 }
-%struct.C2 = type { i8 }
-%struct.A1 = type { i32 }
-%struct.I1 = type { i32 }
-%struct.C1 = type { i8 }
-%struct.A3 = type { i32 }
-%struct.I3 = type { i32 }
-%struct.C3 = type { i8 }
-
-define dso_local void @_Z11not_removedv() !dbg !8 {
-entry:
-  %a2 = alloca %struct.A2, align 4
-  %i2 = alloca %struct.I2, align 4
-  %c2 = alloca %struct.C2, align 1
-  call void @llvm.dbg.declare(metadata %struct.A2* %a2, metadata !12, metadata !DIExpression()), !dbg !18
-  %i = getelementptr inbounds %struct.A2, %struct.A2* %a2, i32 0, i32 0, !dbg !19
-  %0 = load i32, i32* %i, align 4, !dbg !20
-  %inc = add nsw i32 %0, 1, !dbg !20
-  store i32 %inc, i32* %i, align 4, !dbg !20
-  call void @llvm.dbg.declare(metadata %struct.I2* %i2, metadata !21, metadata !DIExpression()), !dbg !27
-  %j = getelementptr inbounds %struct.I2, %struct.I2* %i2, i32 0, i32 0, !dbg !28
-  %1 = load i32, i32* %j, align 4, !dbg !29
-  %inc1 = add nsw i32 %1, 1, !dbg !29
-  store i32 %inc1, i32* %j, align 4, !dbg !29
-  call void @llvm.dbg.declare(metadata %struct.C2* %c2, metadata !30, metadata !DIExpression()), !dbg !36
-  %c = getelementptr inbounds %struct.C2, %struct.C2* %c2, i32 0, i32 0, !dbg !37
-  %2 = load i8, i8* %c, align 1, !dbg !38
-  %inc2 = add i8 %2, 1, !dbg !38
-  store i8 %inc2, i8* %c, align 1, !dbg !38
-  ret void, !dbg !39
-}
-
-declare void @llvm.dbg.declare(metadata, metadata, metadata)
-
-define dso_local void @_Z3foov() !dbg !40 {
-entry:
-  %a1.i = alloca %struct.A1, align 4
-  %i1.i = alloca %struct.I1, align 4
-  %c1.i = alloca %struct.C1, align 1
-  %a2.i = alloca %struct.A2, align 4
-  %i2.i = alloca %struct.I2, align 4
-  %c2.i = alloca %struct.C2, align 1
-  %a3 = alloca %struct.A3, align 4
-  %i3 = alloca %struct.I3, align 4
-  %c3 = alloca %struct.C3, align 1
-  call void @llvm.dbg.declare(metadata %struct.A3* %a3, metadata !41, metadata !DIExpression()), !dbg !47
-  %i = getelementptr inbounds %struct.A3, %struct.A3* %a3, i32 0, i32 0, !dbg !48
-  %0 = load i32, i32* %i, align 4, !dbg !49
-  %inc = add nsw i32 %0, 1, !dbg !49
-  store i32 %inc, i32* %i, align 4, !dbg !49
-  call void @llvm.dbg.declare(metadata %struct.I3* %i3, metadata !50, metadata !DIExpression()), !dbg !56
-  %j = getelementptr inbounds %struct.I3, %struct.I3* %i3, i32 0, i32 0, !dbg !57
-  %1 = load i32, i32* %j, align 4, !dbg !58
-  %inc1 = add nsw i32 %1, 1, !dbg !58
-  store i32 %inc1, i32* %j, align 4, !dbg !58
-  call void @llvm.dbg.declare(metadata %struct.C3* %c3, metadata !59, metadata !DIExpression()), !dbg !64
-  %c = getelementptr inbounds %struct.C3, %struct.C3* %c3, i32 0, i32 0, !dbg !65
-  %2 = load i8, i8* %c, align 1, !dbg !66
-  %inc2 = add i8 %2, 1, !dbg !66
-  store i8 %inc2, i8* %c, align 1, !dbg !66
-  call void @llvm.dbg.declare(metadata %struct.A1* %a1.i, metadata !67, metadata !DIExpression()), !dbg !73
-  %i.i3 = getelementptr inbounds %struct.A1, %struct.A1* %a1.i, i32 0, i32 0, !dbg !75
-  %3 = load i32, i32* %i.i3, align 4, !dbg !76
-  %inc.i4 = add nsw i32 %3, 1, !dbg !76
-  store i32 %inc.i4, i32* %i.i3, align 4, !dbg !76
-  call void @llvm.dbg.declare(metadata %struct.I1* %i1.i, metadata !77, metadata !DIExpression()), !dbg !83
-  %j.i5 = getelementptr inbounds %struct.I1, %struct.I1* %i1.i, i32 0, i32 0, !dbg !84
-  %4 = load i32, i32* %j.i5, align 4, !dbg !85
-  %inc1.i6 = add nsw i32 %4, 1, !dbg !85
-  store i32 %inc1.i6, i32* %j.i5, align 4, !dbg !85
-  call void @llvm.dbg.declare(metadata %struct.C1* %c1.i, metadata !86, metadata !DIExpression()), !dbg !91
-  %c.i7 = getelementptr inbounds %struct.C1, %struct.C1* %c1.i, i32 0, i32 0, !dbg !92
-  %5 = load i8, i8* %c.i7, align 1, !dbg !93
-  %inc2.i8 = add i8 %5, 1, !dbg !93
-  store i8 %inc2.i8, i8* %c.i7, align 1, !dbg !93
-  call void @llvm.dbg.declare(metadata %struct.A2* %a2.i, metadata !12, metadata !DIExpression()), !dbg !94
-  %i.i = getelementptr inbounds %struct.A2, %struct.A2* %a2.i, i32 0, i32 0, !dbg !96
-  %6 = load i32, i32* %i.i, align 4, !dbg !97
-  %inc.i = add nsw i32 %6, 1, !dbg !97
-  store i32 %inc.i, i32* %i.i, align 4, !dbg !97
-  call void @llvm.dbg.declare(metadata %struct.I2* %i2.i, metadata !21, metadata !DIExpression()), !dbg !98
-  %j.i = getelementptr inbounds %struct.I2, %struct.I2* %i2.i, i32 0, i32 0, !dbg !99
-  %7 = load i32, i32* %j.i, align 4, !dbg !100
-  %inc1.i = add nsw i32 %7, 1, !dbg !100
-  store i32 %inc1.i, i32* %j.i, align 4, !dbg !100
-  call void @llvm.dbg.declare(metadata %struct.C2* %c2.i, metadata !30, metadata !DIExpression()), !dbg !101
-  %c.i = getelementptr inbounds %struct.C2, %struct.C2* %c2.i, i32 0, i32 0, !dbg !102
-  %8 = load i8, i8* %c.i, align 1, !dbg !103
-  %inc2.i = add i8 %8, 1, !dbg !103
-  store i8 %inc2.i, i8* %c.i, align 1, !dbg !103
-  ret void, !dbg !104
-}
-
-!llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!2, !3, !4, !5, !6}
-!llvm.ident = !{!7}
-
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
-!1 = !DIFile(filename: "test.cpp", directory: "/")
-!2 = !{i32 7, !"Dwarf Version", i32 4}
-!3 = !{i32 2, !"Debug Info Version", i32 3}
-!4 = !{i32 1, !"wchar_size", i32 4}
-!5 = !{i32 7, !"uwtable", i32 1}
-!6 = !{i32 7, !"frame-pointer", i32 2}
-!7 = !{!"clang version 14.0.0"}
-!8 = distinct !DISubprogram(name: "not_removed", linkageName: "_Z11not_removedv", scope: !1, file: !1, line: 17, type: !9, scopeLine: 17, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !105)
-!9 = !DISubroutineType(types: !10)
-!10 = !{null}
-!11 = !{}
-!12 = !DILocalVariable(name: "a2", scope: !13, file: !1, line: 23, type: !14)
-!13 = distinct !DILexicalBlock(scope: !8, file: !1, line: 20, column: 3)
-!14 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A2", scope: !8, file: !1, line: 18, size: 32, flags: DIFlagTypePassByValue, elements: !15)
-!15 = !{!16}
-!16 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !14, file: !1, line: 18, baseType: !17, size: 32)
-!17 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-!18 = !DILocation(line: 23, column: 8, scope: !13)
-!19 = !DILocation(line: 23, column: 15, scope: !13)
-!20 = !DILocation(line: 23, column: 16, scope: !13)
-!21 = !DILocalVariable(name: "i2", scope: !22, file: !1, line: 25, type: !23)
-!22 = distinct !DILexicalBlock(scope: !13, file: !1, line: 24, column: 5)
-!23 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "I2", scope: !13, file: !1, line: 21, size: 32, flags: DIFlagTypePassByValue, elements: !24)
-!24 = !{!25}
-!25 = !DIDerivedType(tag: DW_TAG_member, name: "j", scope: !23, file: !1, line: 21, baseType: !26, size: 32)
-!26 = !DIDerivedType(tag: DW_TAG_typedef, name: "Int2", scope: !8, file: !1, line: 19, baseType: !17)
-!27 = !DILocation(line: 25, column: 10, scope: !22)
-!28 = !DILocation(line: 25, column: 17, scope: !22)
-!29 = !DILocation(line: 25, column: 18, scope: !22)
-!30 = !DILocalVariable(name: "c2", scope: !22, file: !1, line: 26, type: !31)
-!31 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "C2", scope: !13, file: !1, line: 22, size: 8, flags: DIFlagTypePassByValue, elements: !32)
-!32 = !{!33}
-!33 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !31, file: !1, line: 22, baseType: !34, size: 8)
-!34 = !DIDerivedType(tag: DW_TAG_typedef, name: "Char2", scope: !31, file: !1, line: 22, baseType: !35)
-!35 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
-!36 = !DILocation(line: 26, column: 10, scope: !22)
-!37 = !DILocation(line: 26, column: 17, scope: !22)
-!38 = !DILocation(line: 26, column: 18, scope: !22)
-!39 = !DILocation(line: 29, column: 1, scope: !8)
-!40 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !1, file: !1, line: 31, type: !9, scopeLine: 31, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !107)
-!41 = !DILocalVariable(name: "a3", scope: !42, file: !1, line: 38, type: !44)
-!42 = distinct !DILexicalBlock(scope: !43, file: !1, line: 36, column: 5)
-!43 = distinct !DILexicalBlock(scope: !40, file: !1, line: 34, column: 3)
-!44 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A3", scope: !40, file: !1, line: 32, size: 32, flags: DIFlagTypePassByValue, elements: !45)
-!45 = !{!46}
-!46 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !44, file: !1, line: 32, baseType: !17, size: 32)
-!47 = !DILocation(line: 38, column: 10, scope: !42)
-!48 = !DILocation(line: 38, column: 17, scope: !42)
-!49 = !DILocation(line: 38, column: 18, scope: !42)
-!50 = !DILocalVariable(name: "i3", scope: !51, file: !1, line: 40, type: !52)
-!51 = distinct !DILexicalBlock(scope: !42, file: !1, line: 39, column: 7)
-!52 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "I3", scope: !43, file: !1, line: 35, size: 32, flags: DIFlagTypePassByValue, elements: !53)
-!53 = !{!54}
-!54 = !DIDerivedType(tag: DW_TAG_member, name: "j", scope: !52, file: !1, line: 35, baseType: !55, size: 32)
-!55 = !DIDerivedType(tag: DW_TAG_typedef, name: "Int3", scope: !40, file: !1, line: 33, baseType: !17)
-!56 = !DILocation(line: 40, column: 12, scope: !51)
-!57 = !DILocation(line: 40, column: 19, scope: !51)
-!58 = !DILocation(line: 40, column: 20, scope: !51)
-!59 = !DILocalVariable(name: "c3", scope: !51, file: !1, line: 41, type: !60)
-!60 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "C3", scope: !42, file: !1, line: 37, size: 8, flags: DIFlagTypePassByValue, elements: !61)
-!61 = !{!62}
-!62 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !60, file: !1, line: 37, baseType: !63, size: 8)
-!63 = !DIDerivedType(tag: DW_TAG_typedef, name: "Char3", scope: !60, file: !1, line: 37, baseType: !35)
-!64 = !DILocation(line: 41, column: 12, scope: !51)
-!65 = !DILocation(line: 41, column: 19, scope: !51)
-!66 = !DILocation(line: 41, column: 20, scope: !51)
-!67 = !DILocalVariable(name: "a1", scope: !68, file: !1, line: 8, type: !70)
-!68 = distinct !DILexicalBlock(scope: !69, file: !1, line: 5, column: 3)
-!69 = distinct !DISubprogram(name: "removed", linkageName: "_Z7removedv", scope: !1, file: !1, line: 2, type: !9, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !110)
-!70 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A1", scope: !69, file: !1, line: 3, size: 32, flags: DIFlagTypePassByValue, elements: !71, identifier: "_ZTSZ7removedvE2A1")
-!71 = !{!72}
-!72 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !70, file: !1, line: 3, baseType: !17, size: 32)
-!73 = !DILocation(line: 8, column: 8, scope: !68, inlinedAt: !74)
-!74 = distinct !DILocation(line: 45, column: 3, scope: !40)
-!75 = !DILocation(line: 8, column: 15, scope: !68, inlinedAt: !74)
-!76 = !DILocation(line: 8, column: 16, scope: !68, inlinedAt: !74)
-!77 = !DILocalVariable(name: "i1", scope: !78, file: !1, line: 10, type: !79)
-!78 = distinct !DILexicalBlock(scope: !68, file: !1, line: 9, column: 5)
-!79 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "I1", scope: !68, file: !1, line: 6, size: 32, flags: DIFlagTypePassByValue, elements: !80, identifier: "_ZTSZ7removedvE2I1")
-!80 = !{!81}
-!81 = !DIDerivedType(tag: DW_TAG_member, name: "j", scope: !79, file: !1, line: 6, baseType: !82, size: 32)
-!82 = !DIDerivedType(tag: DW_TAG_typedef, name: "Int1", scope: !69, file: !1, line: 4, baseType: !17)
-!83 = !DILocation(line: 10, column: 10, scope: !78, inlinedAt: !74)
-!84 = !DILocation(line: 10, column: 17, scope: !78, inlinedAt: !74)
-!85 = !DILocation(line: 10, column: 18, scope: !78, inlinedAt: !74)
-!86 = !DILocalVariable(name: "c1", scope: !78, file: !1, line: 11, type: !87)
-!87 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "C1", scope: !68, file: !1, line: 7, size: 8, flags: DIFlagTypePassByValue, elements: !88, identifier: "_ZTSZ7removedvE2C1")
-!88 = !{!89}
-!89 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !87, file: !1, line: 7, baseType: !90, size: 8)
-!90 = !DIDerivedType(tag: DW_TAG_typedef, name: "Char1", scope: !87, file: !1, line: 7, baseType: !35)
-!91 = !DILocation(line: 11, column: 10, scope: !78, inlinedAt: !74)
-!92 = !DILocation(line: 11, column: 17, scope: !78, inlinedAt: !74)
-!93 = !DILocation(line: 11, column: 18, scope: !78, inlinedAt: !74)
-!94 = !DILocation(line: 23, column: 8, scope: !13, inlinedAt: !95)
-!95 = distinct !DILocation(line: 46, column: 3, scope: !40)
-!96 = !DILocation(line: 23, column: 15, scope: !13, inlinedAt: !95)
-!97 = !DILocation(line: 23, column: 16, scope: !13, inlinedAt: !95)
-!98 = !DILocation(line: 25, column: 10, scope: !22, inlinedAt: !95)
-!99 = !DILocation(line: 25, column: 17, scope: !22, inlinedAt: !95)
-!100 = !DILocation(line: 25, column: 18, scope: !22, inlinedAt: !95)
-!101 = !DILocation(line: 26, column: 10, scope: !22, inlinedAt: !95)
-!102 = !DILocation(line: 26, column: 17, scope: !22, inlinedAt: !95)
-!103 = !DILocation(line: 26, column: 18, scope: !22, inlinedAt: !95)
-!104 = !DILocation(line: 47, column: 1, scope: !40)
-!105 = !{!14, !23, !26, !31}
-!107 = !{!44, !52, !55, !60}
-!110 = !{!70, !79, !82, !87}
diff --git a/llvm/test/DebugInfo/Generic/local-type-as-template-parameter.ll b/llvm/test/DebugInfo/Generic/local-type-as-template-parameter.ll
deleted file mode 100644
index 7999e434d139e..0000000000000
--- a/llvm/test/DebugInfo/Generic/local-type-as-template-parameter.ll
+++ /dev/null
@@ -1,160 +0,0 @@
-; RUN: %llc_dwarf -O0 -filetype=obj < %s              \
-; RUN:  | llvm-dwarfdump --show-children --name=foo - \
-; RUN:  | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s
-
-; The test ensures that AsmPrinter doesn't crashed compiling this.
-; It also demostrates misplacement for a local type (see PR55680 for details).
-
-; The test compiled from:
-
-; template<typename T>
-; struct A {
-;   A(T &in) : a(in) {}
-;   T a;
-; };
-;
-; __attribute__((always_inline))
-; void foo() {
-;   struct B { int i; };
-;   B objB;
-;   A<B> objA(objB);
-; }
-;
-; int main() {
-;   foo();
-; }
-
-; Concrete out-of-line tree of foo().
-; CHECK: DW_TAG_subprogram
-; CHECK:   DW_AT_abstract_origin {{.*}} "_Z3foov"
-
-; FIXME: 'struct B' should be in the abstract tree below, not here.
-; CHECK:   DW_TAG_structure_type
-; CHECK:     DW_AT_name	("B")
-; CHECK:     DW_TAG_member
-; CHECK:     NULL
-;
-; CHECK:   DW_TAG_variable
-; CHECK:     DW_AT_abstract_origin {{.*}} "objB"
-; CHECK:   DW_TAG_variable
-; CHECK:     DW_AT_abstract_origin {{.*}} "objA"
-
-; CHECK:   NULL
-
-; Abstract tree of foo().
-; CHECK: DW_TAG_subprogram
-; CHECK:   DW_AT_name	("foo")
-; CHECK:   DW_AT_inline	(DW_INL_inlined)
-
-; CHECK:   DW_TAG_variable
-; CHECK:     DW_AT_name	("objB")
-; CHECK:   DW_TAG_variable
-; CHECK:     DW_AT_name	("objA")
-
-; CHECK:   NULL
-
-; CHECK: DW_TAG_inlined_subroutine
-; CHECK:   DW_AT_abstract_origin {{.*}} "_Z3foov"
-; CHECK:   DW_TAG_variable
-; CHECK:     DW_AT_abstract_origin {{.*}} "objB"
-; CHECK:   DW_TAG_variable
-; CHECK:     DW_AT_abstract_origin {{.*}} "objA"
-; CHECK:   NULL
-
-%struct.B = type { i32 }
-%struct.A = type { %struct.B }
-
-define dso_local void @_Z3foov() !dbg !7 {
-entry:
-  %objB = alloca %struct.B, align 4
-  %objA = alloca %struct.A, align 4
-  call void @llvm.dbg.declare(metadata ptr %objB, metadata !30, metadata !DIExpression()), !dbg !31
-  call void @llvm.dbg.declare(metadata ptr %objA, metadata !32, metadata !DIExpression()), !dbg !33
-  call void @_ZN1AIZ3foovE1BEC2ERS0_(ptr noundef nonnull align 4 dereferenceable(4) %objA, ptr noundef nonnull align 4 dereferenceable(4) %objB), !dbg !33
-  ret void, !dbg !34
-}
-
-declare void @llvm.dbg.declare(metadata, metadata, metadata)
-
-define internal void @_ZN1AIZ3foovE1BEC2ERS0_(ptr noundef nonnull align 4 dereferenceable(4) %this, ptr noundef nonnull align 4 dereferenceable(4) %in) unnamed_addr align 2 !dbg !35 {
-entry:
-  %this.addr = alloca ptr, align 8
-  %in.addr = alloca ptr, align 8
-  store ptr %this, ptr %this.addr, align 8
-  call void @llvm.dbg.declare(metadata ptr %this.addr, metadata !36, metadata !DIExpression()), !dbg !38
-  store ptr %in, ptr %in.addr, align 8
-  call void @llvm.dbg.declare(metadata ptr %in.addr, metadata !39, metadata !DIExpression()), !dbg !40
-  %this1 = load ptr, ptr %this.addr, align 8
-  %a = getelementptr inbounds %struct.A, ptr %this1, i32 0, i32 0, !dbg !41
-  %0 = load ptr, ptr %in.addr, align 8, !dbg !42
-  call void @llvm.memcpy.p0.p0.i64(ptr align 4 %a, ptr align 4 %0, i64 4, i1 false), !dbg !41
-  ret void, !dbg !43
-}
-
-define dso_local noundef i32 @main() !dbg !44 {
-entry:
-  %objB.i = alloca %struct.B, align 4
-  %objA.i = alloca %struct.A, align 4
-  call void @llvm.dbg.declare(metadata ptr %objB.i, metadata !30, metadata !DIExpression()), !dbg !47
-  call void @llvm.dbg.declare(metadata ptr %objA.i, metadata !32, metadata !DIExpression()), !dbg !49
-  call void @_ZN1AIZ3foovE1BEC2ERS0_(ptr noundef nonnull align 4 dereferenceable(4) %objA.i, ptr noundef nonnull align 4 dereferenceable(4) %objB.i), !dbg !49
-  ret i32 0, !dbg !50
-}
-
-declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg)
-
-!llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!22, !23, !24, !25, !26, !27, !28}
-!llvm.ident = !{!29}
-
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 15.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, retainedTypes: !2, splitDebugInlining: false, nameTableKind: None)
-!1 = !DIFile(filename: "test.cpp", directory: "/", checksumkind: CSK_MD5, checksum: "aec7fd397e86f8655ef7f4bb4233b849")
-!2 = !{!3}
-!3 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A<B>", file: !1, line: 2, size: 32, flags: DIFlagTypePassByValue | DIFlagNonTrivial, elements: !4, templateParams: !20)
-!4 = !{!5, !15}
-!5 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !3, file: !1, line: 4, baseType: !6, size: 32)
-!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "B", scope: !7, file: !1, line: 9, size: 32, flags: DIFlagTypePassByValue, elements: !12)
-!7 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !1, file: !1, line: 8, type: !8, scopeLine: 8, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !11)
-!8 = !DISubroutineType(types: !9)
-!9 = !{null}
-!10 = !{}
-!11 = !{!6}
-!12 = !{!13}
-!13 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !6, file: !1, line: 9, baseType: !14, size: 32)
-!14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-!15 = !DISubprogram(name: "A", scope: !3, file: !1, line: 3, type: !16, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagLocalToUnit)
-!16 = !DISubroutineType(types: !17)
-!17 = !{null, !18, !19}
-!18 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !3, size: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
-!19 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !6, size: 64)
-!20 = !{!21}
-!21 = !DITemplateTypeParameter(name: "T", type: !6)
-!22 = !{i32 7, !"Dwarf Version", i32 5}
-!23 = !{i32 2, !"Debug Info Version", i32 3}
-!24 = !{i32 1, !"wchar_size", i32 4}
-!25 = !{i32 7, !"PIC Level", i32 2}
-!26 = !{i32 7, !"PIE Level", i32 2}
-!27 = !{i32 7, !"uwtable", i32 2}
-!28 = !{i32 7, !"frame-pointer", i32 2}
-!29 = !{!"clang version 15.0.0"}
-!30 = !DILocalVariable(name: "objB", scope: !7, file: !1, line: 10, type: !6)
-!31 = !DILocation(line: 10, column: 5, scope: !7)
-!32 = !DILocalVariable(name: "objA", scope: !7, file: !1, line: 11, type: !3)
-!33 = !DILocation(line: 11, column: 8, scope: !7)
-!34 = !DILocation(line: 12, column: 1, scope: !7)
-!35 = distinct !DISubprogram(name: "A", linkageName: "_ZN1AIZ3foovE1BEC2ERS0_", scope: !3, file: !1, line: 3, type: !16, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !0, declaration: !15, retainedNodes: !10)
-!36 = !DILocalVariable(name: "this", arg: 1, scope: !35, type: !37, flags: DIFlagArtificial | DIFlagObjectPointer)
-!37 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !3, size: 64)
-!38 = !DILocation(line: 0, scope: !35)
-!39 = !DILocalVariable(name: "in", arg: 2, scope: !35, file: !1, line: 3, type: !19)
-!40 = !DILocation(line: 3, column: 8, scope: !35)
-!41 = !DILocation(line: 3, column: 14, scope: !35)
-!42 = !DILocation(line: 3, column: 16, scope: !35)
-!43 = !DILocation(line: 3, column: 21, scope: !35)
-!44 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 14, type: !45, scopeLine: 14, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !10)
-!45 = !DISubroutineType(types: !46)
-!46 = !{!14}
-!47 = !DILocation(line: 10, column: 5, scope: !7, inlinedAt: !48)
-!48 = distinct !DILocation(line: 15, column: 3, scope: !44)
-!49 = !DILocation(line: 11, column: 8, scope: !7, inlinedAt: !48)
-!50 = !DILocation(line: 16, column: 1, scope: !44)
diff --git a/llvm/test/DebugInfo/Generic/verifier-invalid-disubprogram.ll b/llvm/test/DebugInfo/Generic/verifier-invalid-disubprogram.ll
index 54ce1c56c6b30..6d4d0e93d38f9 100644
--- a/llvm/test/DebugInfo/Generic/verifier-invalid-disubprogram.ll
+++ b/llvm/test/DebugInfo/Generic/verifier-invalid-disubprogram.ll
@@ -38,7 +38,7 @@ define void @invalid_subprogram_declaration() !dbg !9 { ret void }
 define void @invalid_retained_nodes_list() !dbg !10 { ret void }
 !10 = distinct !DISubprogram(retainedNodes: !0)
 
-; CHECK: invalid retained nodes, expected DILocalVariable, DILabel, DIImportedEntity or DIType
+; CHECK: invalid retained nodes, expected DILocalVariable, DILabel or DIImportedEntity
 define void @invalid_retained_nodes_expected() !dbg !11 { ret void }
 !11 = distinct !DISubprogram(retainedNodes: !{!0})
 
diff --git a/llvm/test/DebugInfo/X86/set.ll b/llvm/test/DebugInfo/X86/set.ll
index 7dbec4520ed6c..292c7c6e4a577 100644
--- a/llvm/test/DebugInfo/X86/set.ll
+++ b/llvm/test/DebugInfo/X86/set.ll
@@ -68,11 +68,11 @@ attributes #1 = { nofree nosync nounwind readnone speculatable willreturn }
 !llvm.module.flags = !{!18, !19, !20}
 
 !0 = !{!"versions- cm3: d5.10.0 llvm: 9.0"}
-!1 = distinct !DICompileUnit(language: DW_LANG_Modula3, file: !2, producer: "cm3", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = distinct !DICompileUnit(language: DW_LANG_Modula3, file: !2, producer: "cm3", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !3)
 !2 = !DIFile(filename: "Main.m3", directory: "/home/cm3/settest/src")
 !3 = !{!4}
 !4 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum", scope: !5, file: !2, line: 11, size: 8, align: 8, elements: !9)
-!5 = distinct !DISubprogram(name: "Test", linkageName: "Main__Test", scope: !2, file: !2, line: 11, type: !6, scopeLine: 11, spFlags: DISPFlagDefinition, unit: !1, retainedNodes: !3)
+!5 = distinct !DISubprogram(name: "Test", linkageName: "Main__Test", scope: !2, file: !2, line: 11, type: !6, scopeLine: 11, spFlags: DISPFlagDefinition, unit: !1, retainedNodes: !8)
 !6 = !DISubroutineType(types: !7)
 !7 = !{null}
 !8 = !{}

From 706b5472d897ca75ebd210e4109637793288bcf2 Mon Sep 17 00:00:00 2001
From: Amy Kwan <amy.kwan1@ibm.com>
Date: Sat, 17 Jun 2023 00:31:35 -0500
Subject: [PATCH 064/130] [AIX][TLS] Relax front end diagnostics to accept the
 local-exec TLS model

This patch relaxes the front end AIX diagnostics added in D102070 to accept the
local-exec TLS model, as we plan to support this model in a series of future patches.

The diagnostics are relaxed when local-exec is used as a compiler option to
`-ftls-model=*` and in the `__attribute__((tls_model("local-exec")))` attribute.

Differential Revision: https://reviews.llvm.org/D149596
---
 clang/lib/Frontend/CompilerInvocation.cpp    | 2 +-
 clang/lib/Sema/SemaDeclAttr.cpp              | 2 +-
 clang/test/CodeGen/PowerPC/aix-tls-model.cpp | 9 ++++++---
 clang/test/Sema/aix-attr-tls_model.c         | 2 +-
 4 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index f6c04febfd192..dca8853f1c204 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -1894,7 +1894,7 @@ bool CompilerInvocation::ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args,
   if (Arg *A = Args.getLastArg(OPT_ftlsmodel_EQ)) {
     if (T.isOSAIX()) {
       StringRef Name = A->getValue();
-      if (Name != "global-dynamic")
+      if (Name != "global-dynamic" && Name != "local-exec")
         Diags.Report(diag::err_aix_unsupported_tls_model) << Name;
     }
   }
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 40438214d2b46..d9146a89d322e 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -2039,7 +2039,7 @@ static void handleTLSModelAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
   }
 
   if (S.Context.getTargetInfo().getTriple().isOSAIX() &&
-      Model != "global-dynamic") {
+      Model != "global-dynamic" && Model != "local-exec") {
     S.Diag(LiteralLoc, diag::err_aix_attr_unsupported_tls_model) << Model;
     return;
   }
diff --git a/clang/test/CodeGen/PowerPC/aix-tls-model.cpp b/clang/test/CodeGen/PowerPC/aix-tls-model.cpp
index a531f558ac796..2b53df54a5e3c 100644
--- a/clang/test/CodeGen/PowerPC/aix-tls-model.cpp
+++ b/clang/test/CodeGen/PowerPC/aix-tls-model.cpp
@@ -2,12 +2,12 @@
 // RUN: %clang_cc1 %s -triple powerpc-unknown-aix -target-cpu pwr8 -ftls-model=global-dynamic -emit-llvm -o - | FileCheck %s -check-prefix=CHECK-GD
 // RUN: not %clang_cc1 %s -triple powerpc-unknown-aix -target-cpu pwr8 -ftls-model=local-dynamic -emit-llvm 2>&1 | FileCheck %s -check-prefix=CHECK-LD-ERROR
 // RUN: not %clang_cc1 %s -triple powerpc-unknown-aix -target-cpu pwr8 -ftls-model=initial-exec -emit-llvm  2>&1 | FileCheck %s -check-prefix=CHECK-IE-ERROR
-// RUN: not %clang_cc1 %s -triple powerpc-unknown-aix -target-cpu pwr8 -ftls-model=local-exec -emit-llvm 2>&1 | FileCheck %s -check-prefix=CHECK-LE-ERROR
+// RUN: %clang_cc1 %s -triple powerpc-unknown-aix -target-cpu pwr8 -ftls-model=local-exec -emit-llvm -o - | FileCheck %s -check-prefix=CHECK-LE
 // RUN: %clang_cc1 %s -triple powerpc64-unknown-aix -target-cpu pwr8 -emit-llvm -o - | FileCheck %s -check-prefix=CHECK-GD
 // RUN: %clang_cc1 %s -triple powerpc64-unknown-aix -target-cpu pwr8 -ftls-model=global-dynamic -emit-llvm -o - | FileCheck %s -check-prefix=CHECK-GD
 // RUN: not %clang_cc1 %s -triple powerpc64-unknown-aix -target-cpu pwr8 -ftls-model=local-dynamic -emit-llvm 2>&1 | FileCheck %s -check-prefix=CHECK-LD-ERROR
 // RUN: not %clang_cc1 %s -triple powerpc64-unknown-aix -target-cpu pwr8 -ftls-model=initial-exec -emit-llvm  2>&1 | FileCheck %s -check-prefix=CHECK-IE-ERROR
-// RUN: not %clang_cc1 %s -triple powerpc64-unknown-aix -target-cpu pwr8 -ftls-model=local-exec -emit-llvm 2>&1 | FileCheck %s -check-prefix=CHECK-LE-ERROR
+// RUN: %clang_cc1 %s -triple powerpc64-unknown-aix -target-cpu pwr8 -ftls-model=local-exec -emit-llvm -o - | FileCheck %s -check-prefix=CHECK-LE
 
 int z1 = 0;
 int z2;
@@ -23,4 +23,7 @@ int f() {
 // CHECK-GD: @_ZZ1fvE1y = internal thread_local global i32 0
 // CHECK-LD-ERROR:  error: TLS model 'local-dynamic' is not yet supported on AIX
 // CHECK-IE-ERROR:  error: TLS model 'initial-exec' is not yet supported on AIX
-// CHECK-LE-ERROR:  error: TLS model 'local-exec' is not yet supported on AIX
+// CHECK-LE: @z1 ={{.*}} global i32 0
+// CHECK-LE: @z2 ={{.*}} global i32 0
+// CHECK-LE: @x ={{.*}} thread_local(localexec) global i32 0
+// CHECK-LE: @_ZZ1fvE1y = internal thread_local(localexec) global i32 0
diff --git a/clang/test/Sema/aix-attr-tls_model.c b/clang/test/Sema/aix-attr-tls_model.c
index 8cf3086fc1aa0..245a4434d35c6 100644
--- a/clang/test/Sema/aix-attr-tls_model.c
+++ b/clang/test/Sema/aix-attr-tls_model.c
@@ -8,4 +8,4 @@
 static __thread int y __attribute((tls_model("global-dynamic"))); // no-warning
 static __thread int y __attribute((tls_model("local-dynamic"))); // expected-error {{TLS model 'local-dynamic' is not yet supported on AIX}}
 static __thread int y __attribute((tls_model("initial-exec"))); // expected-error {{TLS model 'initial-exec' is not yet supported on AIX}}
-static __thread int y __attribute((tls_model("local-exec"))); // expected-error {{TLS model 'local-exec' is not yet supported on AIX}}
+static __thread int y __attribute((tls_model("local-exec"))); // no-warning

From d5659808b2fabadd2fff4f19d7aed2e2ff3435f5 Mon Sep 17 00:00:00 2001
From: Amy Kwan <amy.kwan1@ibm.com>
Date: Sat, 17 Jun 2023 00:33:38 -0500
Subject: [PATCH 065/130] [AIX][TLS] Generate 64-bit local-exec access code
 sequence

This patch adds support for the TLS local-exec access model on AIX to allow
for the ability to generate the 64-bit (specifically, non-optimized) code sequence.

For this patch in particular, the sequence that is generated involves a load of the
variable offset, followed by an add of the loaded variable offset to r13 (which is
thread pointer, respectively). This code sequence looks like the following:
```
ld reg1,var[TC](2)
add reg2, reg1, r13     // r13 contains the thread pointer
```
The TOC (.tc pseudo-op) entries generated in the assembly files are also
changed where we add the @le relocation for the variable offset.

Differential Revision: https://reviews.llvm.org/D149722
---
 llvm/include/llvm/MC/MCExpr.h                 |   1 +
 llvm/lib/MC/MCExpr.cpp                        |   2 +
 .../PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp  |   9 +-
 .../MCTargetDesc/PPCXCOFFObjectWriter.cpp     |   2 +
 llvm/lib/Target/PowerPC/PPC.h                 |   5 +-
 llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp     |  12 +
 llvm/lib/Target/PowerPC/PPCISelLowering.cpp   |  26 +-
 llvm/lib/Target/PowerPC/PPCISelLowering.h     |  10 +-
 llvm/lib/Target/PowerPC/PPCInstr64Bit.td      |  11 +
 .../CodeGen/PowerPC/aix-tls-le-ldst-double.ll | 328 +++++++++++++++++
 .../CodeGen/PowerPC/aix-tls-le-ldst-float.ll  | 328 +++++++++++++++++
 .../CodeGen/PowerPC/aix-tls-le-ldst-int.ll    | 336 ++++++++++++++++++
 .../PowerPC/aix-tls-le-ldst-longlong.ll       | 328 +++++++++++++++++
 .../PowerPC/aix-tls-le-xcoff-reloc-large.ll   | 218 ++++++++++++
 .../CodeGen/PowerPC/aix-tls-le-xcoff-reloc.ll | 194 ++++++++++
 15 files changed, 1796 insertions(+), 14 deletions(-)
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-tls-le-ldst-double.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-tls-le-ldst-float.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-tls-le-ldst-int.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-tls-le-ldst-longlong.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-tls-le-xcoff-reloc-large.ll
 create mode 100644 llvm/test/CodeGen/PowerPC/aix-tls-le-xcoff-reloc.ll

diff --git a/llvm/include/llvm/MC/MCExpr.h b/llvm/include/llvm/MC/MCExpr.h
index bf1f32bb91ba7..49bd6883d4297 100644
--- a/llvm/include/llvm/MC/MCExpr.h
+++ b/llvm/include/llvm/MC/MCExpr.h
@@ -299,6 +299,7 @@ class MCSymbolRefExpr : public MCExpr {
     VK_PPC_TLSGD,           // symbol@tlsgd
     VK_PPC_AIX_TLSGD,       // symbol@gd
     VK_PPC_AIX_TLSGDM,      // symbol@m
+    VK_PPC_AIX_TLSLE,       // symbol@le
     VK_PPC_GOT_TLSLD,       // symbol@got@tlsld
     VK_PPC_GOT_TLSLD_LO,    // symbol@got@tlsld@l
     VK_PPC_GOT_TLSLD_HI,    // symbol@got@tlsld@h
diff --git a/llvm/lib/MC/MCExpr.cpp b/llvm/lib/MC/MCExpr.cpp
index d5b5aa8fba4e0..c2ecc00b5796c 100644
--- a/llvm/lib/MC/MCExpr.cpp
+++ b/llvm/lib/MC/MCExpr.cpp
@@ -327,6 +327,8 @@ StringRef MCSymbolRefExpr::getVariantKindName(VariantKind Kind) {
     return "gd";
   case VK_PPC_AIX_TLSGDM:
     return "m";
+  case VK_PPC_AIX_TLSLE:
+    return "le";
   case VK_PPC_GOT_TLSLD: return "got@tlsld";
   case VK_PPC_GOT_TLSLD_LO: return "got@tlsld@l";
   case VK_PPC_GOT_TLSLD_HI: return "got@tlsld@h";
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
index 37bb90ab2c248..271f7ab757e16 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
@@ -147,12 +147,11 @@ class PPCTargetAsmStreamer : public PPCTargetStreamer {
       MCSymbolXCOFF *TCSym =
           cast<MCSectionXCOFF>(Streamer.getCurrentSectionOnly())
               ->getQualNameSymbol();
-      // If the variant kind is VK_PPC_AIX_TLSGDM the entry represents the
-      // region handle for the symbol, we add the relocation specifier @m.
-      // If the variant kind is VK_PPC_AIX_TLSGD the entry represents the
-      // variable offset for the symbol, we add the relocation specifier @gd.
+      // On AIX, we have a region handle (symbol@m) and the variable offset
+      // (symbol@{gd|le}) for TLS variables, depending on the TLS model.
       if (Kind == MCSymbolRefExpr::VariantKind::VK_PPC_AIX_TLSGD ||
-          Kind == MCSymbolRefExpr::VariantKind::VK_PPC_AIX_TLSGDM)
+          Kind == MCSymbolRefExpr::VariantKind::VK_PPC_AIX_TLSGDM ||
+          Kind == MCSymbolRefExpr::VariantKind::VK_PPC_AIX_TLSLE)
         OS << "\t.tc " << TCSym->getName() << "," << XSym->getName() << "@"
            << MCSymbolRefExpr::getVariantKindName(Kind) << '\n';
       else
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCXCOFFObjectWriter.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCXCOFFObjectWriter.cpp
index b6e749b781804..df671f53cbd8c 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCXCOFFObjectWriter.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCXCOFFObjectWriter.cpp
@@ -108,6 +108,8 @@ std::pair<uint8_t, uint8_t> PPCXCOFFObjectWriter::getRelocTypeAndSignSize(
       return {XCOFF::RelocationType::R_TLS, SignAndSizeForFKData};
     case MCSymbolRefExpr::VK_PPC_AIX_TLSGDM:
       return {XCOFF::RelocationType::R_TLSM, SignAndSizeForFKData};
+    case MCSymbolRefExpr::VK_PPC_AIX_TLSLE:
+      return {XCOFF::RelocationType::R_TLS_LE, SignAndSizeForFKData};
     case MCSymbolRefExpr::VK_None:
       return {XCOFF::RelocationType::R_POS, SignAndSizeForFKData};
     }
diff --git a/llvm/lib/Target/PowerPC/PPC.h b/llvm/lib/Target/PowerPC/PPC.h
index 8f84ae7efc246..0d3d71742bfb7 100644
--- a/llvm/lib/Target/PowerPC/PPC.h
+++ b/llvm/lib/Target/PowerPC/PPC.h
@@ -127,8 +127,9 @@ class ModulePass;
     /// General Dynamic model for AIX.
     MO_TLSGD_FLAG = 32,
 
-    /// MO_TPREL_FLAG - If this bit is set the symbol reference is relative to
-    /// TLS Initial Exec model.
+    /// MO_TPREL_FLAG - If this bit is set, the symbol reference is relative to
+    /// the thread pointer and the symbol can be used for the TLS Initial Exec
+    /// and Local Exec models.
     MO_TPREL_FLAG = 64,
 
     /// MO_TLSLD_FLAG - If this bit is set the symbol reference is relative to
diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
index caaea40ae5169..2a192e6555912 100644
--- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp
@@ -813,6 +813,18 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) {
     return Expr;
   };
   auto GetVKForMO = [&](const MachineOperand &MO) {
+    // For TLS local-exec accesses on AIX, we have one TOC entry for the symbol
+    // (with the variable offset), which is differentiated by MO_TPREL_FLAG.
+    if (MO.getTargetFlags() & PPCII::MO_TPREL_FLAG) {
+      // TODO: Update the query and the comment above to add a check for initial
+      // exec when this TLS model is supported on AIX in the future, as both
+      // local-exec and initial-exec can use MO_TPREL_FLAG.
+      assert(MO.isGlobal() && "Only expecting a global MachineOperand here!\n");
+      TLSModel::Model Model = TM.getTLSModel(MO.getGlobal());
+      if (Model == TLSModel::LocalExec)
+        return MCSymbolRefExpr::VariantKind::VK_PPC_AIX_TLSLE;
+      llvm_unreachable("Only expecting local-exec accesses!");
+    }
     // For GD TLS access on AIX, we have two TOC entries for the symbol (one for
     // the variable offset and the other for the region handle). They are
     // differentiated by MO_TLSGD_FLAG and MO_TLSGDM_FLAG.
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 012052a1be1d3..f4eb89dda19ce 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -3326,9 +3326,31 @@ SDValue PPCTargetLowering::LowerGlobalTLSAddressAIX(SDValue Op,
   SDLoc dl(GA);
   const GlobalValue *GV = GA->getGlobal();
   EVT PtrVT = getPointerTy(DAG.getDataLayout());
+  bool Is64Bit = Subtarget.isPPC64();
+  TLSModel::Model Model = getTargetMachine().getTLSModel(GV);
 
-  // The general-dynamic model is the only access model supported for now, so
-  // all the GlobalTLSAddress nodes are lowered with this model.
+  if (Model == TLSModel::LocalExec) {
+    if (Is64Bit) {
+      // For local-exec on AIX (64-bit), the sequence that is generated involves
+      // a load of the variable offset (from the TOC), followed by an add of the
+      // loaded variable offset to R13 (the thread pointer).
+      // This code sequence looks like:
+      //    ld reg1,var[TC](2)
+      //    add reg2, reg1, r13     // r13 contains the thread pointer
+      SDValue TLSReg = DAG.getRegister(PPC::X13, MVT::i64);
+      SDValue VariableOffsetTGA =
+          DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, PPCII::MO_TPREL_FLAG);
+      SDValue VariableOffset = getTOCEntry(DAG, dl, VariableOffsetTGA);
+      return DAG.getNode(PPCISD::ADD_TLS, dl, PtrVT, TLSReg, VariableOffset);
+    } else {
+      report_fatal_error("On AIX, the local-exec TLS model is only supported "
+                         "on PPC64 for now.");
+    }
+  }
+
+  // The Local-Exec and General-Dynamic TLS models are currently the only
+  // supported access models. If Local-exec is not possible or specified, all
+  // GlobalTLSAddress nodes are lowered using the general-dynamic model.
   // We need to generate two TOC entries, one for the variable offset, one for
   // the region handle. The global address for the TOC entry of the region
   // handle is created with the MO_TLSGDM_FLAG flag and the global address
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h
index 02f7147dfd6bb..2e2514a5e34a4 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.h
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h
@@ -332,11 +332,11 @@ namespace llvm {
     /// finds the offset of "sym" relative to the thread pointer.
     LD_GOT_TPREL_L,
 
-    /// G8RC = ADD_TLS G8RReg, Symbol - Used by the initial-exec TLS
-    /// model, produces an ADD instruction that adds the contents of
-    /// G8RReg to the thread pointer.  Symbol contains a relocation
-    /// sym\@tls which is to be replaced by the thread pointer and
-    /// identifies to the linker that the instruction is part of a
+    /// G8RC = ADD_TLS G8RReg, Symbol - Can be used by the initial-exec
+    /// and local-exec TLS models, produces an ADD instruction that adds
+    /// the contents of G8RReg to the thread pointer.  Symbol contains a
+    /// relocation sym\@tls which is to be replaced by the thread pointer
+    /// and identifies to the linker that the instruction is part of a
     /// TLS sequence.
     ADD_TLS,
 
diff --git a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
index 4487082a3bb8c..f4f058aff237f 100644
--- a/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
+++ b/llvm/lib/Target/PowerPC/PPCInstr64Bit.td
@@ -1910,9 +1910,20 @@ def : Pat<(add i64:$in, (PPChi tblockaddress:$g, 0)),
           (ADDIS8 $in, tblockaddress:$g)>;
 
 // AIX 64-bit small code model TLS access.
+// This is used for global dynamic accesses when loading the region handle and
+// variable offset, and also for local-exec accesses to load the offset of a
+// TLS variable from the TOC, prior to adding it to r13.
 def : Pat<(i64 (PPCtoc_entry tglobaltlsaddr:$disp, i64:$reg)),
           (i64 (LDtoc tglobaltlsaddr:$disp, i64:$reg))>;
 
+// The following pattern matches 64-bit local-exec TLS accesses on AIX.
+// PPCaddTls is used in local-exec accesses in order to:
+//   - Get the address of a variable (adding the variable offset to the thread
+//     pointer in r13).
+//   - Create an opportunity to optimize the user of the loaded address.
+def : Pat<(PPCaddTls i64:$in, i64:$addr),
+          (ADD8TLS $in, $addr)>;
+
 // 64-bits atomic loads and stores
 def : Pat<(atomic_load_64 DSForm:$src), (LD  memrix:$src)>;
 def : Pat<(atomic_load_64 XForm:$src),  (LDX memrr:$src)>;
diff --git a/llvm/test/CodeGen/PowerPC/aix-tls-le-ldst-double.ll b/llvm/test/CodeGen/PowerPC/aix-tls-le-ldst-double.ll
new file mode 100644
index 0000000000000..2e0697d347226
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-tls-le-ldst-double.ll
@@ -0,0 +1,328 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc  -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \
+; RUN:      -mtriple powerpc64-ibm-aix-xcoff < %s | FileCheck %s \
+; RUN:      --check-prefix=SMALL64
+; RUN: llc  -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \
+; RUN:      -mtriple powerpc64-ibm-aix-xcoff --code-model=large < %s \
+; RUN:      | FileCheck %s --check-prefix=LARGE64
+
+@ThreadLocalVarInit = thread_local(localexec) global double 0x4021947AE147AE14, align 8
+@VarInit = global double 8.787000e+01, align 8
+@IThreadLocalVarUninit = internal thread_local(localexec) global double 0.000000e+00, align 8
+@IThreadLocalVarInit = internal thread_local(localexec) global double 5.870000e+00, align 8
+@ThreadLocalVarUninit = thread_local(localexec) global double 0.000000e+00, align 8
+declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull) #1
+
+define void @storeITLUninit(double noundef %x) {
+; SMALL64-LABEL: storeITLUninit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tprel) @IThreadLocalVarUninit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    stfd f1, 0(r3)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: storeITLUninit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C0@u(r2)
+; LARGE64-NEXT:    ld r3, L..C0@l(r3)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    stfd f1, 0(r3)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @IThreadLocalVarUninit)
+  store double %x, ptr %0, align 8
+  ret void
+}
+
+define void @storeITLInit(double noundef %x) {
+; SMALL64-LABEL: storeITLInit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C1(r2) # target-flags(ppc-tprel) @IThreadLocalVarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    stfd f1, 0(r3)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: storeITLInit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C1@u(r2)
+; LARGE64-NEXT:    ld r3, L..C1@l(r3)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    stfd f1, 0(r3)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @IThreadLocalVarInit)
+  store double %x, ptr %0, align 8
+  ret void
+}
+
+define void @storeTLUninit(double noundef %x) {
+; SMALL64-LABEL: storeTLUninit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C2(r2) # target-flags(ppc-tprel) @ThreadLocalVarUninit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    stfd f1, 0(r3)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: storeTLUninit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C2@u(r2)
+; LARGE64-NEXT:    ld r3, L..C2@l(r3)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    stfd f1, 0(r3)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ThreadLocalVarUninit)
+  store double %x, ptr %0, align 8
+  ret void
+}
+
+define void @storeTLInit(double noundef %x) {
+; SMALL64-LABEL: storeTLInit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C3(r2) # target-flags(ppc-tprel) @ThreadLocalVarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    stfd f1, 0(r3)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: storeTLInit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C3@u(r2)
+; LARGE64-NEXT:    ld r3, L..C3@l(r3)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    stfd f1, 0(r3)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ThreadLocalVarInit)
+  store double %x, ptr %0, align 8
+  ret void
+}
+
+define double @loadITLUninit() {
+; SMALL64-LABEL: loadITLUninit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tprel) @IThreadLocalVarUninit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    lfd f1, 0(r3)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadITLUninit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C0@u(r2)
+; LARGE64-NEXT:    ld r3, L..C0@l(r3)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    lfd f1, 0(r3)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @IThreadLocalVarUninit)
+  %1 = load double, ptr %0, align 8
+  ret double %1
+}
+
+define double @loadITLUninit2() {
+; SMALL64-LABEL: loadITLUninit2:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tprel) @IThreadLocalVarUninit
+; SMALL64-NEXT:    ld r4, L..C4(r2) # @VarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    lfd f1, 0(r4)
+; SMALL64-NEXT:    lfd f0, 0(r3)
+; SMALL64-NEXT:    xsadddp f1, f0, f1
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadITLUninit2:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C0@u(r2)
+; LARGE64-NEXT:    addis r4, L..C4@u(r2)
+; LARGE64-NEXT:    ld r3, L..C0@l(r3)
+; LARGE64-NEXT:    ld r4, L..C4@l(r4)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    lfd f1, 0(r4)
+; LARGE64-NEXT:    lfd f0, 0(r3)
+; LARGE64-NEXT:    xsadddp f1, f0, f1
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @IThreadLocalVarUninit)
+  %1 = load double, ptr %0, align 8
+  %2 = load double, ptr @VarInit, align 8
+  %add = fadd double %1, %2
+  ret double %add
+}
+
+define double @loadITLInit() {
+; SMALL64-LABEL: loadITLInit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C1(r2) # target-flags(ppc-tprel) @IThreadLocalVarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    lfd f1, 0(r3)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadITLInit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C1@u(r2)
+; LARGE64-NEXT:    ld r3, L..C1@l(r3)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    lfd f1, 0(r3)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @IThreadLocalVarInit)
+  %1 = load double, ptr %0, align 8
+  ret double %1
+}
+
+define double @loadITLInit2() {
+; SMALL64-LABEL: loadITLInit2:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C1(r2) # target-flags(ppc-tprel) @IThreadLocalVarInit
+; SMALL64-NEXT:    ld r4, L..C4(r2) # @VarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    lfd f1, 0(r4)
+; SMALL64-NEXT:    lfd f0, 0(r3)
+; SMALL64-NEXT:    xsadddp f1, f0, f1
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadITLInit2:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C1@u(r2)
+; LARGE64-NEXT:    addis r4, L..C4@u(r2)
+; LARGE64-NEXT:    ld r3, L..C1@l(r3)
+; LARGE64-NEXT:    ld r4, L..C4@l(r4)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    lfd f1, 0(r4)
+; LARGE64-NEXT:    lfd f0, 0(r3)
+; LARGE64-NEXT:    xsadddp f1, f0, f1
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @IThreadLocalVarInit)
+  %1 = load double, ptr %0, align 8
+  %2 = load double, ptr @VarInit, align 8
+  %add = fadd double %1, %2
+  ret double %add
+}
+
+define double @loadTLUninit() {
+; SMALL64-LABEL: loadTLUninit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C2(r2) # target-flags(ppc-tprel) @ThreadLocalVarUninit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    lfd f1, 0(r3)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadTLUninit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C2@u(r2)
+; LARGE64-NEXT:    ld r3, L..C2@l(r3)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    lfd f1, 0(r3)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ThreadLocalVarUninit)
+  %1 = load double, ptr %0, align 8
+  ret double %1
+}
+
+define double @loadTLUninit2() {
+; SMALL64-LABEL: loadTLUninit2:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C2(r2) # target-flags(ppc-tprel) @ThreadLocalVarUninit
+; SMALL64-NEXT:    ld r4, L..C4(r2) # @VarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    lfd f1, 0(r4)
+; SMALL64-NEXT:    lfd f0, 0(r3)
+; SMALL64-NEXT:    xsadddp f1, f0, f1
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadTLUninit2:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C2@u(r2)
+; LARGE64-NEXT:    addis r4, L..C4@u(r2)
+; LARGE64-NEXT:    ld r3, L..C2@l(r3)
+; LARGE64-NEXT:    ld r4, L..C4@l(r4)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    lfd f1, 0(r4)
+; LARGE64-NEXT:    lfd f0, 0(r3)
+; LARGE64-NEXT:    xsadddp f1, f0, f1
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ThreadLocalVarUninit)
+  %1 = load double, ptr %0, align 8
+  %2 = load double, ptr @VarInit, align 8
+  %add = fadd double %1, %2
+  ret double %add
+}
+
+define double @loadTLInit() {
+; SMALL64-LABEL: loadTLInit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C3(r2) # target-flags(ppc-tprel) @ThreadLocalVarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    lfd f1, 0(r3)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadTLInit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C3@u(r2)
+; LARGE64-NEXT:    ld r3, L..C3@l(r3)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    lfd f1, 0(r3)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ThreadLocalVarInit)
+  %1 = load double, ptr %0, align 8
+  ret double %1
+}
+
+define double @loadTLInit2() {
+; SMALL64-LABEL: loadTLInit2:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C3(r2) # target-flags(ppc-tprel) @ThreadLocalVarInit
+; SMALL64-NEXT:    ld r4, L..C4(r2) # @VarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    lfd f1, 0(r4)
+; SMALL64-NEXT:    lfd f0, 0(r3)
+; SMALL64-NEXT:    xsadddp f1, f0, f1
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadTLInit2:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C3@u(r2)
+; LARGE64-NEXT:    addis r4, L..C4@u(r2)
+; LARGE64-NEXT:    ld r3, L..C3@l(r3)
+; LARGE64-NEXT:    ld r4, L..C4@l(r4)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    lfd f1, 0(r4)
+; LARGE64-NEXT:    lfd f0, 0(r3)
+; LARGE64-NEXT:    xsadddp f1, f0, f1
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ThreadLocalVarInit)
+  %1 = load double, ptr %0, align 8
+  %2 = load double, ptr @VarInit, align 8
+  %add = fadd double %1, %2
+  ret double %add
+}
+
+; TOC Entry Checks.
+
+; SMALL64-LABEL: .toc
+; SMALL64-LABEL: L..C0:
+; SMALL64-NEXT: .tc IThreadLocalVarUninit[TC],IThreadLocalVarUninit[UL]@le
+; SMALL64-LABEL: L..C1:
+; SMALL64-NEXT: .tc IThreadLocalVarInit[TC],IThreadLocalVarInit[TL]@le
+; SMALL64-LABEL: L..C2:
+; SMALL64-NEXT: .tc ThreadLocalVarUninit[TC],ThreadLocalVarUninit[TL]@le
+; SMALL64-LABEL: L..C3:
+; SMALL64-NEXT: .tc ThreadLocalVarInit[TC],ThreadLocalVarInit[TL]@le
+; SMALL64-LABEL: L..C4:
+; SMALL64-NEXT: .tc VarInit[TC],VarInit[RW]
+
+; LARGE64-LABEL: .toc
+; LARGE64-LABEL: L..C0:
+; LARGE64-NEXT: .tc IThreadLocalVarUninit[TE],IThreadLocalVarUninit[UL]@le
+; LARGE64-LABEL: L..C1:
+; LARGE64-NEXT: .tc IThreadLocalVarInit[TE],IThreadLocalVarInit[TL]@le
+; LARGE64-LABEL: L..C2:
+; LARGE64-NEXT: .tc ThreadLocalVarUninit[TE],ThreadLocalVarUninit[TL]@le
+; LARGE64-LABEL: L..C3:
+; LARGE64-NEXT: .tc ThreadLocalVarInit[TE],ThreadLocalVarInit[TL]@le
+; LARGE64-LABEL: L..C4:
+; LARGE64-NEXT: .tc VarInit[TE],VarInit[RW]
diff --git a/llvm/test/CodeGen/PowerPC/aix-tls-le-ldst-float.ll b/llvm/test/CodeGen/PowerPC/aix-tls-le-ldst-float.ll
new file mode 100644
index 0000000000000..2b93616bfe8bd
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-tls-le-ldst-float.ll
@@ -0,0 +1,328 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc  -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \
+; RUN:      -mtriple powerpc64-ibm-aix-xcoff < %s | FileCheck %s \
+; RUN:      --check-prefix=SMALL64
+; RUN: llc  -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \
+; RUN:      -mtriple powerpc64-ibm-aix-xcoff --code-model=large < %s \
+; RUN:      | FileCheck %s --check-prefix=LARGE64
+
+@ThreadLocalVarInit = thread_local(localexec) global float 0x401D333340000000, align 4
+@VarInit = global float 0x4021666660000000, align 4
+@IThreadLocalVarUninit = internal thread_local(localexec) global float 0.000000e+00, align 4
+@IThreadLocalVarInit = internal thread_local(localexec) global float 0x4018CCCCC0000000, align 4
+@ThreadLocalVarUninit = thread_local(localexec) global float 0.000000e+00, align 4
+declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull) #1
+
+define void @storeITLUninit(float noundef %x) {
+; SMALL64-LABEL: storeITLUninit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tprel) @IThreadLocalVarUninit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    stfs f1, 0(r3)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: storeITLUninit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C0@u(r2)
+; LARGE64-NEXT:    ld r3, L..C0@l(r3)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    stfs f1, 0(r3)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @IThreadLocalVarUninit)
+  store float %x, ptr %0, align 4
+  ret void
+}
+
+define void @storeITLInit(float noundef %x) {
+; SMALL64-LABEL: storeITLInit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C1(r2) # target-flags(ppc-tprel) @IThreadLocalVarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    stfs f1, 0(r3)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: storeITLInit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C1@u(r2)
+; LARGE64-NEXT:    ld r3, L..C1@l(r3)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    stfs f1, 0(r3)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @IThreadLocalVarInit)
+  store float %x, ptr %0, align 4
+  ret void
+}
+
+define void @storeTLUninit(float noundef %x) {
+; SMALL64-LABEL: storeTLUninit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C2(r2) # target-flags(ppc-tprel) @ThreadLocalVarUninit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    stfs f1, 0(r3)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: storeTLUninit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C2@u(r2)
+; LARGE64-NEXT:    ld r3, L..C2@l(r3)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    stfs f1, 0(r3)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ThreadLocalVarUninit)
+  store float %x, ptr %0, align 4
+  ret void
+}
+
+define void @storeTLInit(float noundef %x) {
+; SMALL64-LABEL: storeTLInit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C3(r2) # target-flags(ppc-tprel) @ThreadLocalVarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    stfs f1, 0(r3)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: storeTLInit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C3@u(r2)
+; LARGE64-NEXT:    ld r3, L..C3@l(r3)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    stfs f1, 0(r3)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ThreadLocalVarInit)
+  store float %x, ptr %0, align 4
+  ret void
+}
+
+define float @loadITLUninit() {
+; SMALL64-LABEL: loadITLUninit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tprel) @IThreadLocalVarUninit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    lfs f1, 0(r3)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadITLUninit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C0@u(r2)
+; LARGE64-NEXT:    ld r3, L..C0@l(r3)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    lfs f1, 0(r3)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @IThreadLocalVarUninit)
+  %1 = load float, ptr %0, align 4
+  ret float %1
+}
+
+define float @loadITLUninit2() {
+; SMALL64-LABEL: loadITLUninit2:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tprel) @IThreadLocalVarUninit
+; SMALL64-NEXT:    ld r4, L..C4(r2) # @VarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    lfs f1, 0(r4)
+; SMALL64-NEXT:    lfs f0, 0(r3)
+; SMALL64-NEXT:    fadds f1, f0, f1
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadITLUninit2:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C0@u(r2)
+; LARGE64-NEXT:    addis r4, L..C4@u(r2)
+; LARGE64-NEXT:    ld r3, L..C0@l(r3)
+; LARGE64-NEXT:    ld r4, L..C4@l(r4)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    lfs f1, 0(r4)
+; LARGE64-NEXT:    lfs f0, 0(r3)
+; LARGE64-NEXT:    fadds f1, f0, f1
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @IThreadLocalVarUninit)
+  %1 = load float, ptr %0, align 4
+  %2 = load float, ptr @VarInit, align 4
+  %add = fadd float %1, %2
+  ret float %add
+}
+
+define float @loadITLInit() {
+; SMALL64-LABEL: loadITLInit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C1(r2) # target-flags(ppc-tprel) @IThreadLocalVarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    lfs f1, 0(r3)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadITLInit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C1@u(r2)
+; LARGE64-NEXT:    ld r3, L..C1@l(r3)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    lfs f1, 0(r3)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @IThreadLocalVarInit)
+  %1 = load float, ptr %0, align 4
+  ret float %1
+}
+
+define float @loadITLInit2() {
+; SMALL64-LABEL: loadITLInit2:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C1(r2) # target-flags(ppc-tprel) @IThreadLocalVarInit
+; SMALL64-NEXT:    ld r4, L..C4(r2) # @VarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    lfs f1, 0(r4)
+; SMALL64-NEXT:    lfs f0, 0(r3)
+; SMALL64-NEXT:    fadds f1, f0, f1
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadITLInit2:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C1@u(r2)
+; LARGE64-NEXT:    addis r4, L..C4@u(r2)
+; LARGE64-NEXT:    ld r3, L..C1@l(r3)
+; LARGE64-NEXT:    ld r4, L..C4@l(r4)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    lfs f1, 0(r4)
+; LARGE64-NEXT:    lfs f0, 0(r3)
+; LARGE64-NEXT:    fadds f1, f0, f1
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @IThreadLocalVarInit)
+  %1 = load float, ptr %0, align 4
+  %2 = load float, ptr @VarInit, align 4
+  %add = fadd float %1, %2
+  ret float %add
+}
+
+define float @loadTLUninit() {
+; SMALL64-LABEL: loadTLUninit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C2(r2) # target-flags(ppc-tprel) @ThreadLocalVarUninit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    lfs f1, 0(r3)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadTLUninit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C2@u(r2)
+; LARGE64-NEXT:    ld r3, L..C2@l(r3)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    lfs f1, 0(r3)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ThreadLocalVarUninit)
+  %1 = load float, ptr %0, align 4
+  ret float %1
+}
+
+define float @loadTLUninit2() {
+; SMALL64-LABEL: loadTLUninit2:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C2(r2) # target-flags(ppc-tprel) @ThreadLocalVarUninit
+; SMALL64-NEXT:    ld r4, L..C4(r2) # @VarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    lfs f1, 0(r4)
+; SMALL64-NEXT:    lfs f0, 0(r3)
+; SMALL64-NEXT:    fadds f1, f0, f1
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadTLUninit2:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C2@u(r2)
+; LARGE64-NEXT:    addis r4, L..C4@u(r2)
+; LARGE64-NEXT:    ld r3, L..C2@l(r3)
+; LARGE64-NEXT:    ld r4, L..C4@l(r4)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    lfs f1, 0(r4)
+; LARGE64-NEXT:    lfs f0, 0(r3)
+; LARGE64-NEXT:    fadds f1, f0, f1
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ThreadLocalVarUninit)
+  %1 = load float, ptr %0, align 4
+  %2 = load float, ptr @VarInit, align 4
+  %add = fadd float %1, %2
+  ret float %add
+}
+
+define float @loadTLInit() {
+; SMALL64-LABEL: loadTLInit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C3(r2) # target-flags(ppc-tprel) @ThreadLocalVarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    lfs f1, 0(r3)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadTLInit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C3@u(r2)
+; LARGE64-NEXT:    ld r3, L..C3@l(r3)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    lfs f1, 0(r3)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ThreadLocalVarInit)
+  %1 = load float, ptr %0, align 4
+  ret float %1
+}
+
+define float @loadTLInit2() {
+; SMALL64-LABEL: loadTLInit2:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C3(r2) # target-flags(ppc-tprel) @ThreadLocalVarInit
+; SMALL64-NEXT:    ld r4, L..C4(r2) # @VarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    lfs f1, 0(r4)
+; SMALL64-NEXT:    lfs f0, 0(r3)
+; SMALL64-NEXT:    fadds f1, f0, f1
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadTLInit2:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C3@u(r2)
+; LARGE64-NEXT:    addis r4, L..C4@u(r2)
+; LARGE64-NEXT:    ld r3, L..C3@l(r3)
+; LARGE64-NEXT:    ld r4, L..C4@l(r4)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    lfs f1, 0(r4)
+; LARGE64-NEXT:    lfs f0, 0(r3)
+; LARGE64-NEXT:    fadds f1, f0, f1
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ThreadLocalVarInit)
+  %1 = load float, ptr %0, align 4
+  %2 = load float, ptr @VarInit, align 4
+  %add = fadd float %1, %2
+  ret float %add
+}
+
+; TOC Entry Checks.
+
+; SMALL64-LABEL: .toc
+; SMALL64-LABEL: L..C0:
+; SMALL64-NEXT: .tc IThreadLocalVarUninit[TC],IThreadLocalVarUninit[UL]@le
+; SMALL64-LABEL: L..C1:
+; SMALL64-NEXT: .tc IThreadLocalVarInit[TC],IThreadLocalVarInit[TL]@le
+; SMALL64-LABEL: L..C2:
+; SMALL64-NEXT: .tc ThreadLocalVarUninit[TC],ThreadLocalVarUninit[TL]@le
+; SMALL64-LABEL: L..C3:
+; SMALL64-NEXT: .tc ThreadLocalVarInit[TC],ThreadLocalVarInit[TL]@le
+; SMALL64-LABEL: L..C4:
+; SMALL64-NEXT: .tc VarInit[TC],VarInit[RW]
+
+; LARGE64-LABEL: .toc
+; LARGE64-LABEL: L..C0:
+; LARGE64-NEXT: .tc IThreadLocalVarUninit[TE],IThreadLocalVarUninit[UL]@le
+; LARGE64-LABEL: L..C1:
+; LARGE64-NEXT: .tc IThreadLocalVarInit[TE],IThreadLocalVarInit[TL]@le
+; LARGE64-LABEL: L..C2:
+; LARGE64-NEXT: .tc ThreadLocalVarUninit[TE],ThreadLocalVarUninit[TL]@le
+; LARGE64-LABEL: L..C3:
+; LARGE64-NEXT: .tc ThreadLocalVarInit[TE],ThreadLocalVarInit[TL]@le
+; LARGE64-LABEL: L..C4:
+; LARGE64-NEXT: .tc VarInit[TE],VarInit[RW]
diff --git a/llvm/test/CodeGen/PowerPC/aix-tls-le-ldst-int.ll b/llvm/test/CodeGen/PowerPC/aix-tls-le-ldst-int.ll
new file mode 100644
index 0000000000000..1df848481da4b
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-tls-le-ldst-int.ll
@@ -0,0 +1,336 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc  -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \
+; RUN:      -mtriple powerpc64-ibm-aix-xcoff < %s | FileCheck %s \
+; RUN:      --check-prefix=SMALL64
+; RUN: llc  -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \
+; RUN:      -mtriple powerpc64-ibm-aix-xcoff --code-model=large < %s \
+; RUN:      | FileCheck %s --check-prefix=LARGE64
+
+@ThreadLocalVarInit = thread_local(localexec) global i32 1, align 4
+@VarInit = global i32 87, align 4
+@IThreadLocalVarUninit = internal thread_local(localexec) global i32 0, align 4
+@IThreadLocalVarInit = internal thread_local(localexec) global i32 1, align 4
+@ThreadLocalVarUninit = thread_local(localexec) global i32 0, align 4
+declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull) #1
+
+define void @storeITLUninit(i32 noundef signext %x) {
+; SMALL64-LABEL: storeITLUninit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r4, L..C0(r2) # target-flags(ppc-tprel) @IThreadLocalVarUninit
+; SMALL64-NEXT:    add r4, r13, r4
+; SMALL64-NEXT:    stw r3, 0(r4)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: storeITLUninit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r4, L..C0@u(r2)
+; LARGE64-NEXT:    ld r4, L..C0@l(r4)
+; LARGE64-NEXT:    add r4, r13, r4
+; LARGE64-NEXT:    stw r3, 0(r4)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @IThreadLocalVarUninit)
+  store i32 %x, ptr %0, align 4
+  ret void
+}
+
+define void @storeITLInit(i32 noundef signext %x) {
+; SMALL64-LABEL: storeITLInit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r4, L..C1(r2) # target-flags(ppc-tprel) @IThreadLocalVarInit
+; SMALL64-NEXT:    add r4, r13, r4
+; SMALL64-NEXT:    stw r3, 0(r4)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: storeITLInit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r4, L..C1@u(r2)
+; LARGE64-NEXT:    ld r4, L..C1@l(r4)
+; LARGE64-NEXT:    add r4, r13, r4
+; LARGE64-NEXT:    stw r3, 0(r4)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @IThreadLocalVarInit)
+  store i32 %x, ptr %0, align 4
+  ret void
+}
+
+define void @storeTLUninit(i32 noundef signext %x) {
+; SMALL64-LABEL: storeTLUninit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r4, L..C2(r2) # target-flags(ppc-tprel) @ThreadLocalVarUninit
+; SMALL64-NEXT:    add r4, r13, r4
+; SMALL64-NEXT:    stw r3, 0(r4)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: storeTLUninit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r4, L..C2@u(r2)
+; LARGE64-NEXT:    ld r4, L..C2@l(r4)
+; LARGE64-NEXT:    add r4, r13, r4
+; LARGE64-NEXT:    stw r3, 0(r4)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ThreadLocalVarUninit)
+  store i32 %x, ptr %0, align 4
+  ret void
+}
+
+define void @storeTLInit(i32 noundef signext %x) {
+; SMALL64-LABEL: storeTLInit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r4, L..C3(r2) # target-flags(ppc-tprel) @ThreadLocalVarInit
+; SMALL64-NEXT:    add r4, r13, r4
+; SMALL64-NEXT:    stw r3, 0(r4)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: storeTLInit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r4, L..C3@u(r2)
+; LARGE64-NEXT:    ld r4, L..C3@l(r4)
+; LARGE64-NEXT:    add r4, r13, r4
+; LARGE64-NEXT:    stw r3, 0(r4)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ThreadLocalVarInit)
+  store i32 %x, ptr %0, align 4
+  ret void
+}
+
+define signext i32 @loadITLUninit() {
+; SMALL64-LABEL: loadITLUninit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tprel) @IThreadLocalVarUninit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    lwa r3, 0(r3)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadITLUninit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C0@u(r2)
+; LARGE64-NEXT:    ld r3, L..C0@l(r3)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    lwa r3, 0(r3)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @IThreadLocalVarUninit)
+  %1 = load i32, ptr %0, align 4
+  ret i32 %1
+}
+
+define signext i32 @loadITLUninit2() {
+; SMALL64-LABEL: loadITLUninit2:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tprel) @IThreadLocalVarUninit
+; SMALL64-NEXT:    ld r4, L..C4(r2) # @VarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    lwz r4, 0(r4)
+; SMALL64-NEXT:    lwz r3, 0(r3)
+; SMALL64-NEXT:    add r3, r4, r3
+; SMALL64-NEXT:    extsw r3, r3
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadITLUninit2:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C0@u(r2)
+; LARGE64-NEXT:    addis r4, L..C4@u(r2)
+; LARGE64-NEXT:    ld r3, L..C0@l(r3)
+; LARGE64-NEXT:    ld r4, L..C4@l(r4)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    lwz r4, 0(r4)
+; LARGE64-NEXT:    lwz r3, 0(r3)
+; LARGE64-NEXT:    add r3, r4, r3
+; LARGE64-NEXT:    extsw r3, r3
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @IThreadLocalVarUninit)
+  %1 = load i32, ptr %0, align 4
+  %2 = load i32, ptr @VarInit, align 4
+  %add = add nsw i32 %2, %1
+  ret i32 %add
+}
+
+define signext i32 @loadITLInit() {
+; SMALL64-LABEL: loadITLInit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C1(r2) # target-flags(ppc-tprel) @IThreadLocalVarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    lwa r3, 0(r3)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadITLInit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C1@u(r2)
+; LARGE64-NEXT:    ld r3, L..C1@l(r3)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    lwa r3, 0(r3)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @IThreadLocalVarInit)
+  %1 = load i32, ptr %0, align 4
+  ret i32 %1
+}
+
+define signext i32 @loadITLInit2() {
+; SMALL64-LABEL: loadITLInit2:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C1(r2) # target-flags(ppc-tprel) @IThreadLocalVarInit
+; SMALL64-NEXT:    ld r4, L..C4(r2) # @VarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    lwz r4, 0(r4)
+; SMALL64-NEXT:    lwz r3, 0(r3)
+; SMALL64-NEXT:    add r3, r4, r3
+; SMALL64-NEXT:    extsw r3, r3
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadITLInit2:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C1@u(r2)
+; LARGE64-NEXT:    addis r4, L..C4@u(r2)
+; LARGE64-NEXT:    ld r3, L..C1@l(r3)
+; LARGE64-NEXT:    ld r4, L..C4@l(r4)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    lwz r4, 0(r4)
+; LARGE64-NEXT:    lwz r3, 0(r3)
+; LARGE64-NEXT:    add r3, r4, r3
+; LARGE64-NEXT:    extsw r3, r3
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @IThreadLocalVarInit)
+  %1 = load i32, ptr %0, align 4
+  %2 = load i32, ptr @VarInit, align 4
+  %add = add nsw i32 %2, %1
+  ret i32 %add
+}
+
+define signext i32 @loadTLUninit() {
+; SMALL64-LABEL: loadTLUninit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C2(r2) # target-flags(ppc-tprel) @ThreadLocalVarUninit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    lwa r3, 0(r3)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadTLUninit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C2@u(r2)
+; LARGE64-NEXT:    ld r3, L..C2@l(r3)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    lwa r3, 0(r3)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ThreadLocalVarUninit)
+  %1 = load i32, ptr %0, align 4
+  ret i32 %1
+}
+
+define signext i32 @loadTLUninit2() {
+; SMALL64-LABEL: loadTLUninit2:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C2(r2) # target-flags(ppc-tprel) @ThreadLocalVarUninit
+; SMALL64-NEXT:    ld r4, L..C4(r2) # @VarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    lwz r4, 0(r4)
+; SMALL64-NEXT:    lwz r3, 0(r3)
+; SMALL64-NEXT:    add r3, r4, r3
+; SMALL64-NEXT:    extsw r3, r3
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadTLUninit2:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C2@u(r2)
+; LARGE64-NEXT:    addis r4, L..C4@u(r2)
+; LARGE64-NEXT:    ld r3, L..C2@l(r3)
+; LARGE64-NEXT:    ld r4, L..C4@l(r4)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    lwz r4, 0(r4)
+; LARGE64-NEXT:    lwz r3, 0(r3)
+; LARGE64-NEXT:    add r3, r4, r3
+; LARGE64-NEXT:    extsw r3, r3
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ThreadLocalVarUninit)
+  %1 = load i32, ptr %0, align 4
+  %2 = load i32, ptr @VarInit, align 4
+  %add = add nsw i32 %2, %1
+  ret i32 %add
+}
+
+define signext i32 @loadTLInit() {
+; SMALL64-LABEL: loadTLInit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C3(r2) # target-flags(ppc-tprel) @ThreadLocalVarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    lwa r3, 0(r3)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadTLInit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C3@u(r2)
+; LARGE64-NEXT:    ld r3, L..C3@l(r3)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    lwa r3, 0(r3)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ThreadLocalVarInit)
+  %1 = load i32, ptr %0, align 4
+  ret i32 %1
+}
+
+define signext i32 @loadTLInit2() {
+; SMALL64-LABEL: loadTLInit2:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C3(r2) # target-flags(ppc-tprel) @ThreadLocalVarInit
+; SMALL64-NEXT:    ld r4, L..C4(r2) # @VarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    lwz r4, 0(r4)
+; SMALL64-NEXT:    lwz r3, 0(r3)
+; SMALL64-NEXT:    add r3, r4, r3
+; SMALL64-NEXT:    extsw r3, r3
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadTLInit2:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C3@u(r2)
+; LARGE64-NEXT:    addis r4, L..C4@u(r2)
+; LARGE64-NEXT:    ld r3, L..C3@l(r3)
+; LARGE64-NEXT:    ld r4, L..C4@l(r4)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    lwz r4, 0(r4)
+; LARGE64-NEXT:    lwz r3, 0(r3)
+; LARGE64-NEXT:    add r3, r4, r3
+; LARGE64-NEXT:    extsw r3, r3
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ThreadLocalVarInit)
+  %1 = load i32, ptr %0, align 4
+  %2 = load i32, ptr @VarInit, align 4
+  %add = add nsw i32 %2, %1
+  ret i32 %add
+}
+
+; TOC Entry Checks.
+
+; SMALL64-LABEL: .toc
+; SMALL64-LABEL: L..C0:
+; SMALL64-NEXT: .tc IThreadLocalVarUninit[TC],IThreadLocalVarUninit[UL]@le
+; SMALL64-LABEL: L..C1:
+; SMALL64-NEXT: .tc IThreadLocalVarInit[TC],IThreadLocalVarInit[TL]@le
+; SMALL64-LABEL: L..C2:
+; SMALL64-NEXT: .tc ThreadLocalVarUninit[TC],ThreadLocalVarUninit[TL]@le
+; SMALL64-LABEL: L..C3:
+; SMALL64-NEXT: .tc ThreadLocalVarInit[TC],ThreadLocalVarInit[TL]@le
+; SMALL64-LABEL: L..C4:
+; SMALL64-NEXT: .tc VarInit[TC],VarInit[RW]
+
+; LARGE64-LABEL: .toc
+; LARGE64-LABEL: L..C0:
+; LARGE64-NEXT: .tc IThreadLocalVarUninit[TE],IThreadLocalVarUninit[UL]@le
+; LARGE64-LABEL: L..C1:
+; LARGE64-NEXT: .tc IThreadLocalVarInit[TE],IThreadLocalVarInit[TL]@le
+; LARGE64-LABEL: L..C2:
+; LARGE64-NEXT: .tc ThreadLocalVarUninit[TE],ThreadLocalVarUninit[TL]@le
+; LARGE64-LABEL: L..C3:
+; LARGE64-NEXT: .tc ThreadLocalVarInit[TE],ThreadLocalVarInit[TL]@le
+; LARGE64-LABEL: L..C4:
+; LARGE64-NEXT: .tc VarInit[TE],VarInit[RW]
diff --git a/llvm/test/CodeGen/PowerPC/aix-tls-le-ldst-longlong.ll b/llvm/test/CodeGen/PowerPC/aix-tls-le-ldst-longlong.ll
new file mode 100644
index 0000000000000..8ef680909f91b
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-tls-le-ldst-longlong.ll
@@ -0,0 +1,328 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc  -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \
+; RUN:      -mtriple powerpc64-ibm-aix-xcoff < %s | FileCheck %s \
+; RUN:      --check-prefix=SMALL64
+; RUN: llc  -verify-machineinstrs -mcpu=pwr7 -ppc-asm-full-reg-names \
+; RUN:      -mtriple powerpc64-ibm-aix-xcoff --code-model=large < %s \
+; RUN:      | FileCheck %s --check-prefix=LARGE64
+
+@ThreadLocalVarInit = thread_local(localexec) global i64 1, align 8
+@VarInit = global i64 87, align 8
+@IThreadLocalVarUninit = internal thread_local(localexec) global i64 0, align 8
+@IThreadLocalVarInit = internal thread_local(localexec) global i64 1, align 8
+@ThreadLocalVarUninit = thread_local(localexec) global i64 0, align 8
+declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull) #1
+
+define void @storeITLUninit(i64 noundef %x) {
+; SMALL64-LABEL: storeITLUninit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r4, L..C0(r2) # target-flags(ppc-tprel) @IThreadLocalVarUninit
+; SMALL64-NEXT:    add r4, r13, r4
+; SMALL64-NEXT:    std r3, 0(r4)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: storeITLUninit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r4, L..C0@u(r2)
+; LARGE64-NEXT:    ld r4, L..C0@l(r4)
+; LARGE64-NEXT:    add r4, r13, r4
+; LARGE64-NEXT:    std r3, 0(r4)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @IThreadLocalVarUninit)
+  store i64 %x, ptr %0, align 8
+  ret void
+}
+
+define void @storeITLInit(i64 noundef %x) {
+; SMALL64-LABEL: storeITLInit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r4, L..C1(r2) # target-flags(ppc-tprel) @IThreadLocalVarInit
+; SMALL64-NEXT:    add r4, r13, r4
+; SMALL64-NEXT:    std r3, 0(r4)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: storeITLInit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r4, L..C1@u(r2)
+; LARGE64-NEXT:    ld r4, L..C1@l(r4)
+; LARGE64-NEXT:    add r4, r13, r4
+; LARGE64-NEXT:    std r3, 0(r4)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @IThreadLocalVarInit)
+  store i64 %x, ptr %0, align 8
+  ret void
+}
+
+define void @storeTLUninit(i64 noundef %x) {
+; SMALL64-LABEL: storeTLUninit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r4, L..C2(r2) # target-flags(ppc-tprel) @ThreadLocalVarUninit
+; SMALL64-NEXT:    add r4, r13, r4
+; SMALL64-NEXT:    std r3, 0(r4)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: storeTLUninit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r4, L..C2@u(r2)
+; LARGE64-NEXT:    ld r4, L..C2@l(r4)
+; LARGE64-NEXT:    add r4, r13, r4
+; LARGE64-NEXT:    std r3, 0(r4)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ThreadLocalVarUninit)
+  store i64 %x, ptr %0, align 8
+  ret void
+}
+
+define void @storeTLInit(i64 noundef %x) {
+; SMALL64-LABEL: storeTLInit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r4, L..C3(r2) # target-flags(ppc-tprel) @ThreadLocalVarInit
+; SMALL64-NEXT:    add r4, r13, r4
+; SMALL64-NEXT:    std r3, 0(r4)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: storeTLInit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r4, L..C3@u(r2)
+; LARGE64-NEXT:    ld r4, L..C3@l(r4)
+; LARGE64-NEXT:    add r4, r13, r4
+; LARGE64-NEXT:    std r3, 0(r4)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ThreadLocalVarInit)
+  store i64 %x, ptr %0, align 8
+  ret void
+}
+
+define i64 @loadITLUninit() {
+; SMALL64-LABEL: loadITLUninit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tprel) @IThreadLocalVarUninit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    ld r3, 0(r3)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadITLUninit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C0@u(r2)
+; LARGE64-NEXT:    ld r3, L..C0@l(r3)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    ld r3, 0(r3)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @IThreadLocalVarUninit)
+  %1 = load i64, ptr %0, align 8
+  ret i64 %1
+}
+
+define i64 @loadITLUninit2() {
+; SMALL64-LABEL: loadITLUninit2:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C0(r2) # target-flags(ppc-tprel) @IThreadLocalVarUninit
+; SMALL64-NEXT:    ld r4, L..C4(r2) # @VarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    ld r4, 0(r4)
+; SMALL64-NEXT:    ld r3, 0(r3)
+; SMALL64-NEXT:    add r3, r4, r3
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadITLUninit2:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C0@u(r2)
+; LARGE64-NEXT:    addis r4, L..C4@u(r2)
+; LARGE64-NEXT:    ld r3, L..C0@l(r3)
+; LARGE64-NEXT:    ld r4, L..C4@l(r4)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    ld r4, 0(r4)
+; LARGE64-NEXT:    ld r3, 0(r3)
+; LARGE64-NEXT:    add r3, r4, r3
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @IThreadLocalVarUninit)
+  %1 = load i64, ptr %0, align 8
+  %2 = load i64, ptr @VarInit, align 8
+  %add = add nsw i64 %2, %1
+  ret i64 %add
+}
+
+define i64 @loadITLInit() {
+; SMALL64-LABEL: loadITLInit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C1(r2) # target-flags(ppc-tprel) @IThreadLocalVarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    ld r3, 0(r3)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadITLInit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C1@u(r2)
+; LARGE64-NEXT:    ld r3, L..C1@l(r3)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    ld r3, 0(r3)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @IThreadLocalVarInit)
+  %1 = load i64, ptr %0, align 8
+  ret i64 %1
+}
+
+define i64 @loadITLInit2() {
+; SMALL64-LABEL: loadITLInit2:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C1(r2) # target-flags(ppc-tprel) @IThreadLocalVarInit
+; SMALL64-NEXT:    ld r4, L..C4(r2) # @VarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    ld r4, 0(r4)
+; SMALL64-NEXT:    ld r3, 0(r3)
+; SMALL64-NEXT:    add r3, r4, r3
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadITLInit2:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C1@u(r2)
+; LARGE64-NEXT:    addis r4, L..C4@u(r2)
+; LARGE64-NEXT:    ld r3, L..C1@l(r3)
+; LARGE64-NEXT:    ld r4, L..C4@l(r4)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    ld r4, 0(r4)
+; LARGE64-NEXT:    ld r3, 0(r3)
+; LARGE64-NEXT:    add r3, r4, r3
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @IThreadLocalVarInit)
+  %1 = load i64, ptr %0, align 8
+  %2 = load i64, ptr @VarInit, align 8
+  %add = add nsw i64 %2, %1
+  ret i64 %add
+}
+
+define i64 @loadTLUninit() {
+; SMALL64-LABEL: loadTLUninit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C2(r2) # target-flags(ppc-tprel) @ThreadLocalVarUninit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    ld r3, 0(r3)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadTLUninit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C2@u(r2)
+; LARGE64-NEXT:    ld r3, L..C2@l(r3)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    ld r3, 0(r3)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ThreadLocalVarUninit)
+  %1 = load i64, ptr %0, align 8
+  ret i64 %1
+}
+
+define i64 @loadTLUninit2() {
+; SMALL64-LABEL: loadTLUninit2:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C2(r2) # target-flags(ppc-tprel) @ThreadLocalVarUninit
+; SMALL64-NEXT:    ld r4, L..C4(r2) # @VarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    ld r4, 0(r4)
+; SMALL64-NEXT:    ld r3, 0(r3)
+; SMALL64-NEXT:    add r3, r4, r3
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadTLUninit2:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C2@u(r2)
+; LARGE64-NEXT:    addis r4, L..C4@u(r2)
+; LARGE64-NEXT:    ld r3, L..C2@l(r3)
+; LARGE64-NEXT:    ld r4, L..C4@l(r4)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    ld r4, 0(r4)
+; LARGE64-NEXT:    ld r3, 0(r3)
+; LARGE64-NEXT:    add r3, r4, r3
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ThreadLocalVarUninit)
+  %1 = load i64, ptr %0, align 8
+  %2 = load i64, ptr @VarInit, align 8
+  %add = add nsw i64 %2, %1
+  ret i64 %add
+}
+
+define i64 @loadTLInit() {
+; SMALL64-LABEL: loadTLInit:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C3(r2) # target-flags(ppc-tprel) @ThreadLocalVarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    ld r3, 0(r3)
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadTLInit:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C3@u(r2)
+; LARGE64-NEXT:    ld r3, L..C3@l(r3)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    ld r3, 0(r3)
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ThreadLocalVarInit)
+  %1 = load i64, ptr %0, align 8
+  ret i64 %1
+}
+
+define i64 @loadTLInit2() {
+; SMALL64-LABEL: loadTLInit2:
+; SMALL64:       # %bb.0: # %entry
+; SMALL64-NEXT:    ld r3, L..C3(r2) # target-flags(ppc-tprel) @ThreadLocalVarInit
+; SMALL64-NEXT:    ld r4, L..C4(r2) # @VarInit
+; SMALL64-NEXT:    add r3, r13, r3
+; SMALL64-NEXT:    ld r4, 0(r4)
+; SMALL64-NEXT:    ld r3, 0(r3)
+; SMALL64-NEXT:    add r3, r4, r3
+; SMALL64-NEXT:    blr
+;
+; LARGE64-LABEL: loadTLInit2:
+; LARGE64:       # %bb.0: # %entry
+; LARGE64-NEXT:    addis r3, L..C3@u(r2)
+; LARGE64-NEXT:    addis r4, L..C4@u(r2)
+; LARGE64-NEXT:    ld r3, L..C3@l(r3)
+; LARGE64-NEXT:    ld r4, L..C4@l(r4)
+; LARGE64-NEXT:    add r3, r13, r3
+; LARGE64-NEXT:    ld r4, 0(r4)
+; LARGE64-NEXT:    ld r3, 0(r3)
+; LARGE64-NEXT:    add r3, r4, r3
+; LARGE64-NEXT:    blr
+entry:
+  %0 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ThreadLocalVarInit)
+  %1 = load i64, ptr %0, align 8
+  %2 = load i64, ptr @VarInit, align 8
+  %add = add nsw i64 %2, %1
+  ret i64 %add
+}
+
+; TOC Entry Checks.
+
+; SMALL64-LABEL: .toc
+; SMALL64-LABEL: L..C0:
+; SMALL64-NEXT: .tc IThreadLocalVarUninit[TC],IThreadLocalVarUninit[UL]@le
+; SMALL64-LABEL: L..C1:
+; SMALL64-NEXT: .tc IThreadLocalVarInit[TC],IThreadLocalVarInit[TL]@le
+; SMALL64-LABEL: L..C2:
+; SMALL64-NEXT: .tc ThreadLocalVarUninit[TC],ThreadLocalVarUninit[TL]@le
+; SMALL64-LABEL: L..C3:
+; SMALL64-NEXT: .tc ThreadLocalVarInit[TC],ThreadLocalVarInit[TL]@le
+; SMALL64-LABEL: L..C4:
+; SMALL64-NEXT: .tc VarInit[TC],VarInit[RW]
+
+; LARGE64-LABEL: .toc
+; LARGE64-LABEL: L..C0:
+; LARGE64-NEXT: .tc IThreadLocalVarUninit[TE],IThreadLocalVarUninit[UL]@le
+; LARGE64-LABEL: L..C1:
+; LARGE64-NEXT: .tc IThreadLocalVarInit[TE],IThreadLocalVarInit[TL]@le
+; LARGE64-LABEL: L..C2:
+; LARGE64-NEXT: .tc ThreadLocalVarUninit[TE],ThreadLocalVarUninit[TL]@le
+; LARGE64-LABEL: L..C3:
+; LARGE64-NEXT: .tc ThreadLocalVarInit[TE],ThreadLocalVarInit[TL]@le
+; LARGE64-LABEL: L..C4:
+; LARGE64-NEXT: .tc VarInit[TE],VarInit[RW]
diff --git a/llvm/test/CodeGen/PowerPC/aix-tls-le-xcoff-reloc-large.ll b/llvm/test/CodeGen/PowerPC/aix-tls-le-xcoff-reloc-large.ll
new file mode 100644
index 0000000000000..5404fdae4c03a
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-tls-le-xcoff-reloc-large.ll
@@ -0,0 +1,218 @@
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff \
+; RUN:     -xcoff-traceback-table=false --code-model=large -filetype=obj -o %t.o < %s
+; RUN: llvm-readobj --relocs --expand-relocs %t.o | FileCheck --check-prefix=RELOC %s
+; RUN: llvm-readobj --syms %t.o | FileCheck --check-prefix=SYM %s
+; RUN: llvm-objdump -D -r --symbol-description %t.o | FileCheck --check-prefix=DIS %s
+
+@ThreadLocalVarInit = thread_local(localexec) global i64 1, align 8
+@VarInit = global i64 87, align 8
+@IThreadLocalVarUninit = internal thread_local(localexec) global i64 0, align 8
+declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull)
+
+define void @storeITLUninit(i64 noundef %x) {
+entry:
+  %0 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @IThreadLocalVarUninit)
+  store i64 %x, ptr %0, align 8
+  ret void
+}
+
+define i64 @loadTLInit() {
+entry:
+  %0 = tail call align 8 ptr @llvm.threadlocal.address.p0(ptr align 8 @ThreadLocalVarInit)
+  %1 = load i64, ptr %0, align 8
+  %2 = load i64, ptr @VarInit, align 8
+  %add = add nsw i64 %2, %1
+  ret i64 %add
+}
+
+; RELOC:      File: {{.*}}aix-tls-le-xcoff-reloc-large.ll.tmp.o
+; RELOC-NEXT: Format: aix5coff64-rs6000
+; RELOC-NEXT: Arch: powerpc64
+; RELOC-NEXT: AddressSize: 64bit
+; RELOC-NEXT: Relocations [
+; RELOC:       Virtual Address: 0x2
+; RELOC-NEXT:       Symbol: IThreadLocalVarUninit (15)
+; RELOC-NEXT:       IsSigned: No
+; RELOC-NEXT:       FixupBitValue: 0
+; RELOC-NEXT:       Length: 16
+; RELOC-NEXT:       Type: R_TOCU (0x30)
+; RELOC-NEXT:     }
+; RELOC:       Virtual Address: 0x6
+; RELOC-NEXT:       Symbol: IThreadLocalVarUninit (15)
+; RELOC-NEXT:       IsSigned: No
+; RELOC-NEXT:       FixupBitValue: 0
+; RELOC-NEXT:       Length: 16
+; RELOC-NEXT:       Type: R_TOCL (0x31)
+; RELOC-NEXT:     }
+; RELOC:       Virtual Address: 0x22
+; RELOC-NEXT:       Symbol: ThreadLocalVarInit (17)
+; RELOC-NEXT:       IsSigned: No
+; RELOC-NEXT:       FixupBitValue: 0
+; RELOC-NEXT:       Length: 16
+; RELOC-NEXT:       Type: R_TOCU (0x30)
+; RELOC-NEXT:     }
+; RELOC:       Virtual Address: 0x2A
+; RELOC-NEXT:       Symbol: ThreadLocalVarInit (17)
+; RELOC-NEXT:       IsSigned: No
+; RELOC-NEXT:       FixupBitValue: 0
+; RELOC-NEXT:       Length: 16
+; RELOC-NEXT:       Type: R_TOCL (0x31)
+; RELOC-NEXT:     }
+; RELOC:       Virtual Address: 0x80
+; RELOC-NEXT:       Symbol: IThreadLocalVarUninit (23)
+; RELOC-NEXT:       IsSigned: No
+; RELOC-NEXT:       FixupBitValue: 0
+; RELOC-NEXT:       Length: 64
+; RELOC-NEXT:       Type: R_TLS_LE (0x23)
+; RELOC-NEXT:     }
+; RELOC:       Virtual Address: 0x88
+; RELOC-NEXT:       Symbol: ThreadLocalVarInit (21)
+; RELOC-NEXT:       IsSigned: No
+; RELOC-NEXT:       FixupBitValue: 0
+; RELOC-NEXT:       Length: 64
+; RELOC-NEXT:       Type: R_TLS_LE (0x23)
+; RELOC-NEXT:     }
+
+; SYM:      File: {{.*}}aix-tls-le-xcoff-reloc-large.ll.tmp.o
+; SYM-NEXT: Format: aix5coff64-rs6000
+; SYM-NEXT: Arch: powerpc64
+; SYM-NEXT: AddressSize: 64bit
+; SYM-NEXT: Symbols [
+; SYM:     Index: 15
+; SYM-NEXT:     Name: IThreadLocalVarUninit
+; SYM-NEXT:     Value (RelocatableAddress): 0x80
+; SYM-NEXT:     Section: .data
+; SYM-NEXT:     Type: 0x0
+; SYM-NEXT:     StorageClass: C_HIDEXT (0x6B)
+; SYM-NEXT:     NumberOfAuxEntries: 1
+; SYM-NEXT:     CSECT Auxiliary Entry {
+; SYM-NEXT:       Index: 16
+; SYM-NEXT:       SectionLen: 8
+; SYM-NEXT:       ParameterHashIndex: 0x0
+; SYM-NEXT:       TypeChkSectNum: 0x0
+; SYM-NEXT:       SymbolAlignmentLog2: 3
+; SYM-NEXT:       SymbolType: XTY_SD (0x1)
+; SYM-NEXT:       StorageMappingClass: XMC_TE (0x16)
+; SYM-NEXT:       Auxiliary Type: AUX_CSECT (0xFB)
+; SYM-NEXT:     }
+; SYM-NEXT:   }
+; SYM:     Index: 17
+; SYM-NEXT:     Name: ThreadLocalVarInit
+; SYM-NEXT:     Value (RelocatableAddress): 0x88
+; SYM-NEXT:     Section: .data
+; SYM-NEXT:     Type: 0x0
+; SYM-NEXT:     StorageClass: C_HIDEXT (0x6B)
+; SYM-NEXT:     NumberOfAuxEntries: 1
+; SYM-NEXT:     CSECT Auxiliary Entry {
+; SYM-NEXT:       Index: 18
+; SYM-NEXT:       SectionLen: 8
+; SYM-NEXT:       ParameterHashIndex: 0x0
+; SYM-NEXT:       TypeChkSectNum: 0x0
+; SYM-NEXT:       SymbolAlignmentLog2: 3
+; SYM-NEXT:       SymbolType: XTY_SD (0x1)
+; SYM-NEXT:       StorageMappingClass: XMC_TE (0x16)
+; SYM-NEXT:       Auxiliary Type: AUX_CSECT (0xFB)
+; SYM-NEXT:     }
+; SYM-NEXT:   }
+; SYM:     Index: 21
+; SYM-NEXT:     Name: ThreadLocalVarInit
+; SYM-NEXT:     Value (RelocatableAddress): 0x0
+; SYM-NEXT:     Section: .tdata
+; SYM-NEXT:     Type: 0x0
+; SYM-NEXT:     StorageClass: C_EXT (0x2)
+; SYM-NEXT:     NumberOfAuxEntries: 1
+; SYM-NEXT:     CSECT Auxiliary Entry {
+; SYM-NEXT:       Index: 22
+; SYM-NEXT:       SectionLen: 8
+; SYM-NEXT:       ParameterHashIndex: 0x0
+; SYM-NEXT:       TypeChkSectNum: 0x0
+; SYM-NEXT:       SymbolAlignmentLog2: 3
+; SYM-NEXT:       SymbolType: XTY_SD (0x1)
+; SYM-NEXT:       StorageMappingClass: XMC_TL (0x14)
+; SYM-NEXT:       Auxiliary Type: AUX_CSECT (0xFB)
+; SYM-NEXT:     }
+; SYM-NEXT:   }
+; SYM:     Index: 23
+; SYM-NEXT:     Name: IThreadLocalVarUninit
+; SYM-NEXT:     Value (RelocatableAddress): 0x8
+; SYM-NEXT:     Section: .tbss
+; SYM-NEXT:     Type: 0x0
+; SYM-NEXT:     StorageClass: C_HIDEXT (0x6B)
+; SYM-NEXT:     NumberOfAuxEntries: 1
+; SYM-NEXT:     CSECT Auxiliary Entry {
+; SYM-NEXT:       Index: 24
+; SYM-NEXT:       SectionLen: 8
+; SYM-NEXT:       ParameterHashIndex: 0x0
+; SYM-NEXT:       TypeChkSectNum: 0x0
+; SYM-NEXT:       SymbolAlignmentLog2: 3
+; SYM-NEXT:       SymbolType: XTY_CM (0x3)
+; SYM-NEXT:       StorageMappingClass: XMC_UL (0x15)
+; SYM-NEXT:       Auxiliary Type: AUX_CSECT (0xFB)
+; SYM-NEXT:     }
+; SYM-NEXT:   }
+
+; DIS:      {{.*}}aix-tls-le-xcoff-reloc-large.ll.tmp.o:	file format aix5coff64-rs6000
+; DIS:      Disassembly of section .text:
+; DIS:      0000000000000000 (idx: 3) .storeITLUninit:
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                addis 4, 2, 0
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU	(idx: 15) IThreadLocalVarUninit[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 4, 0(4)
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL	(idx: 15) IThreadLocalVarUninit[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                add 4, 13, 4
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                std 3, 0(4)
+; DIS-NEXT:                                       blr
+; DIS:      0000000000000020 (idx: 5) .loadTLInit:
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                addis 3, 2, 0
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU       (idx: 17) ThreadLocalVarInit[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                addis 4, 2, 0
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCU       (idx: 19) VarInit[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 3, 8(3)
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL       (idx: 17) ThreadLocalVarInit[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 4, 16(4)
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOCL       (idx: 19) VarInit[TE]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                add 3, 13, 3
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 4, 0(4)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                ld 3, 0(3)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                add 3, 4, 3
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}                blr
+
+; DIS:      Disassembly of section .data:
+; DIS:      0000000000000048 (idx: 7) VarInit[RW]:
+; DIS-NEXT:       48: 00 00 00 00
+; DIS-NEXT:       4c: 00 00 00 57
+; DIS:      0000000000000050 (idx: 9) storeITLUninit[DS]:
+; DIS-NEXT:       50: 00 00 00 00
+; DIS-NEXT: 0000000000000050:  R_POS        (idx: 3) .storeITLUninit
+; DIS-NEXT:       54: 00 00 00 00
+; DIS-NEXT:       58: 00 00 00 00
+; DIS-NEXT: 0000000000000058:  R_POS        (idx: 13) TOC[TC0]
+; DIS-NEXT:       5c: 00 00 00 80
+; DIS:      0000000000000068 (idx: 11) loadTLInit[DS]:
+; DIS-NEXT:       68: 00 00 00 00
+; DIS-NEXT: 0000000000000068:  R_POS        (idx: 5) .loadTLInit
+; DIS-NEXT:       6c: 00 00 00 20
+; DIS-NEXT:       70: 00 00 00 00
+; DIS-NEXT: 0000000000000070:  R_POS        (idx: 13) TOC[TC0]
+; DIS-NEXT:       74: 00 00 00 80
+; DIS:      0000000000000080 (idx: 15) IThreadLocalVarUninit[TE]:
+; DIS-NEXT:       80: 00 00 00 00
+; DIS-NEXT: 0000000000000080:  R_TLS_LE     (idx: 23) IThreadLocalVarUninit[UL]
+; DIS-NEXT:       84: 00 00 00 00
+; DIS:      0000000000000088 (idx: 17) ThreadLocalVarInit[TE]:
+; DIS-NEXT:       88: 00 00 00 00
+; DIS-NEXT: 0000000000000088:  R_TLS_LE     (idx: 21) ThreadLocalVarInit[TL]
+; DIS-NEXT:       8c: 00 00 00 00
+; DIS:      0000000000000090 (idx: 19) VarInit[TE]:
+; DIS-NEXT:       90: 00 00 00 00
+; DIS-NEXT: 0000000000000090:  R_POS        (idx: 7) VarInit[RW]
+; DIS-NEXT:       94: 00 00 00 48
+
+; DIS:      Disassembly of section .tdata:
+; DIS:      0000000000000000 (idx: 21) ThreadLocalVarInit[TL]:
+; DIS-NEXT:        0: 00 00 00 00
+; DIS-NEXT:        4: 00 00 00 01
+
+; DIS:      Disassembly of section .tbss:
+; DIS:      0000000000000008 (idx: 23) IThreadLocalVarUninit[UL]:
+; DIS-NEXT: ...
+
diff --git a/llvm/test/CodeGen/PowerPC/aix-tls-le-xcoff-reloc.ll b/llvm/test/CodeGen/PowerPC/aix-tls-le-xcoff-reloc.ll
new file mode 100644
index 0000000000000..73a81a3d6f681
--- /dev/null
+++ b/llvm/test/CodeGen/PowerPC/aix-tls-le-xcoff-reloc.ll
@@ -0,0 +1,194 @@
+; RUN: llc -verify-machineinstrs -mcpu=pwr7 -mattr=-altivec -mtriple powerpc64-ibm-aix-xcoff \
+; RUN:     -xcoff-traceback-table=false -data-sections=false -filetype=obj -o %t.o < %s
+; RUN: llvm-readobj --relocs --expand-relocs %t.o | FileCheck --check-prefix=RELOC %s
+; RUN: llvm-readobj --syms %t.o | FileCheck --check-prefix=SYM %s
+; RUN: llvm-objdump -D -r --symbol-description %t.o | FileCheck --check-prefix=DIS %s
+
+@ThreadLocalVarInit = thread_local(localexec) global i32 1, align 4
+@VarInit = global i32 87, align 4
+@IThreadLocalVarUninit = internal thread_local(localexec) global i32 0, align 4
+declare nonnull ptr @llvm.threadlocal.address.p0(ptr nonnull)
+
+define void @storeITLUninit(i32 noundef signext %x) {
+entry:
+  %0 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @IThreadLocalVarUninit)
+  store i32 %x, ptr %0, align 4
+  ret void
+}
+
+define signext i32 @loadTLInit() {
+entry:
+  %0 = tail call align 4 ptr @llvm.threadlocal.address.p0(ptr align 4 @ThreadLocalVarInit)
+  %1 = load i32, ptr %0, align 4
+  %2 = load i32, ptr @VarInit, align 4
+  %add = add nsw i32 %2, %1
+  ret i32 %add
+}
+
+; RELOC:      File: {{.*}}aix-tls-le-xcoff-reloc.ll.tmp.o
+; RELOC-NEXT: Format: aix5coff64-rs6000
+; RELOC-NEXT: Arch: powerpc64
+; RELOC-NEXT: AddressSize: 64bit
+; RELOC-NEXT: Relocations [
+; RELOC:       Virtual Address: 0x2
+; RELOC-NEXT:       Symbol: IThreadLocalVarUninit (17)
+; RELOC-NEXT:       IsSigned: No
+; RELOC-NEXT:       FixupBitValue: 0
+; RELOC-NEXT:       Length: 16
+; RELOC-NEXT:       Type: R_TOC (0x3)
+; RELOC-NEXT:     }
+; RELOC:       Virtual Address: 0x12
+; RELOC-NEXT:       Symbol: ThreadLocalVarInit (19)
+; RELOC-NEXT:       IsSigned: No
+; RELOC-NEXT:       FixupBitValue: 0
+; RELOC-NEXT:       Length: 16
+; RELOC-NEXT:       Type: R_TOC (0x3)
+; RELOC-NEXT:     }
+; RELOC:       Virtual Address: 0x68
+; RELOC-NEXT:       Symbol: IThreadLocalVarUninit (27)
+; RELOC-NEXT:       IsSigned: No
+; RELOC-NEXT:       FixupBitValue: 0
+; RELOC-NEXT:       Length: 64
+; RELOC-NEXT:       Type: R_TLS_LE (0x23)
+; RELOC-NEXT:     }
+; RELOC:       Virtual Address: 0x70
+; RELOC-NEXT:       Symbol: ThreadLocalVarInit (25)
+; RELOC-NEXT:       IsSigned: No
+; RELOC-NEXT:       FixupBitValue: 0
+; RELOC-NEXT:       Length: 64
+; RELOC-NEXT:       Type: R_TLS_LE (0x23)
+; RELOC-NEXT:     }
+
+; SYM:      File: {{.*}}aix-tls-le-xcoff-reloc.ll.tmp.o
+; SYM-NEXT: Format: aix5coff64-rs6000
+; SYM-NEXT: Arch: powerpc64
+; SYM-NEXT: AddressSize: 64bit
+; SYM-NEXT: Symbols [
+; SYM:     Index: 17
+; SYM-NEXT:     Name: IThreadLocalVarUninit
+; SYM-NEXT:     Value (RelocatableAddress): 0x68
+; SYM-NEXT:     Section: .data
+; SYM-NEXT:     Type: 0x0
+; SYM-NEXT:     StorageClass: C_HIDEXT (0x6B)
+; SYM-NEXT:     NumberOfAuxEntries: 1
+; SYM-NEXT:     CSECT Auxiliary Entry {
+; SYM-NEXT:       Index: 18
+; SYM-NEXT:       SectionLen: 8
+; SYM-NEXT:       ParameterHashIndex: 0x0
+; SYM-NEXT:       TypeChkSectNum: 0x0
+; SYM-NEXT:       SymbolAlignmentLog2: 3
+; SYM-NEXT:       SymbolType: XTY_SD (0x1)
+; SYM-NEXT:       StorageMappingClass: XMC_TC (0x3)
+; SYM-NEXT:       Auxiliary Type: AUX_CSECT (0xFB)
+; SYM-NEXT:     }
+; SYM-NEXT:   }
+; SYM:     Index: 19
+; SYM-NEXT:     Name: ThreadLocalVarInit
+; SYM-NEXT:     Value (RelocatableAddress): 0x70
+; SYM-NEXT:     Section: .data
+; SYM-NEXT:     Type: 0x0
+; SYM-NEXT:     StorageClass: C_HIDEXT (0x6B)
+; SYM-NEXT:     NumberOfAuxEntries: 1
+; SYM-NEXT:     CSECT Auxiliary Entry {
+; SYM-NEXT:       Index: 20
+; SYM-NEXT:       SectionLen: 8
+; SYM-NEXT:       ParameterHashIndex: 0x0
+; SYM-NEXT:       TypeChkSectNum: 0x0
+; SYM-NEXT:       SymbolAlignmentLog2: 3
+; SYM-NEXT:       SymbolType: XTY_SD (0x1)
+; SYM-NEXT:       StorageMappingClass: XMC_TC (0x3)
+; SYM-NEXT:       Auxiliary Type: AUX_CSECT (0xFB)
+; SYM-NEXT:     }
+; SYM-NEXT:   }
+; SYM:     Index: 25
+; SYM-NEXT:     Name: ThreadLocalVarInit
+; SYM-NEXT:     Value (RelocatableAddress): 0x0
+; SYM-NEXT:     Section: .tdata
+; SYM-NEXT:     Type: 0x0
+; SYM-NEXT:     StorageClass: C_EXT (0x2)
+; SYM-NEXT:     NumberOfAuxEntries: 1
+; SYM-NEXT:     CSECT Auxiliary Entry {
+; SYM-NEXT:       Index: 26
+; SYM-NEXT:       ContainingCsectSymbolIndex: 23
+; SYM-NEXT:       ParameterHashIndex: 0x0
+; SYM-NEXT:       TypeChkSectNum: 0x0
+; SYM-NEXT:       SymbolAlignmentLog2: 0
+; SYM-NEXT:       SymbolType: XTY_LD (0x2)
+; SYM-NEXT:       StorageMappingClass: XMC_TL (0x14)
+; SYM-NEXT:       Auxiliary Type: AUX_CSECT (0xFB)
+; SYM-NEXT:     }
+; SYM-NEXT:   }
+; SYM:     Index: 27
+; SYM-NEXT:     Name: IThreadLocalVarUninit
+; SYM-NEXT:     Value (RelocatableAddress): 0x4
+; SYM-NEXT:     Section: .tbss
+; SYM-NEXT:     Type: 0x0
+; SYM-NEXT:     StorageClass: C_HIDEXT (0x6B)
+; SYM-NEXT:     NumberOfAuxEntries: 1
+; SYM-NEXT:     CSECT Auxiliary Entry {
+; SYM-NEXT:       Index: 28
+; SYM-NEXT:       SectionLen: 4
+; SYM-NEXT:       ParameterHashIndex: 0x0
+; SYM-NEXT:       TypeChkSectNum: 0x0
+; SYM-NEXT:       SymbolAlignmentLog2: 2
+; SYM-NEXT:       SymbolType: XTY_CM (0x3)
+; SYM-NEXT:       StorageMappingClass: XMC_UL (0x15)
+; SYM-NEXT:       Auxiliary Type: AUX_CSECT (0xFB)
+; SYM-NEXT:     }
+; SYM-NEXT:   }
+
+; DIS:      {{.*}}aix-tls-le-xcoff-reloc.ll.tmp.o:	file format aix5coff64-rs6000
+; DIS:      Disassembly of section .text:
+; DIS:      0000000000000000 (idx: 3) .storeITLUninit:
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}               ld 4, 0(2)
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOC        (idx: 17) IThreadLocalVarUninit[TC]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}               add 4, 13, 4
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}               stw 3, 0(4)
+; DIS-NEXT:                                      blr
+; DIS:      0000000000000010 (idx: 5) .loadTLInit:
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}               ld 3, 8(2)
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOC        (idx: 19) ThreadLocalVarInit[TC]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}               ld 4, 16(2)
+; DIS-NEXT: {{0*}}[[#ADDR + 2]]: R_TOC        (idx: 21) VarInit[TC]
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}               add 3, 13, 3
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}               lwz 4, 0(4)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}               lwz 3, 0(3)
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}               add 3, 4, 3
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}               extsw 3, 3
+; DIS-NEXT: [[#%x, ADDR:]]: {{.*}}               blr
+
+; DIS:      Disassembly of section .data:
+; DIS:      0000000000000030 (idx: 9) VarInit:
+; DIS-NEXT:       30: 00 00 00 57
+; DIS:      0000000000000038 (idx: 11) storeITLUninit[DS]:
+; DIS-NEXT:       8: 00 00 00 00
+; DIS-NEXT: 0000000000000038:  R_POS        (idx: 3) .storeITLUninit
+; DIS-NEXT:       3c: 00 00 00 00
+; DIS-NEXT:       40: 00 00 00 00
+; DIS-NEXT: 0000000000000040:  R_POS        (idx: 15) TOC[TC0]
+; DIS-NEXT:       44: 00 00 00 68
+; DIS:      0000000000000050 (idx: 13) loadTLInit[DS]:
+; DIS-NEXT:       50: 00 00 00 00
+; DIS-NEXT: 0000000000000050:  R_POS        (idx: 5) .loadTLInit
+; DIS-NEXT:       54: 00 00 00 10
+; DIS-NEXT:       58: 00 00 00 00
+; DIS-NEXT: 0000000000000058:  R_POS        (idx: 15) TOC[TC0]
+; DIS-NEXT:       5c: 00 00 00 68
+; DIS:      0000000000000068 (idx: 17) IThreadLocalVarUninit[TC]:
+; DIS-NEXT:       68: 00 00 00 00
+; DIS-NEXT: 0000000000000068:  R_TLS_LE     (idx: 27) IThreadLocalVarUninit[UL]
+; DIS:      0000000000000070 (idx: 19) ThreadLocalVarInit[TC]:
+; DIS-NEXT:       70: 00 00 00 00
+; DIS-NEXT: 0000000000000070:  R_TLS_LE     (idx: 25) ThreadLocalVarInit
+; DIS:      0000000000000078 (idx: 21) VarInit[TC]:
+; DIS-NEXT:       78: 00 00 00 00
+; DIS-NEXT: 0000000000000078:  R_POS        (idx: 9) VarInit
+
+; DIS:      Disassembly of section .tdata:
+; DIS:      0000000000000000 (idx: 25) ThreadLocalVarInit:
+; DIS-NEXT:        0: 00 00 00 01
+
+; DIS:      Disassembly of section .tbss:
+; DIS:      0000000000000004 (idx: 27) IThreadLocalVarUninit[UL]:
+; DIS-NEXT: ...
+

From 13906755427611fe2d1b1bf915e87b97ddffd236 Mon Sep 17 00:00:00 2001
From: Ellis Hoag <ellis.sparky.hoag@gmail.com>
Date: Mon, 19 Jun 2023 10:39:13 -0700
Subject: [PATCH 066/130] [SpecialCaseList] Remove TrigramIndex
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`TrigramIndex` was added back in https://reviews.llvm.org/D27188 as an optimization to make `SpecialCaseList::match()` faster. I've found that `TrigramIndex` actually makes the function slower and it has no functional use, so we can remove it.

I grabbed the list of queries passed to `SpecialCaseList::match()` on a random very large file (`AArch64ISelLowering.cpp`) and measured the runtime to call `match()` on all of them with [this line](https://github.com/llvm/llvm-project/blob/8e1f820bb4eadf5c0704818f6063e0db1006e32d/llvm/lib/Support/SpecialCaseList.cpp#L64) disabled and then enabled.

```
$ hyperfine --warmup 3 'GTEST_FILTER="SpecialCaseListTest.Large" USE_TRIGRAMS=1 build/unittests/Support/SupportTests' 'GTEST_FILTER="SpecialCaseListTest.Large" USE_TRIGRAMS=0 build/unittests/Support/SupportTests'
Benchmark 1: GTEST_FILTER="SpecialCaseListTest.Large" USE_TRIGRAMS=1 build/unittests/Support/SupportTests
  Time (mean ± σ):     575.9 ms ±  20.3 ms    [User: 573.1 ms, System: 2.7 ms]
  Range (min … max):   555.5 ms … 620.0 ms    10 runs

Benchmark 2: GTEST_FILTER="SpecialCaseListTest.Large" USE_TRIGRAMS=0 build/unittests/Support/SupportTests
  Time (mean ± σ):     283.4 ms ±   6.7 ms    [User: 280.3 ms, System: 3.0 ms]
  Range (min … max):   277.0 ms … 294.9 ms    10 runs

Summary
  'GTEST_FILTER="SpecialCaseListTest.Large" USE_TRIGRAMS=0 build/unittests/Support/SupportTests' ran
    2.03 ± 0.09 times faster than 'GTEST_FILTER="SpecialCaseListTest.Large" USE_TRIGRAMS=1 build/unittests/Support/SupportTests'
```

Using `perf` I found that most of the runtime in `TrigramIndex::isDefinitelyOut()` comes from a division operation that seems to come from `std::unordered_map`: https://github.com/llvm/llvm-project/blob/8e1f820bb4eadf5c0704818f6063e0db1006e32d/llvm/include/llvm/Support/TrigramIndex.h#L62

Removing `TrigramIndex` will make it easier to potentially switch to using `GlobPattern` instead of a full regex for `SpecialCaseList`. See discussion in https://reviews.llvm.org/D152762 for details.

Reviewed By: MaskRay, #sanitizers, vitalybuka

Differential Revision: https://reviews.llvm.org/D153171
---
 llvm/include/llvm/Support/SpecialCaseList.h   |   5 +-
 llvm/include/llvm/Support/TrigramIndex.h      |  67 ---------
 llvm/lib/Support/CMakeLists.txt               |   1 -
 llvm/lib/Support/SpecialCaseList.cpp          |   3 -
 llvm/lib/Support/TrigramIndex.cpp             | 107 --------------
 llvm/unittests/Support/CMakeLists.txt         |   1 -
 llvm/unittests/Support/TrigramIndexTest.cpp   | 131 ------------------
 .../gn/secondary/llvm/lib/Support/BUILD.gn    |   1 -
 .../secondary/llvm/unittests/Support/BUILD.gn |   1 -
 9 files changed, 1 insertion(+), 316 deletions(-)
 delete mode 100644 llvm/include/llvm/Support/TrigramIndex.h
 delete mode 100644 llvm/lib/Support/TrigramIndex.cpp
 delete mode 100644 llvm/unittests/Support/TrigramIndexTest.cpp

diff --git a/llvm/include/llvm/Support/SpecialCaseList.h b/llvm/include/llvm/Support/SpecialCaseList.h
index 0d56c4b9912db..b6d1b56a09623 100644
--- a/llvm/include/llvm/Support/SpecialCaseList.h
+++ b/llvm/include/llvm/Support/SpecialCaseList.h
@@ -54,7 +54,6 @@
 
 #include "llvm/ADT/StringMap.h"
 #include "llvm/Support/Regex.h"
-#include "llvm/Support/TrigramIndex.h"
 #include <memory>
 #include <string>
 #include <vector>
@@ -128,7 +127,6 @@ class SpecialCaseList {
 
   private:
     StringMap<unsigned> Strings;
-    TrigramIndex Trigrams;
     std::vector<std::pair<std::unique_ptr<Regex>, unsigned>> RegExes;
   };
 
@@ -155,5 +153,4 @@ class SpecialCaseList {
 
 }  // namespace llvm
 
-#endif  // LLVM_SUPPORT_SPECIALCASELIST_H
-
+#endif // LLVM_SUPPORT_SPECIALCASELIST_H
diff --git a/llvm/include/llvm/Support/TrigramIndex.h b/llvm/include/llvm/Support/TrigramIndex.h
deleted file mode 100644
index 0bfac498393f7..0000000000000
--- a/llvm/include/llvm/Support/TrigramIndex.h
+++ /dev/null
@@ -1,67 +0,0 @@
-//===-- TrigramIndex.h - a heuristic for SpecialCaseList --------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//===----------------------------------------------------------------------===//
-//
-// TrigramIndex implements a heuristic for SpecialCaseList that allows to
-// filter out ~99% incoming queries when all regular expressions in the
-// SpecialCaseList are simple wildcards with '*' and '.'. If rules are more
-// complicated, the check is defeated and it will always pass the queries to a
-// full regex.
-//
-// The basic idea is that in order for a wildcard to match a query, the query
-// needs to have all trigrams which occur in the wildcard. We create a trigram
-// index (trigram -> list of rules with it) and then count trigrams in the query
-// for each rule. If the count for one of the rules reaches the expected value,
-// the check passes the query to a regex. If none of the rules got enough
-// trigrams, the check tells that the query is definitely not matched by any
-// of the rules, and no regex matching is needed.
-// A similar idea was used in Google Code Search as described in the blog post:
-// https://swtch.com/~rsc/regexp/regexp4.html
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_SUPPORT_TRIGRAMINDEX_H
-#define LLVM_SUPPORT_TRIGRAMINDEX_H
-
-#include "llvm/ADT/SmallVector.h"
-#include <string>
-#include <unordered_map>
-#include <vector>
-
-namespace llvm {
-class StringRef;
-
-class TrigramIndex {
- public:
-  /// Inserts a new Regex into the index.
-  void insert(const std::string &Regex);
-
-  /// Returns true, if special case list definitely does not have a line
-  /// that matches the query. Returns false, if it's not sure.
-  bool isDefinitelyOut(StringRef Query) const;
-
-  /// Returned true, iff the heuristic is defeated and not useful.
-  /// In this case isDefinitelyOut always returns false.
-  bool isDefeated() { return Defeated; }
- private:
-  // If true, the rules are too complicated for the check to work, and full
-  // regex matching is needed for every rule.
-  bool Defeated = false;
-  // The minimum number of trigrams which should match for a rule to have a
-  // chance to match the query. The number of elements equals the number of
-  // regex rules in the SpecialCaseList.
-  std::vector<unsigned> Counts;
-  // Index holds a list of rules indices for each trigram. The same indices
-  // are used in Counts to store per-rule limits.
-  // If a trigram is too common (>4 rules with it), we stop tracking it,
-  // which increases the probability for a need to match using regex, but
-  // decreases the costs in the regular case.
-  std::unordered_map<unsigned, SmallVector<size_t, 4>> Index{256};
-};
-
-}  // namespace llvm
-
-#endif  // LLVM_SUPPORT_TRIGRAMINDEX_H
diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt
index e6bff119b094f..976ae88d37c49 100644
--- a/llvm/lib/Support/CMakeLists.txt
+++ b/llvm/lib/Support/CMakeLists.txt
@@ -226,7 +226,6 @@ add_llvm_component_library(LLVMSupport
   TimeProfiler.cpp
   Timer.cpp
   ToolOutputFile.cpp
-  TrigramIndex.cpp
   Twine.cpp
   TypeSize.cpp
   Unicode.cpp
diff --git a/llvm/lib/Support/SpecialCaseList.cpp b/llvm/lib/Support/SpecialCaseList.cpp
index 46503477cdda4..64f66e0f81792 100644
--- a/llvm/lib/Support/SpecialCaseList.cpp
+++ b/llvm/lib/Support/SpecialCaseList.cpp
@@ -37,7 +37,6 @@ bool SpecialCaseList::Matcher::insert(std::string Regexp,
     Strings[Regexp] = LineNumber;
     return true;
   }
-  Trigrams.insert(Regexp);
 
   // Replace * with .*
   for (size_t pos = 0; (pos = Regexp.find('*', pos)) != std::string::npos;
@@ -61,8 +60,6 @@ unsigned SpecialCaseList::Matcher::match(StringRef Query) const {
   auto It = Strings.find(Query);
   if (It != Strings.end())
     return It->second;
-  if (Trigrams.isDefinitelyOut(Query))
-    return false;
   for (const auto &RegExKV : RegExes)
     if (RegExKV.first->match(Query))
       return RegExKV.second;
diff --git a/llvm/lib/Support/TrigramIndex.cpp b/llvm/lib/Support/TrigramIndex.cpp
deleted file mode 100644
index 40a20ccc6583f..0000000000000
--- a/llvm/lib/Support/TrigramIndex.cpp
+++ /dev/null
@@ -1,107 +0,0 @@
-//===-- TrigramIndex.cpp - a heuristic for SpecialCaseList ----------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// TrigramIndex implements a heuristic for SpecialCaseList that allows to
-// filter out ~99% incoming queries when all regular expressions in the
-// SpecialCaseList are simple wildcards with '*' and '.'. If rules are more
-// complicated, the check is defeated and it will always pass the queries to a
-// full regex.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Support/TrigramIndex.h"
-#include "llvm/ADT/StringRef.h"
-#include <set>
-
-using namespace llvm;
-
-static const char RegexAdvancedMetachars[] = "()^$|+?[]\\{}";
-
-static bool isAdvancedMetachar(unsigned Char) {
-  return strchr(RegexAdvancedMetachars, Char) != nullptr;
-}
-
-void TrigramIndex::insert(const std::string &Regex) {
-  if (Defeated) return;
-  std::set<unsigned> Was;
-  unsigned Cnt = 0;
-  unsigned Tri = 0;
-  unsigned Len = 0;
-  bool Escaped = false;
-  for (unsigned Char : Regex) {
-    if (!Escaped) {
-      // Regular expressions allow escaping symbols by preceding it with '\'.
-      if (Char == '\\') {
-        Escaped = true;
-        continue;
-      }
-      if (isAdvancedMetachar(Char)) {
-        // This is a more complicated regex than we can handle here.
-        Defeated = true;
-        return;
-      }
-      if (Char == '.' || Char == '*') {
-        Tri = 0;
-        Len = 0;
-        continue;
-      }
-    }
-    if (Escaped && Char >= '1' && Char <= '9') {
-      Defeated = true;
-      return;
-    }
-    // We have already handled escaping and can reset the flag.
-    Escaped = false;
-    Tri = ((Tri << 8) + Char) & 0xFFFFFF;
-    Len++;
-    if (Len < 3)
-      continue;
-    // We don't want the index to grow too much for the popular trigrams,
-    // as they are weak signals. It's ok to still require them for the
-    // rules we have already processed. It's just a small additional
-    // computational cost.
-    if (Index[Tri].size() >= 4)
-      continue;
-    Cnt++;
-    if (!Was.count(Tri)) {
-      // Adding the current rule to the index.
-      Index[Tri].push_back(Counts.size());
-      Was.insert(Tri);
-    }
-  }
-  if (!Cnt) {
-    // This rule does not have remarkable trigrams to rely on.
-    // We have to always call the full regex chain.
-    Defeated = true;
-    return;
-  }
-  Counts.push_back(Cnt);
-}
-
-bool TrigramIndex::isDefinitelyOut(StringRef Query) const {
-  if (Defeated)
-    return false;
-  std::vector<unsigned> CurCounts(Counts.size());
-  unsigned Tri = 0;
-  for (size_t I = 0; I < Query.size(); I++) {
-    Tri = ((Tri << 8) + Query[I]) & 0xFFFFFF;
-    if (I < 2)
-      continue;
-    const auto &II = Index.find(Tri);
-    if (II == Index.end())
-      continue;
-    for (size_t J : II->second) {
-      CurCounts[J]++;
-      // If we have reached a desired limit, we have to look at the query
-      // more closely by running a full regex.
-      if (CurCounts[J] >= Counts[J])
-        return false;
-    }
-  }
-  return true;
-}
diff --git a/llvm/unittests/Support/CMakeLists.txt b/llvm/unittests/Support/CMakeLists.txt
index 9c2c62b0b389c..e828640451cae 100644
--- a/llvm/unittests/Support/CMakeLists.txt
+++ b/llvm/unittests/Support/CMakeLists.txt
@@ -89,7 +89,6 @@ add_llvm_unittest(SupportTests
   TypeSizeTest.cpp
   TypeTraitsTest.cpp
   TrailingObjectsTest.cpp
-  TrigramIndexTest.cpp
   UnicodeTest.cpp
   VersionTupleTest.cpp
   VirtualFileSystemTest.cpp
diff --git a/llvm/unittests/Support/TrigramIndexTest.cpp b/llvm/unittests/Support/TrigramIndexTest.cpp
deleted file mode 100644
index 42b3fcdbd535f..0000000000000
--- a/llvm/unittests/Support/TrigramIndexTest.cpp
+++ /dev/null
@@ -1,131 +0,0 @@
-//===- TrigramIndexTest.cpp - Unit tests for TrigramIndex -----------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Support/TrigramIndex.h"
-#include "llvm/ADT/STLExtras.h"
-#include "gtest/gtest.h"
-
-#include <string>
-#include <vector>
-
-using namespace llvm;
-
-namespace {
-
-class TrigramIndexTest : public ::testing::Test {
-protected:
-  std::unique_ptr<TrigramIndex> makeTrigramIndex(
-      std::vector<std::string> Rules) {
-    std::unique_ptr<TrigramIndex> TI =
-        std::make_unique<TrigramIndex>();
-    for (auto &Rule : Rules)
-      TI->insert(Rule);
-    return TI;
-  }
-};
-
-TEST_F(TrigramIndexTest, Empty) {
-  std::unique_ptr<TrigramIndex> TI =
-      makeTrigramIndex({});
-  EXPECT_FALSE(TI->isDefeated());
-  EXPECT_TRUE(TI->isDefinitelyOut("foo"));
-}
-
-TEST_F(TrigramIndexTest, Basic) {
-  std::unique_ptr<TrigramIndex> TI =
-      makeTrigramIndex({"*hello*", "*wor.d*"});
-  EXPECT_FALSE(TI->isDefeated());
-  EXPECT_TRUE(TI->isDefinitelyOut("foo"));
-}
-
-TEST_F(TrigramIndexTest, NoTrigramsInRules) {
-  std::unique_ptr<TrigramIndex> TI =
-      makeTrigramIndex({"b.r", "za*az"});
-  EXPECT_TRUE(TI->isDefeated());
-  EXPECT_FALSE(TI->isDefinitelyOut("foo"));
-  EXPECT_FALSE(TI->isDefinitelyOut("bar"));
-  EXPECT_FALSE(TI->isDefinitelyOut("zakaz"));
-}
-
-TEST_F(TrigramIndexTest, NoTrigramsInARule) {
-  std::unique_ptr<TrigramIndex> TI =
-      makeTrigramIndex({"*hello*", "*wo.ld*"});
-  EXPECT_TRUE(TI->isDefeated());
-  EXPECT_FALSE(TI->isDefinitelyOut("foo"));
-}
-
-TEST_F(TrigramIndexTest, RepetitiveRule) {
-  std::unique_ptr<TrigramIndex> TI =
-      makeTrigramIndex({"*bar*bar*bar*bar*bar", "bar*bar"});
-  EXPECT_FALSE(TI->isDefeated());
-  EXPECT_TRUE(TI->isDefinitelyOut("foo"));
-  EXPECT_TRUE(TI->isDefinitelyOut("bar"));
-  EXPECT_FALSE(TI->isDefinitelyOut("barbara"));
-  EXPECT_FALSE(TI->isDefinitelyOut("bar+bar"));
-}
-
-TEST_F(TrigramIndexTest, PopularTrigram) {
-  std::unique_ptr<TrigramIndex> TI =
-      makeTrigramIndex({"*aaa*", "*aaaa*", "*aaaaa*", "*aaaaa*", "*aaaaaa*"});
-  EXPECT_TRUE(TI->isDefeated());
-}
-
-TEST_F(TrigramIndexTest, PopularTrigram2) {
-  std::unique_ptr<TrigramIndex> TI =
-      makeTrigramIndex({"class1.h", "class2.h", "class3.h", "class4.h", "class.h"});
-  EXPECT_TRUE(TI->isDefeated());
-}
-
-TEST_F(TrigramIndexTest, TooComplicatedRegex) {
-  std::unique_ptr<TrigramIndex> TI =
-      makeTrigramIndex({"[0-9]+"});
-  EXPECT_TRUE(TI->isDefeated());
-}
-
-TEST_F(TrigramIndexTest, TooComplicatedRegex2) {
-  std::unique_ptr<TrigramIndex> TI =
-      makeTrigramIndex({"foo|bar"});
-  EXPECT_TRUE(TI->isDefeated());
-}
-
-TEST_F(TrigramIndexTest, EscapedSymbols) {
-  std::unique_ptr<TrigramIndex> TI =
-      makeTrigramIndex({"*c\\+\\+*", "*hello\\\\world*", "a\\tb", "a\\0b"});
-  EXPECT_FALSE(TI->isDefeated());
-  EXPECT_FALSE(TI->isDefinitelyOut("c++"));
-  EXPECT_TRUE(TI->isDefinitelyOut("c\\+\\+"));
-  EXPECT_FALSE(TI->isDefinitelyOut("hello\\world"));
-  EXPECT_TRUE(TI->isDefinitelyOut("hello\\\\world"));
-  EXPECT_FALSE(TI->isDefinitelyOut("atb"));
-  EXPECT_TRUE(TI->isDefinitelyOut("a\\tb"));
-  EXPECT_TRUE(TI->isDefinitelyOut("a\tb"));
-  EXPECT_FALSE(TI->isDefinitelyOut("a0b"));
-}
-
-TEST_F(TrigramIndexTest, Backreference1) {
-  std::unique_ptr<TrigramIndex> TI =
-      makeTrigramIndex({"*foo\\1*"});
-  EXPECT_TRUE(TI->isDefeated());
-}
-
-TEST_F(TrigramIndexTest, Backreference2) {
-  std::unique_ptr<TrigramIndex> TI =
-      makeTrigramIndex({"*foo\\2*"});
-  EXPECT_TRUE(TI->isDefeated());
-}
-
-TEST_F(TrigramIndexTest, Sequence) {
-  std::unique_ptr<TrigramIndex> TI =
-      makeTrigramIndex({"class1.h", "class2.h", "class3.h", "class4.h"});
-  EXPECT_FALSE(TI->isDefeated());
-  EXPECT_FALSE(TI->isDefinitelyOut("class1"));
-  EXPECT_TRUE(TI->isDefinitelyOut("class.h"));
-  EXPECT_TRUE(TI->isDefinitelyOut("class"));
-}
-
-}  // namespace
diff --git a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
index 8c1a29a3c4cf4..989e805f51178 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
@@ -141,7 +141,6 @@ static_library("Support") {
     "TimeProfiler.cpp",
     "Timer.cpp",
     "ToolOutputFile.cpp",
-    "TrigramIndex.cpp",
     "Twine.cpp",
     "TypeSize.cpp",
     "Unicode.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
index 7f17a69db3d3d..5143c361c0d1e 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
@@ -89,7 +89,6 @@ unittest("SupportTests") {
     "TimerTest.cpp",
     "ToolOutputFileTest.cpp",
     "TrailingObjectsTest.cpp",
-    "TrigramIndexTest.cpp",
     "TypeNameTest.cpp",
     "TypeSizeTest.cpp",
     "TypeTraitsTest.cpp",

From 3350ec9b3e9323b31c475b206aec7318d2cb709f Mon Sep 17 00:00:00 2001
From: Louis Dionne <ldionne.2@gmail.com>
Date: Mon, 19 Jun 2023 13:43:17 -0400
Subject: [PATCH 067/130] [libc++] Add missing 'return 0' from main functions
 in tests

---
 .../algorithms/alg.sorting/pstl.is_partitioned.pass.cpp       | 2 ++
 .../robust_against_using_non_transparent_comparators.pass.cpp | 4 +++-
 .../std/thread/thread.stoptoken/stopcallback/dtor.pass.cpp    | 2 ++
 libcxx/test/std/utilities/expol/policies.compile.pass.cpp     | 2 ++
 4 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/libcxx/test/libcxx/algorithms/alg.sorting/pstl.is_partitioned.pass.cpp b/libcxx/test/libcxx/algorithms/alg.sorting/pstl.is_partitioned.pass.cpp
index 7d07ab77b9579..86647ccfdef8e 100644
--- a/libcxx/test/libcxx/algorithms/alg.sorting/pstl.is_partitioned.pass.cpp
+++ b/libcxx/test/libcxx/algorithms/alg.sorting/pstl.is_partitioned.pass.cpp
@@ -26,4 +26,6 @@ int main(int, char**) {
     return i < 5;
   }));
   assert(call_count == std::size(a));
+
+  return 0;
 }
diff --git a/libcxx/test/libcxx/algorithms/robust_against_using_non_transparent_comparators.pass.cpp b/libcxx/test/libcxx/algorithms/robust_against_using_non_transparent_comparators.pass.cpp
index 66b620e9cfc46..eaa5d44385be6 100644
--- a/libcxx/test/libcxx/algorithms/robust_against_using_non_transparent_comparators.pass.cpp
+++ b/libcxx/test/libcxx/algorithms/robust_against_using_non_transparent_comparators.pass.cpp
@@ -58,7 +58,7 @@ struct Iterator {
   T* ptr_;
 };
 
-int main() {
+int main(int, char**) {
   int array[5] = {1, 2, 3, 4, 5};
   Iterator<int> first(array);
   Iterator<int> middle(array + 3);
@@ -74,4 +74,6 @@ int main() {
   (void)std::min_element(first, last);
   (void)std::minmax_element(first, last);
   (void)std::upper_bound(first, last, 3);
+
+  return 0;
 }
diff --git a/libcxx/test/std/thread/thread.stoptoken/stopcallback/dtor.pass.cpp b/libcxx/test/std/thread/thread.stoptoken/stopcallback/dtor.pass.cpp
index 62ef6ff2a4050..debc29a55ee98 100644
--- a/libcxx/test/std/thread/thread.stoptoken/stopcallback/dtor.pass.cpp
+++ b/libcxx/test/std/thread/thread.stoptoken/stopcallback/dtor.pass.cpp
@@ -163,4 +163,6 @@ int main(int, char**) {
     ss.request_stop(); // the callbacks deletes itself. if the destructor blocks, it would be deadlock
     assert(holder.callback_ == nullptr);
   }
+
+  return 0;
 }
diff --git a/libcxx/test/std/utilities/expol/policies.compile.pass.cpp b/libcxx/test/std/utilities/expol/policies.compile.pass.cpp
index ebea9d4f5c424..a4cecc4d1dcda 100644
--- a/libcxx/test/std/utilities/expol/policies.compile.pass.cpp
+++ b/libcxx/test/std/utilities/expol/policies.compile.pass.cpp
@@ -47,4 +47,6 @@ int main(int, char**) {
 #if TEST_STD_VER >= 20
   use(std::execution::unseq);
 #endif
+
+  return 0;
 }

From 2da45172c4bcd42f704c57c656926f56f32fc5ce Mon Sep 17 00:00:00 2001
From: Vladislav Dzhidzhoev <vdzhidzhoev@accesssoftek.com>
Date: Mon, 19 Jun 2023 16:42:05 +0200
Subject: [PATCH 068/130] Reland "[DebugMetadata][DwarfDebug] Support
 function-local types in lexical block scopes (4/7)"

Test "local-type-as-template-parameter.ll" now requires linux-system.

Authored-by: Kristina Bessonova <kbessonova@accesssoftek.com>

Differential Revision: https://reviews.llvm.org/D144006

Depends on D144005
---
 .../CodeGen/debug-info-codeview-unnamed.c     |  16 +-
 clang/test/CodeGen/debug-info-unused-types.c  |  14 +-
 .../test/CodeGen/debug-info-unused-types.cpp  |  12 +-
 clang/test/CodeGenCXX/debug-info-access.cpp   |   2 +-
 .../CodeGenCXX/debug-info-anon-union-vars.cpp |  12 +-
 .../debug-info-codeview-unnamed.cpp           | 110 +++--
 .../debug-info-gline-tables-only-codeview.cpp |   4 +-
 clang/test/CodeGenCXX/debug-lambda-this.cpp   |   4 +-
 llvm/include/llvm/IR/DIBuilder.h              |   6 +-
 llvm/lib/Bitcode/Reader/MetadataLoader.cpp    |  87 ++--
 .../CodeGen/AsmPrinter/DwarfCompileUnit.cpp   |  60 ++-
 .../lib/CodeGen/AsmPrinter/DwarfCompileUnit.h |  16 +-
 llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp    |  13 +-
 llvm/lib/IR/DIBuilder.cpp                     |  34 +-
 llvm/lib/IR/Verifier.cpp                      |   6 +-
 llvm/test/Bitcode/upgrade-cu-locals.ll        |  68 +--
 llvm/test/Bitcode/upgrade-cu-locals.ll.bc     | Bin 2688 -> 2780 bytes
 .../DebugInfo/Generic/inlined-local-type.ll   | 128 ++++++
 .../Generic/lexical-block-retained-types.ll   |  55 +++
 .../DebugInfo/Generic/lexical-block-types.ll  | 425 ++++++++++++++++++
 .../local-type-as-template-parameter.ll       | 161 +++++++
 .../Generic/verifier-invalid-disubprogram.ll  |   2 +-
 llvm/test/DebugInfo/X86/set.ll                |   4 +-
 .../split-dwarf-local-import.ll               |   1 -
 .../split-dwarf-local-import2.ll              |   1 -
 .../split-dwarf-local-import3.ll              |   0
 26 files changed, 1052 insertions(+), 189 deletions(-)
 create mode 100644 llvm/test/DebugInfo/Generic/inlined-local-type.ll
 create mode 100644 llvm/test/DebugInfo/Generic/lexical-block-retained-types.ll
 create mode 100644 llvm/test/DebugInfo/Generic/lexical-block-types.ll
 create mode 100644 llvm/test/DebugInfo/Generic/local-type-as-template-parameter.ll
 rename llvm/test/DebugInfo/{Generic => X86}/split-dwarf-local-import.ll (98%)
 rename llvm/test/DebugInfo/{Generic => X86}/split-dwarf-local-import2.ll (98%)
 rename llvm/test/DebugInfo/{Generic => X86}/split-dwarf-local-import3.ll (100%)

diff --git a/clang/test/CodeGen/debug-info-codeview-unnamed.c b/clang/test/CodeGen/debug-info-codeview-unnamed.c
index bd2a7543e56b2..16ffb3682236f 100644
--- a/clang/test/CodeGen/debug-info-codeview-unnamed.c
+++ b/clang/test/CodeGen/debug-info-codeview-unnamed.c
@@ -8,23 +8,23 @@ int main(int argc, char* argv[], char* arge[]) {
   //
   struct { int bar; } one = {42};
   //
-  // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "one"
-  // LINUX-SAME:     type: [[TYPE_OF_ONE:![0-9]+]]
-  // LINUX-SAME:     )
-  // LINUX:      [[TYPE_OF_ONE]] = distinct !DICompositeType(
+  // LINUX:      [[TYPE_OF_ONE:![0-9]+]] = distinct !DICompositeType(
   // LINUX-SAME:     tag: DW_TAG_structure_type
   // LINUX-NOT:      name:
   // LINUX-NOT:      identifier:
   // LINUX-SAME:     )
+  // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "one"
+  // LINUX-SAME:     type: [[TYPE_OF_ONE]]
+  // LINUX-SAME:     )
   //
-  // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "one"
-  // MSVC-SAME:      type: [[TYPE_OF_ONE:![0-9]+]]
-  // MSVC-SAME:      )
-  // MSVC:       [[TYPE_OF_ONE]] = distinct !DICompositeType
+  // MSVC:       [[TYPE_OF_ONE:![0-9]+]] = distinct !DICompositeType
   // MSVC-SAME:      tag: DW_TAG_structure_type
   // MSVC-NOT:       name:
   // MSVC-NOT:       identifier:
   // MSVC-SAME:      )
+  // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "one"
+  // MSVC-SAME:      type: [[TYPE_OF_ONE]]
+  // MSVC-SAME:      )
 
   return 0;
 }
diff --git a/clang/test/CodeGen/debug-info-unused-types.c b/clang/test/CodeGen/debug-info-unused-types.c
index 3e9f7b07658e3..5bcc71f710a5c 100644
--- a/clang/test/CodeGen/debug-info-unused-types.c
+++ b/clang/test/CodeGen/debug-info-unused-types.c
@@ -18,13 +18,13 @@ void quux(void) {
 // CHECK: !DICompileUnit{{.+}}retainedTypes: [[RETTYPES:![0-9]+]]
 // CHECK: [[TYPE0:![0-9]+]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "bar"
 // CHECK: [[TYPE1:![0-9]+]] = !DIEnumerator(name: "BAR"
-// CHECK: [[TYPE2:![0-9]+]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "z"
-// CHECK: [[TYPE3:![0-9]+]] = !DIEnumerator(name: "Z"
-// CHECK: [[RETTYPES]] = !{[[TYPE4:![0-9]+]], [[TYPE5:![0-9]+]], [[TYPE0]], [[TYPE6:![0-9]+]], {{![0-9]+}}, [[TYPE7:![0-9]+]], [[TYPE2]], [[TYPE8:![0-9]+]]}
-// CHECK: [[TYPE4]] = !DIDerivedType(tag: DW_TAG_typedef, name: "my_int"
-// CHECK: [[TYPE5]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "foo"
-// CHECK: [[TYPE6]] = distinct !DICompositeType(tag: DW_TAG_union_type, name: "baz"
-// CHECK: [[TYPE7]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "y"
+// CHECK: [[RETTYPES]] = !{[[TYPE2:![0-9]+]], [[TYPE3:![0-9]+]], [[TYPE0]], [[TYPE4:![0-9]+]], {{![0-9]+}}, [[TYPE5:![0-9]+]], [[TYPE6:![0-9]+]], [[TYPE8:![0-9]+]]}
+// CHECK: [[TYPE2]] = !DIDerivedType(tag: DW_TAG_typedef, name: "my_int"
+// CHECK: [[TYPE3]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "foo"
+// CHECK: [[TYPE4]] = distinct !DICompositeType(tag: DW_TAG_union_type, name: "baz"
+// CHECK: [[TYPE5]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "y"
+// CHECK: [[TYPE6]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "z"
+// CHECK: [[TYPE7:![0-9]+]] = !DIEnumerator(name: "Z"
 // CHECK: [[TYPE8]] = distinct !DICompositeType(tag: DW_TAG_union_type, name: "w"
 
 // Check that debug info is not emitted for the typedef, struct, enum, and
diff --git a/clang/test/CodeGen/debug-info-unused-types.cpp b/clang/test/CodeGen/debug-info-unused-types.cpp
index 023cac159faa4..715fada47eab8 100644
--- a/clang/test/CodeGen/debug-info-unused-types.cpp
+++ b/clang/test/CodeGen/debug-info-unused-types.cpp
@@ -13,12 +13,12 @@ void quux() {
 // CHECK: !DICompileUnit{{.+}}retainedTypes: [[RETTYPES:![0-9]+]]
 // CHECK: [[TYPE0:![0-9]+]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "baz"
 // CHECK: [[TYPE1:![0-9]+]] = !DIEnumerator(name: "BAZ"
-// CHECK: [[TYPE2:![0-9]+]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "z"
-// CHECK: [[TYPE3:![0-9]+]] = !DIEnumerator(name: "Z"
-// CHECK: [[RETTYPES]] = !{[[TYPE4:![0-9]+]], [[TYPE5:![0-9]+]], [[TYPE0]], {{![0-9]+}}, [[TYPE6:![0-9]+]], [[TYPE2]]}
-// CHECK: [[TYPE4]] = !DIDerivedType(tag: DW_TAG_typedef, name: "foo"
-// CHECK: [[TYPE5]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "bar"
-// CHECK: [[TYPE6]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "y"
+// CHECK: [[RETTYPES]] = !{[[TYPE2:![0-9]+]], [[TYPE3:![0-9]+]], [[TYPE0]], {{![0-9]+}}, [[TYPE4:![0-9]+]], [[TYPE5:![0-9]+]]}
+// CHECK: [[TYPE2]] = !DIDerivedType(tag: DW_TAG_typedef, name: "foo"
+// CHECK: [[TYPE3]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "bar"
+// CHECK: [[TYPE4]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "y"
+// CHECK: [[TYPE5]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "z"
+// CHECK: [[TYPE6:![0-9]+]] = !DIEnumerator(name: "Z"
 
 // NODBG-NOT: !DI{{CompositeType|Enumerator|DerivedType}}
 
diff --git a/clang/test/CodeGenCXX/debug-info-access.cpp b/clang/test/CodeGenCXX/debug-info-access.cpp
index 9f2c044843d0f..7c0bf8ccb0384 100644
--- a/clang/test/CodeGenCXX/debug-info-access.cpp
+++ b/clang/test/CodeGenCXX/debug-info-access.cpp
@@ -18,9 +18,9 @@ class B : public A {
   static int public_static;
 
 protected:
+  // CHECK-DAG: !DIDerivedType(tag: DW_TAG_typedef, name: "prot_using",{{.*}} line: [[@LINE+3]],{{.*}} flags: DIFlagProtected)
   // CHECK-DAG: !DIDerivedType(tag: DW_TAG_typedef, name: "prot_typedef",{{.*}} line: [[@LINE+1]],{{.*}} flags: DIFlagProtected)
   typedef int prot_typedef;
-  // CHECK-DAG: !DIDerivedType(tag: DW_TAG_typedef, name: "prot_using",{{.*}} line: [[@LINE+1]],{{.*}} flags: DIFlagProtected)
   using prot_using = prot_typedef;
   prot_using prot_member;
 
diff --git a/clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp b/clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp
index 61b3c7c0526c8..c91cf83c0405f 100644
--- a/clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp
+++ b/clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp
@@ -51,13 +51,13 @@ void instantiate(int x) {
 // CHECK: !DIGlobalVariable(name: "b",{{.*}} file: [[FILE]], line: 6,{{.*}} isLocal: true, isDefinition: true
 // CHECK: !DIGlobalVariable(name: "result", {{.*}} isLocal: false, isDefinition: true
 // CHECK: !DIGlobalVariable(name: "value", {{.*}} isLocal: false, isDefinition: true
-// CHECK: !DILocalVariable(name: "i", {{.*}}, flags: DIFlagArtificial
-// CHECK: !DILocalVariable(name: "c", {{.*}}, flags: DIFlagArtificial
-// CHECK: !DILocalVariable(
-// CHECK-NOT: name:
-// CHECK: type: ![[UNION:[0-9]+]]
-// CHECK: ![[UNION]] = distinct !DICompositeType(tag: DW_TAG_union_type,
+// CHECK: ![[UNION:[0-9]+]] = distinct !DICompositeType(tag: DW_TAG_union_type,
 // CHECK-NOT: name:
 // CHECK: elements
 // CHECK: !DIDerivedType(tag: DW_TAG_member, name: "i", scope: ![[UNION]],
 // CHECK: !DIDerivedType(tag: DW_TAG_member, name: "c", scope: ![[UNION]],
+// CHECK: !DILocalVariable(name: "i", {{.*}}, flags: DIFlagArtificial
+// CHECK: !DILocalVariable(name: "c", {{.*}}, flags: DIFlagArtificial
+// CHECK: !DILocalVariable(
+// CHECK-NOT: name:
+// CHECK: type: ![[UNION]]
diff --git a/clang/test/CodeGenCXX/debug-info-codeview-unnamed.cpp b/clang/test/CodeGenCXX/debug-info-codeview-unnamed.cpp
index b4c79936ab33e..9602ac1b02497 100644
--- a/clang/test/CodeGenCXX/debug-info-codeview-unnamed.cpp
+++ b/clang/test/CodeGenCXX/debug-info-codeview-unnamed.cpp
@@ -3,6 +3,60 @@
 
 int main(int argc, char* argv[], char* arge[]) {
   //
+  // LINUX:      [[TYPE_OF_ONE:![0-9]+]] = distinct !DICompositeType(
+  // LINUX-SAME:     tag: DW_TAG_structure_type
+  // LINUX-NOT:      name:
+  // LINUX-NOT:      identifier:
+  // LINUX-SAME:     )
+  //
+  // MSVC:       [[TYPE_OF_ONE:![0-9]+]] = distinct !DICompositeType
+  // MSVC-SAME:      tag: DW_TAG_structure_type
+  // MSVC-SAME:      name: "<unnamed-type-one>"
+  // MSVC-SAME:      identifier: ".?AU<unnamed-type-one>@?1??main@@9@"
+  // MSVC-SAME:      )
+
+
+  //
+  // LINUX:      [[TYPE_OF_TWO:![0-9]+]] = distinct !DICompositeType(
+  // LINUX-SAME:     tag: DW_TAG_structure_type
+  // LINUX-NOT:      name:
+  // LINUX-NOT:      identifier:
+  // LINUX-SAME:     )
+  //
+  // MSVC:       [[TYPE_OF_TWO:![0-9]+]] = distinct !DICompositeType
+  // MSVC-SAME:      tag: DW_TAG_structure_type
+  // MSVC-SAME:      name: "<unnamed-type-two>"
+  // MSVC-SAME:      identifier: ".?AU<unnamed-type-two>@?2??main@@9@"
+  // MSVC-SAME:      )
+
+
+  //
+  // LINUX:      [[TYPE_OF_THREE:![0-9]+]] = distinct !DICompositeType(
+  // LINUX-SAME:     tag: DW_TAG_structure_type
+  // LINUX-SAME:     name: "named"
+  // LINUX-NOT:      identifier:
+  // LINUX-SAME:     )
+  //
+  // MSVC:       [[TYPE_OF_THREE:![0-9]+]] = distinct !DICompositeType
+  // MSVC-SAME:      tag: DW_TAG_structure_type
+  // MSVC-SAME:      name: "named"
+  // MSVC-SAME:      identifier: ".?AUnamed@?1??main@@9@"
+  // MSVC-SAME:      )
+
+  //
+  // LINUX:      [[TYPE_OF_FOUR:![0-9]+]] = distinct !DICompositeType(
+  // LINUX-SAME:     tag: DW_TAG_class_type
+  // LINUX-NOT:      name:
+  // LINUX-NOT:      identifier:
+  // LINUX-SAME:     )
+  //
+  // MSVC:       [[TYPE_OF_FOUR:![0-9]+]] = distinct !DICompositeType
+  // MSVC-SAME:      tag: DW_TAG_class_type
+  // MSVC-SAME:      name: "<lambda_0>"
+  // MSVC-SAME:      identifier: ".?AV<lambda_0>@?0??main@@9@"
+  // MSVC-SAME:      )
+
+
   // In CodeView, the LF_MFUNCTION entry for "bar()" refers to the forward
   // reference of the unnamed struct. Visual Studio requires a unique
   // identifier to match the LF_STRUCTURE forward reference to the definition.
@@ -10,21 +64,11 @@ int main(int argc, char* argv[], char* arge[]) {
   struct { void bar() {} } one;
   //
   // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "one"
-  // LINUX-SAME:     type: [[TYPE_OF_ONE:![0-9]+]]
-  // LINUX-SAME:     )
-  // LINUX:      [[TYPE_OF_ONE]] = distinct !DICompositeType(
-  // LINUX-SAME:     tag: DW_TAG_structure_type
-  // LINUX-NOT:      name:
-  // LINUX-NOT:      identifier:
+  // LINUX-SAME:     type: [[TYPE_OF_ONE]]
   // LINUX-SAME:     )
   //
   // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "one"
-  // MSVC-SAME:      type: [[TYPE_OF_ONE:![0-9]+]]
-  // MSVC-SAME:      )
-  // MSVC:       [[TYPE_OF_ONE]] = distinct !DICompositeType
-  // MSVC-SAME:      tag: DW_TAG_structure_type
-  // MSVC-SAME:      name: "<unnamed-type-one>"
-  // MSVC-SAME:      identifier: ".?AU<unnamed-type-one>@?1??main@@9@"
+  // MSVC-SAME:      type: [[TYPE_OF_ONE]]
   // MSVC-SAME:      )
 
 
@@ -36,21 +80,11 @@ int main(int argc, char* argv[], char* arge[]) {
   int decltype(two)::*ptr2unnamed = &decltype(two)::bar;
   //
   // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "two"
-  // LINUX-SAME:     type: [[TYPE_OF_TWO:![0-9]+]]
-  // LINUX-SAME:     )
-  // LINUX:      [[TYPE_OF_TWO]] = distinct !DICompositeType(
-  // LINUX-SAME:     tag: DW_TAG_structure_type
-  // LINUX-NOT:      name:
-  // LINUX-NOT:      identifier:
+  // LINUX-SAME:     type: [[TYPE_OF_TWO]]
   // LINUX-SAME:     )
   //
   // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "two"
-  // MSVC-SAME:      type: [[TYPE_OF_TWO:![0-9]+]]
-  // MSVC-SAME:      )
-  // MSVC:       [[TYPE_OF_TWO]] = distinct !DICompositeType
-  // MSVC-SAME:      tag: DW_TAG_structure_type
-  // MSVC-SAME:      name: "<unnamed-type-two>"
-  // MSVC-SAME:      identifier: ".?AU<unnamed-type-two>@?2??main@@9@"
+  // MSVC-SAME:      type: [[TYPE_OF_TWO]]
   // MSVC-SAME:      )
 
 
@@ -61,21 +95,11 @@ int main(int argc, char* argv[], char* arge[]) {
   struct named { int bar; int named::* p2mem; } three = { 42, &named::bar };
   //
   // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "three"
-  // LINUX-SAME:     type: [[TYPE_OF_THREE:![0-9]+]]
-  // LINUX-SAME:     )
-  // LINUX:      [[TYPE_OF_THREE]] = distinct !DICompositeType(
-  // LINUX-SAME:     tag: DW_TAG_structure_type
-  // LINUX-SAME:     name: "named"
-  // LINUX-NOT:      identifier:
+  // LINUX-SAME:     type: [[TYPE_OF_THREE]]
   // LINUX-SAME:     )
   //
   // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "three"
-  // MSVC-SAME:      type: [[TYPE_OF_THREE:![0-9]+]]
-  // MSVC-SAME:      )
-  // MSVC:       [[TYPE_OF_THREE]] = distinct !DICompositeType
-  // MSVC-SAME:      tag: DW_TAG_structure_type
-  // MSVC-SAME:      name: "named"
-  // MSVC-SAME:      identifier: ".?AUnamed@?1??main@@9@"
+  // MSVC-SAME:      type: [[TYPE_OF_THREE]]
   // MSVC-SAME:      )
 
 
@@ -87,21 +111,11 @@ int main(int argc, char* argv[], char* arge[]) {
   auto four = [argc](int i) -> int { return argc == i ? 1 : 0; };
   //
   // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "four"
-  // LINUX-SAME:     type: [[TYPE_OF_FOUR:![0-9]+]]
-  // LINUX-SAME:     )
-  // LINUX:      [[TYPE_OF_FOUR]] = distinct !DICompositeType(
-  // LINUX-SAME:     tag: DW_TAG_class_type
-  // LINUX-NOT:      name:
-  // LINUX-NOT:      identifier:
+  // LINUX-SAME:     type: [[TYPE_OF_FOUR]]
   // LINUX-SAME:     )
   //
   // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "four"
-  // MSVC-SAME:      type: [[TYPE_OF_FOUR:![0-9]+]]
-  // MSVC-SAME:      )
-  // MSVC:       [[TYPE_OF_FOUR]] = distinct !DICompositeType
-  // MSVC-SAME:      tag: DW_TAG_class_type
-  // MSVC-SAME:      name: "<lambda_0>"
-  // MSVC-SAME:      identifier: ".?AV<lambda_0>@?0??main@@9@"
+  // MSVC-SAME:      type: [[TYPE_OF_FOUR]]
   // MSVC-SAME:      )
 
   return 0;
diff --git a/clang/test/CodeGenCXX/debug-info-gline-tables-only-codeview.cpp b/clang/test/CodeGenCXX/debug-info-gline-tables-only-codeview.cpp
index 6b9c9a243decd..122e4aa62ea7d 100644
--- a/clang/test/CodeGenCXX/debug-info-gline-tables-only-codeview.cpp
+++ b/clang/test/CodeGenCXX/debug-info-gline-tables-only-codeview.cpp
@@ -51,9 +51,9 @@ void test() {
   // CHECK-SAME:                              name: "<lambda_2_1>",
   c.lambda_params();
 
-  // CHECK: !DISubprogram(name: "operator()", scope: ![[LAMBDA1:[0-9]+]],
-  // CHECK: ![[LAMBDA1]] = !DICompositeType(tag: DW_TAG_class_type,
+  // CHECK: ![[LAMBDA1:[0-9]+]] = !DICompositeType(tag: DW_TAG_class_type,
   // CHECK-SAME:                            name: "<lambda_1>",
   // CHECK-SAME:                            flags: DIFlagFwdDecl
+  // CHECK: !DISubprogram(name: "operator()", scope: ![[LAMBDA1]],
   c.lambda2();
 }
diff --git a/clang/test/CodeGenCXX/debug-lambda-this.cpp b/clang/test/CodeGenCXX/debug-lambda-this.cpp
index eecbac6520ac9..3d659e7bfd004 100644
--- a/clang/test/CodeGenCXX/debug-lambda-this.cpp
+++ b/clang/test/CodeGenCXX/debug-lambda-this.cpp
@@ -13,10 +13,10 @@ int D::d(int x) {
 }
 
 // CHECK: ![[D:[0-9]+]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "D",
-// CHECK: ![[POINTER:.*]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: ![[D]], size: 64)
 // CHECK: !DIDerivedType(tag: DW_TAG_member, name: "this",
 // CHECK-SAME:           line: 11
-// CHECK-SAME:           baseType: ![[POINTER]]
+// CHECK-SAME:           baseType: ![[POINTER:[0-9]+]]
 // CHECK-SAME:           size: 64
 // CHECK-NOT:            offset: 0
 // CHECK-SAME:           ){{$}}
+// CHECK: ![[POINTER]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: ![[D]], size: 64)
diff --git a/llvm/include/llvm/IR/DIBuilder.h b/llvm/include/llvm/IR/DIBuilder.h
index ecd6dd7b0a4f8..0aa180aec6d8a 100644
--- a/llvm/include/llvm/IR/DIBuilder.h
+++ b/llvm/include/llvm/IR/DIBuilder.h
@@ -49,7 +49,7 @@ namespace llvm {
     Function *LabelFn;       ///< llvm.dbg.label
     Function *AssignFn;      ///< llvm.dbg.assign
 
-    SmallVector<TrackingMDNodeRef, 4> AllEnumTypes;
+    SmallVector<TrackingMDNodeRef, 4> EnumTypes;
     /// Track the RetainTypes, since they can be updated later on.
     SmallVector<TrackingMDNodeRef, 4> AllRetainTypes;
     SmallVector<DISubprogram *, 4> AllSubprograms;
@@ -64,8 +64,8 @@ namespace llvm {
     SmallVector<TrackingMDNodeRef, 4> UnresolvedNodes;
     bool AllowUnresolvedNodes;
 
-    /// Each subprogram's preserved local variables, labels and imported
-    /// entities.
+    /// Each subprogram's preserved local variables, labels, imported entities,
+    /// and types.
     ///
     /// Do not use a std::vector.  Some versions of libc++ apparently copy
     /// instead of move on grow operations, and TrackingMDRef is expensive to
diff --git a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
index 0a9a80688a419..321eae6fcb3b5 100644
--- a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
+++ b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
@@ -548,6 +548,8 @@ class MetadataLoader::MetadataLoaderImpl {
 
   /// Move local imports from DICompileUnit's 'imports' field to
   /// DISubprogram's retainedNodes.
+  /// Move fucntion-local enums from DICompileUnit's enums
+  /// to DISubprogram's retainedNodes.
   void upgradeCULocals() {
     if (NamedMDNode *CUNodes = TheModule.getNamedMetadata("llvm.dbg.cu")) {
       for (unsigned I = 0, E = CUNodes->getNumOperands(); I != E; ++I) {
@@ -555,48 +557,71 @@ class MetadataLoader::MetadataLoaderImpl {
         if (!CU)
           continue;
 
-        if (auto *RawImported = CU->getRawImportedEntities()) {
-          // Collect a set of imported entities to be moved.
-          SetVector<Metadata *> EntitiesToRemove;
+        SetVector<Metadata *> MetadataToRemove;
+        // Collect imported entities to be moved.
+        if (CU->getRawImportedEntities()) {
           for (Metadata *Op : CU->getImportedEntities()->operands()) {
             auto *IE = cast<DIImportedEntity>(Op);
-            if (auto *S = dyn_cast_or_null<DILocalScope>(IE->getScope())) {
-              EntitiesToRemove.insert(IE);
+            if (dyn_cast_or_null<DILocalScope>(IE->getScope())) {
+              MetadataToRemove.insert(IE);
             }
           }
+        }
+        // Collect enums to be moved.
+        if (CU->getRawEnumTypes()) {
+          for (Metadata *Op : CU->getEnumTypes()->operands()) {
+            auto *Enum = cast<DICompositeType>(Op);
+            if (dyn_cast_or_null<DILocalScope>(Enum->getScope())) {
+              MetadataToRemove.insert(Enum);
+            }
+          }
+        }
 
-          if (!EntitiesToRemove.empty()) {
-            // Make a new list of CU's 'imports'.
-            SmallVector<Metadata *> NewImports;
-            for (Metadata *Op : CU->getImportedEntities()->operands()) {
-              if (!EntitiesToRemove.contains(cast<DIImportedEntity>(Op))) {
+        if (!MetadataToRemove.empty()) {
+          // Make a new list of CU's 'imports'.
+          SmallVector<Metadata *> NewImports;
+          if (CU->getRawImportedEntities())
+            for (Metadata *Op : CU->getImportedEntities()->operands())
+              if (!MetadataToRemove.contains(Op))
                 NewImports.push_back(Op);
-              }
-            }
 
-            // Find DISubprogram corresponding to each entity.
-            std::map<DISubprogram *, SmallVector<Metadata *>> SPToEntities;
-            for (auto *I : EntitiesToRemove) {
-              auto *Entity = cast<DIImportedEntity>(I);
-              if (auto *SP = findEnclosingSubprogram(
-                      cast<DILocalScope>(Entity->getScope()))) {
-                SPToEntities[SP].push_back(Entity);
-              }
+          // Make a new list of CU's 'enums'.
+          SmallVector<Metadata *> NewEnums;
+          if (CU->getRawEnumTypes())
+            for (Metadata *Op : CU->getEnumTypes()->operands())
+              if (!MetadataToRemove.contains(Op))
+                NewEnums.push_back(Op);
+
+          // Find DISubprogram corresponding to each entity.
+          std::map<DISubprogram *, SmallVector<Metadata *>> SPToEntities;
+          for (auto *I : MetadataToRemove) {
+            DILocalScope *Scope = nullptr;
+            if (auto *Entity = dyn_cast<DIImportedEntity>(I))
+              Scope = cast<DILocalScope>(Entity->getScope());
+            else if (auto *Enum = dyn_cast<DICompositeType>(I))
+              Scope = cast<DILocalScope>(Enum->getScope());
+
+            if (auto *SP = findEnclosingSubprogram(Scope)) {
+              SPToEntities[SP].push_back(I);
             }
+          }
 
-            // Update DISubprograms' retainedNodes.
-            for (auto I = SPToEntities.begin(); I != SPToEntities.end(); ++I) {
-              auto *SP = I->first;
-              auto RetainedNodes = SP->getRetainedNodes();
-              SmallVector<Metadata *> MDs(RetainedNodes.begin(),
-                                          RetainedNodes.end());
-              MDs.append(I->second);
-              SP->replaceRetainedNodes(MDNode::get(Context, MDs));
-            }
+          // Update DISubprograms' retainedNodes.
+          for (auto I = SPToEntities.begin(); I != SPToEntities.end(); ++I) {
+            auto *SP = I->first;
+            auto RetainedNodes = SP->getRetainedNodes();
+            SmallVector<Metadata *> MDs(RetainedNodes.begin(),
+                                        RetainedNodes.end());
+            MDs.append(I->second);
+            SP->replaceRetainedNodes(MDNode::get(Context, MDs));
+          }
 
-            // Remove entities with local scope from CU.
+          // Remove entities with local scope from CU.
+          if (CU->getRawImportedEntities())
             CU->replaceImportedEntities(MDTuple::get(Context, NewImports));
-          }
+          // Remove enums with local scope from CU.
+          if (CU->getRawEnumTypes())
+            CU->replaceEnumTypes(MDTuple::get(Context, NewEnums));
         }
       }
     }
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
index 58ed21379d29b..bf8bdbb7da49f 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
@@ -598,10 +598,9 @@ void DwarfCompileUnit::constructScopeDIE(LexicalScope *Scope,
     return;
 
   // Emit lexical blocks.
-  DIE *ScopeDIE = constructLexicalScopeDIE(Scope);
+  DIE *ScopeDIE = getOrCreateLexicalBlockDIE(Scope, ParentScopeDIE);
   assert(ScopeDIE && "Scope DIE should not be null.");
 
-  ParentScopeDIE.addChild(ScopeDIE);
   createAndAddScopeChildren(Scope, *ScopeDIE);
 }
 
@@ -721,24 +720,39 @@ DIE *DwarfCompileUnit::constructInlinedScopeDIE(LexicalScope *Scope,
   return ScopeDIE;
 }
 
-// Construct new DW_TAG_lexical_block for this scope and attach
-// DW_AT_low_pc/DW_AT_high_pc labels.
-DIE *DwarfCompileUnit::constructLexicalScopeDIE(LexicalScope *Scope) {
+DIE *DwarfCompileUnit::getOrCreateLexicalBlockDIE(LexicalScope *Scope,
+                                                  DIE &ParentScopeDIE) {
   if (DD->isLexicalScopeDIENull(Scope))
     return nullptr;
   const auto *DS = Scope->getScopeNode();
-
-  auto ScopeDIE = DIE::get(DIEValueAllocator, dwarf::DW_TAG_lexical_block);
-  if (Scope->isAbstractScope()) {
-    assert(!getAbstractScopeDIEs().count(DS) &&
-           "Abstract DIE for this scope exists!");
-    getAbstractScopeDIEs()[DS] = ScopeDIE;
-    return ScopeDIE;
+  DIE *ScopeDIE = nullptr;
+
+  // FIXME: We may have a concrete DIE for this scope already created.
+  // This may happen when we emit local variables for an abstract tree of
+  // an inlined function: if a local variable has a templated type with
+  // a function-local type as a template parameter. See PR55680 for details
+  // (see also llvm/test/DebugInfo/Generic/local-type-as-template-parameter.ll).
+  if (!Scope->isAbstractScope() && !Scope->getInlinedAt()) {
+    if (auto It = LexicalBlockDIEs.find(DS); It != LexicalBlockDIEs.end()) {
+      ScopeDIE = It->second;
+      assert(!ScopeDIE->findAttribute(dwarf::DW_AT_low_pc) &&
+             !ScopeDIE->findAttribute(dwarf::DW_AT_ranges));
+      assert(ScopeDIE->getParent() == &ParentScopeDIE);
+    }
   }
-  if (!Scope->getInlinedAt()) {
-    assert(!LexicalBlockDIEs.count(DS) &&
-           "Concrete out-of-line DIE for this scope exists!");
-    LexicalBlockDIEs[DS] = ScopeDIE;
+  if (!ScopeDIE) {
+    ScopeDIE = DIE::get(DIEValueAllocator, dwarf::DW_TAG_lexical_block);
+    ParentScopeDIE.addChild(ScopeDIE);
+
+    if (Scope->isAbstractScope()) {
+      assert(!getAbstractScopeDIEs().count(DS) &&
+             "Abstract DIE for this scope exists!");
+      getAbstractScopeDIEs()[DS] = ScopeDIE;
+      return ScopeDIE;
+    }
+
+    if (!Scope->getInlinedAt())
+      LexicalBlockDIEs[DS] = ScopeDIE;
   }
 
   attachRangesOrLowHighPC(*ScopeDIE, Scope->getRanges());
@@ -1681,15 +1695,21 @@ void DwarfCompileUnit::createBaseTypeDIEs() {
   }
 }
 
-DIE *DwarfCompileUnit::getLexicalBlockDIE(const DILexicalBlock *LB) {
+DIE *DwarfCompileUnit::getLocalContextDIE(const DILexicalBlock *LB) {
   // Assume if there is an abstract tree all the DIEs are already emitted.
   bool isAbstract = getAbstractScopeDIEs().count(LB->getSubprogram());
   if (isAbstract && getAbstractScopeDIEs().count(LB))
     return getAbstractScopeDIEs()[LB];
   assert(!isAbstract && "Missed lexical block DIE in abstract tree!");
 
-  // Return a concrete DIE if it exists or nullptr otherwise.
-  return LexicalBlockDIEs.lookup(LB);
+  // Check if we have a concrete DIE.
+  if (auto It = LexicalBlockDIEs.find(LB); It != LexicalBlockDIEs.end())
+    return It->second;
+
+  // If nothing available found, we cannot just create a new lexical block,
+  // because it isn't known where to put it into the DIE tree.
+  // So, we may only try to find the most close avaiable parent DIE.
+  return getOrCreateContextDIE(LB->getScope()->getNonLexicalBlockFileScope());
 }
 
 DIE *DwarfCompileUnit::getOrCreateContextDIE(const DIScope *Context) {
@@ -1697,7 +1717,7 @@ DIE *DwarfCompileUnit::getOrCreateContextDIE(const DIScope *Context) {
     if (auto *LFScope = dyn_cast<DILexicalBlockFile>(Context))
       Context = LFScope->getNonLexicalBlockFileScope();
     if (auto *LScope = dyn_cast<DILexicalBlock>(Context))
-      return getLexicalBlockDIE(LScope);
+      return getLocalContextDIE(LScope);
 
     // Otherwise the context must be a DISubprogram.
     auto *SPScope = cast<DISubprogram>(Context);
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
index 6ef73ebd4f7f2..5662087bec322 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
@@ -210,14 +210,9 @@ class DwarfCompileUnit final : public DwarfUnit {
   /// DIE to represent this concrete inlined copy of the function.
   DIE *constructInlinedScopeDIE(LexicalScope *Scope, DIE &ParentScopeDIE);
 
-  /// Construct new DW_TAG_lexical_block for this scope and
-  /// attach DW_AT_low_pc/DW_AT_high_pc labels.
-  DIE *constructLexicalScopeDIE(LexicalScope *Scope);
-
-  /// Get a DIE for the given DILexicalBlock.
-  /// Note that this function assumes that the DIE has been already created
-  /// and it's an error, if it hasn't.
-  DIE *getLexicalBlockDIE(const DILexicalBlock *LB);
+  /// Get if available or create a new DW_TAG_lexical_block for the given
+  /// LexicalScope and attach DW_AT_low_pc/DW_AT_high_pc labels.
+  DIE *getOrCreateLexicalBlockDIE(LexicalScope *Scope, DIE &ParentDIE);
 
   /// constructVariableDIE - Construct a DIE for the given DbgVariable.
   DIE *constructVariableDIE(DbgVariable &DV, bool Abstract = false);
@@ -234,6 +229,11 @@ class DwarfCompileUnit final : public DwarfUnit {
   /// This instance of 'getOrCreateContextDIE()' can handle DILocalScope.
   DIE *getOrCreateContextDIE(const DIScope *Ty) override;
 
+  /// Get DW_TAG_lexical_block for the given DILexicalBlock if available,
+  /// or the most close parent DIE, if no correspoding DW_TAG_lexical_block
+  /// exists.
+  DIE *getLocalContextDIE(const DILexicalBlock *LB);
+
   /// Construct a DIE for this subprogram scope.
   DIE &constructSubprogramScopeDIE(const DISubprogram *Sub,
                                    LexicalScope *Scope);
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index b8502b455f587..557291c97265f 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -1229,12 +1229,13 @@ void DwarfDebug::beginModule(Module *M) {
         CU.getOrCreateGlobalVariableDIE(GV, sortGlobalExprs(GVMap[GV]));
     }
 
-    for (auto *Ty : CUNode->getEnumTypes())
+    for (auto *Ty : CUNode->getEnumTypes()) {
+      assert(!isa_and_nonnull<DILocalScope>(Ty->getScope()) &&
+             "Unexpected function-local entity in 'enums' CU field.");
       CU.getOrCreateTypeDIE(cast<DIType>(Ty));
+    }
 
     for (auto *Ty : CUNode->getRetainedTypes()) {
-      // The retained types array by design contains pointers to
-      // MDNodes rather than DIRefs. Unique them here.
       if (DIType *RT = dyn_cast<DIType>(Ty))
         // There is no point in force-emitting a forward declaration.
         CU.getOrCreateTypeDIE(RT);
@@ -1428,9 +1429,13 @@ void DwarfDebug::endModule() {
              "Unexpected function-local entity in 'imports' CU field.");
       CU->getOrCreateImportedEntityDIE(IE);
     }
+
+    // Emit function-local entities.
     for (const auto *D : CU->getDeferredLocalDecls()) {
       if (auto *IE = dyn_cast<DIImportedEntity>(D))
         CU->getOrCreateImportedEntityDIE(IE);
+      else if (auto *Ty = dyn_cast<DIType>(D))
+        CU->getOrCreateTypeDIE(Ty);
       else
         llvm_unreachable("Unexpected local retained node!");
     }
@@ -1528,6 +1533,8 @@ static const DILocalScope *getRetainedNodeScope(const MDNode *N) {
     S = L->getScope();
   else if (const auto *IE = dyn_cast<DIImportedEntity>(N))
     S = IE->getScope();
+  else if (const auto *T = dyn_cast<DIType>(N))
+    S = T->getScope();
   else
     llvm_unreachable("Unexpected retained node!");
 
diff --git a/llvm/lib/IR/DIBuilder.cpp b/llvm/lib/IR/DIBuilder.cpp
index ae1a1a37d7c74..242ca44794262 100644
--- a/llvm/lib/IR/DIBuilder.cpp
+++ b/llvm/lib/IR/DIBuilder.cpp
@@ -29,7 +29,7 @@ DIBuilder::DIBuilder(Module &m, bool AllowUnresolvedNodes, DICompileUnit *CU)
       AllowUnresolvedNodes(AllowUnresolvedNodes) {
   if (CUNode) {
     if (const auto &ETs = CUNode->getEnumTypes())
-      AllEnumTypes.assign(ETs.begin(), ETs.end());
+      EnumTypes.assign(ETs.begin(), ETs.end());
     if (const auto &RTs = CUNode->getRetainedTypes())
       AllRetainTypes.assign(RTs.begin(), RTs.end());
     if (const auto &GVs = CUNode->getGlobalVariables())
@@ -66,10 +66,10 @@ void DIBuilder::finalize() {
     return;
   }
 
-  if (!AllEnumTypes.empty())
+  if (!EnumTypes.empty())
     CUNode->replaceEnumTypes(MDTuple::get(
-        VMContext, SmallVector<Metadata *, 16>(AllEnumTypes.begin(),
-                                               AllEnumTypes.end())));
+        VMContext, SmallVector<Metadata *, 16>(EnumTypes.begin(),
+                                               EnumTypes.end())));
 
   SmallVector<Metadata *, 16> RetainValues;
   // Declarations and definitions of the same type may be retained. Some
@@ -334,10 +334,13 @@ DIDerivedType *DIBuilder::createTypedef(DIType *Ty, StringRef Name,
                                         DIScope *Context, uint32_t AlignInBits,
                                         DINode::DIFlags Flags,
                                         DINodeArray Annotations) {
-  return DIDerivedType::get(VMContext, dwarf::DW_TAG_typedef, Name, File,
-                            LineNo, getNonCompileUnitScope(Context), Ty, 0,
-                            AlignInBits, 0, std::nullopt, Flags, nullptr,
-                            Annotations);
+  auto *T =
+      DIDerivedType::get(VMContext, dwarf::DW_TAG_typedef, Name, File, LineNo,
+                         getNonCompileUnitScope(Context), Ty, 0, AlignInBits, 0,
+                         std::nullopt, Flags, nullptr, Annotations);
+  if (isa_and_nonnull<DILocalScope>(Context))
+    getSubprogramNodesTrackingVector(Context).emplace_back(T);
+  return T;
 }
 
 DIDerivedType *DIBuilder::createFriend(DIType *Ty, DIType *FriendTy) {
@@ -485,6 +488,8 @@ DICompositeType *DIBuilder::createClassType(
       OffsetInBits, Flags, Elements, 0, VTableHolder,
       cast_or_null<MDTuple>(TemplateParams), UniqueIdentifier);
   trackIfUnresolved(R);
+  if (isa_and_nonnull<DILocalScope>(Context))
+    getSubprogramNodesTrackingVector(Context).emplace_back(R);
   return R;
 }
 
@@ -498,6 +503,8 @@ DICompositeType *DIBuilder::createStructType(
       getNonCompileUnitScope(Context), DerivedFrom, SizeInBits, AlignInBits, 0,
       Flags, Elements, RunTimeLang, VTableHolder, nullptr, UniqueIdentifier);
   trackIfUnresolved(R);
+  if (isa_and_nonnull<DILocalScope>(Context))
+    getSubprogramNodesTrackingVector(Context).emplace_back(R);
   return R;
 }
 
@@ -510,6 +517,8 @@ DICompositeType *DIBuilder::createUnionType(
       getNonCompileUnitScope(Scope), nullptr, SizeInBits, AlignInBits, 0, Flags,
       Elements, RunTimeLang, nullptr, nullptr, UniqueIdentifier);
   trackIfUnresolved(R);
+  if (isa_and_nonnull<DILocalScope>(Scope))
+    getSubprogramNodesTrackingVector(Scope).emplace_back(R);
   return R;
 }
 
@@ -542,7 +551,10 @@ DICompositeType *DIBuilder::createEnumerationType(
       getNonCompileUnitScope(Scope), UnderlyingType, SizeInBits, AlignInBits, 0,
       IsScoped ? DINode::FlagEnumClass : DINode::FlagZero, Elements, 0, nullptr,
       nullptr, UniqueIdentifier);
-  AllEnumTypes.emplace_back(CTy);
+  if (isa_and_nonnull<DILocalScope>(Scope))
+    getSubprogramNodesTrackingVector(Scope).emplace_back(CTy);
+  else
+    EnumTypes.emplace_back(CTy);
   trackIfUnresolved(CTy);
   return CTy;
 }
@@ -640,6 +652,8 @@ DIBuilder::createForwardDecl(unsigned Tag, StringRef Name, DIScope *Scope,
       SizeInBits, AlignInBits, 0, DINode::FlagFwdDecl, nullptr, RuntimeLang,
       nullptr, nullptr, UniqueIdentifier);
   trackIfUnresolved(RetTy);
+  if (isa_and_nonnull<DILocalScope>(Scope))
+    getSubprogramNodesTrackingVector(Scope).emplace_back(RetTy);
   return RetTy;
 }
 
@@ -656,6 +670,8 @@ DICompositeType *DIBuilder::createReplaceableCompositeType(
           nullptr, Annotations)
           .release();
   trackIfUnresolved(RetTy);
+  if (isa_and_nonnull<DILocalScope>(Scope))
+    getSubprogramNodesTrackingVector(Scope).emplace_back(RetTy);
   return RetTy;
 }
 
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 293d443d754a4..8f929dc7f48ab 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -1384,9 +1384,9 @@ void Verifier::visitDISubprogram(const DISubprogram &N) {
     CheckDI(Node, "invalid retained nodes list", &N, RawNode);
     for (Metadata *Op : Node->operands()) {
       CheckDI(Op && (isa<DILocalVariable>(Op) || isa<DILabel>(Op) ||
-                     isa<DIImportedEntity>(Op)),
-              "invalid retained nodes, expected DILocalVariable, DILabel or "
-              "DIImportedEntity",
+                     isa<DIImportedEntity>(Op) || isa<DIType>(Op)),
+              "invalid retained nodes, expected DILocalVariable, DILabel, "
+              "DIImportedEntity or DIType",
               &N, Node, Op);
     }
   }
diff --git a/llvm/test/Bitcode/upgrade-cu-locals.ll b/llvm/test/Bitcode/upgrade-cu-locals.ll
index 9a590f0fc0774..7052ab379bea4 100644
--- a/llvm/test/Bitcode/upgrade-cu-locals.ll
+++ b/llvm/test/Bitcode/upgrade-cu-locals.ll
@@ -1,4 +1,4 @@
-; Test moving of local imports from DICompileUnit's 'imports' to DISubprogram's 'retainedNodes'
+; Test moving of local imports/enums from DICompileUnit to DISubprogram's 'retainedNodes'
 ;
 ; RUN: llvm-dis -o - %s.bc | FileCheck %s
 
@@ -31,31 +31,36 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo
 ; CHECK: !4 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t4"
 
 ; CHECK: !5 = !{}
-; CHECK: !6 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !7, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, imports: !5, nameTableKind: GNU)
-
-; CHECK: !14 = distinct !DISubprogram(name: "main", scope: !7, file: !7, line: 2, type: !15, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !18)
-; CHECK: !18 = !{!19}
-; CHECK: !19 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !20, entity: !23,
-; CHECK: !20 = !DILexicalBlock(scope: !21, file: !7, line: 7, column: 35)
-; CHECK: !21 = !DILexicalBlock(scope: !22, file: !7, line: 7, column: 35)
-; CHECK: !22 = !DILexicalBlock(scope: !14, file: !7, line: 7, column: 35)
-; CHECK: !23 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t5", scope: !20,
-
-; CHECK: !25 = distinct !DISubprogram(name: "f1", linkageName: "_Z2f1v", scope: !1, file: !1, line: 3, type: !26, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !28)
-; CHECK: !28 = !{!29, !32, !34}
-; CHECK: !29 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !25, entity: !30,
-; CHECK: !30 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t1",
-; CHECK: !32 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !25, entity: !33,
-; CHECK: !33 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t2",
-; CHECK: !34 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !25, entity: !35,
-; CHECK: !35 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t3",
-
-; CHECK: !40 = distinct !DISubprogram(name: "main2", scope: !7, file: !7, line: 10, type: !15, scopeLine: 10, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !41)
-; CHECK: !41 = !{!42, !44}
-; CHECK: !42 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !40, entity: !43,
-; CHECK: !43 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t6"
-; CHECK: !44 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !40, entity: !45,
-; CHECK: !45 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t7",
+; CHECK: !6 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !7, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !8, imports: !5, nameTableKind: GNU)
+; CHECK: !7 = !DIFile(filename: "b.cpp"
+; CHECK: !8 = !{!9}
+; CHECK: !9 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum2", scope: !6, file: !7, line: 4, size: 8, align: 8, elements: !5)
+
+; CHECK: !16 = distinct !DISubprogram(name: "main", scope: !7, file: !7, line: 2, type: !17, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !20)
+; CHECK: !20 = !{!21}
+; CHECK: !21 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !22, entity: !25,
+; CHECK: !22 = !DILexicalBlock(scope: !23, file: !7, line: 7, column: 35)
+; CHECK: !23 = !DILexicalBlock(scope: !24, file: !7, line: 7, column: 35)
+; CHECK: !24 = !DILexicalBlock(scope: !16, file: !7, line: 7, column: 35)
+; CHECK: !25 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t5", scope: !22,
+
+; CHECK: !27 = distinct !DISubprogram(name: "f1", linkageName: "_Z2f1v", scope: !1, file: !1, line: 3, type: !28, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !30)
+; CHECK: !30 = !{!31, !34, !36}
+; CHECK: !31 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !27, entity: !32,
+; CHECK: !32 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t1",
+; CHECK: !34 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !27, entity: !35,
+; CHECK: !35 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t2",
+; CHECK: !36 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !27, entity: !37,
+; CHECK: !37 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t3",
+
+; CHECK: !42 = distinct !DISubprogram(name: "main2", scope: !7, file: !7, line: 10, type: !17, scopeLine: 10, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !43)
+; CHECK: !43 = !{!44, !46, !48, !49}
+; CHECK: !44 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !42, entity: !45,
+; CHECK: !45 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t6"
+; CHECK: !46 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !42, entity: !47,
+; CHECK: !47 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t7",
+; CHECK: !48 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum", scope: !42, file: !7, line: 3, size: 8, align: 8, elements: !5)
+; CHECK: !49 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum3", scope: !42, file: !7, line: 5, size: 8, align: 8, elements: !5)
 
 
 !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, imports: !2, nameTableKind: GNU)
@@ -82,7 +87,7 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo
 ; Leave t4 in CU
 !14 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !0, entity: !15, file: !1, line: 3)
 !15 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t4", scope: !0, file: !1, line: 1, size: 8, flags: DIFlagTypePassByValue, elements: !7, identifier: "_ZTSN2ns2t4E")
-!16 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !17, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, imports: !18, nameTableKind: GNU)
+!16 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !17, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, imports: !18, enums: !50, nameTableKind: GNU)
 !17 = !DIFile(filename: "b.cpp", directory: "/")
 !18 = !{!19, !28, !31}
 
@@ -116,3 +121,12 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo
 !41 = distinct !DILocation(line: 3, column: 3, scope: !23)
 !42 = !DILocation(line: 3, column: 41, scope: !4, inlinedAt: !41)
 !43 = !DILocation(line: 4, column: 1, scope: !23)
+
+!50 = !{!51, !52, !53}
+; Move to main2
+!51 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum", scope: !29, file: !17, line: 3, size: 8, align: 8, elements: !7)
+; Leave in b.cpp's CU
+!52 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum2", scope: !16, file: !17, line: 4, size: 8, align: 8, elements: !7)
+; Move to main2
+!53 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum3", scope: !29, file: !17, line: 5, size: 8, align: 8, elements: !7)
+
diff --git a/llvm/test/Bitcode/upgrade-cu-locals.ll.bc b/llvm/test/Bitcode/upgrade-cu-locals.ll.bc
index 9d0ea8fd9a3704ea1fb3a92250500addc75f0587..f73d05668af5b16f9ee03e400e2877faab339a0d 100644
GIT binary patch
delta 1080
zcmYk5e`p(J7{{NxT<<RFwOop6V|rQMH7e;k%bOqVBKYItmQkU&t8>^v2T79^Yj3eX
zCa4ST^$4!hY-CFq`$I=`D*iELuKy_e=R!ISm`cGRL56hTA0ibxhWMA%_q{6H%X`oB
z-1ohE-p}*!zJ}+nr}PA4Tzv8e=ETMO&yU}{_3PDJt*yNt=um-!|A&k}H9J|6zN*x)
zz`pltPnw1RR8~Q?N^cBEwg8N+ilr9PDjEQ4=T<*p+z#fi@<&_{iTssC`dw0W77=EH
z*c_9SV-gA2gRx8VvGQFm-JQ(6b&5U_YNDso)49lW?sWONHl>_yJF*18yXh_f*wo~p
zm<*emHz?jF4h4_b4>b}uctVLpliLG87yzu1pRs>l{*5sRvWb=j0!DRwX=wDE!o5S^
zOf<4d(z>W#%y#_LCnWZe`V1s?y-g$s)sK#yRhD8G6zL2mOMNo{bO~2Hg=K}-PyV<3
zq#>kxqe*`rGx=)KWD%^aF+h`3j$?WG#v>f#G%w}o32^!&9A2suj-yWr%l!{I44%|a
zL4<$~0r4859CHsjn1Ia@0;sw`ra%slionW?#uo{nWGL`pFMPtkj$s~}XJN(nVGSA#
z4zOq@#XFE-No&eR!r!V%R>}e1C})ip;CBXV%(E=PadK-*f}<Bi)~=kQK=3&0zR$P?
zFab!z?O}~vR!8z0DHMT0x@`|!-w+nJg$D{euBG5HOG<>AheRU;84oDiwJ=Y7|E^bU
zPf#aaZ~liW_I^`?Z3Zt9R(*qaScJ9SUwv_3Q<kDq+B#R0>fX~?T-W%k-X<>8UgL=j
zd}ekQW|i;f7Vq_tsi(X3-JsT;L+v-o1XsyBp4;T7(%n-D;dAR-CS<UohPl7N`l(K4
zr0Ys`qSe~^Wumbb`*GsCKMuR^K<6vz%<9a-OTmGKYrRRW?O0>w{Q6hxpZ$6M`h_o>
zw~J42>AA|JXZ&N&#J$xHv5k3on>Xg(ukv3b;h>Ln<a)b(XM#LwT?+4icV}nkh0TA6
zo&B4bdNC2MEc==)o>a+U<_nRGCQVoQRnT!plNqI)a{8Ff@)_sAV5(!c!vJ$=_ozlX
z<-P*D&c#bd?4=WAs{s;qIV!309p_*u{7g7Dd~zy26wOR#W081lI2)ci5Q#<(#PJXw
bjz`1MY<74kUf$<AES)45abZx`X{vt#JnkA(

delta 1011
zcmYjPT}&fY6h1R;?+nAR8K7ukpbT3~NOmQ+|Ckt^#_fv6bXy??4aQDuk;YOAbd5%0
zrv<}iQ#P<|%$oR6v-sc!%?9*sAEv1*sKCpvhU}79AB@J-1png8D&A>B@aE3>`OdxH
z`DW%Ebf_IG{nX^@Bi~TU>Tgesr^SEYyYtha=j~yWY!HBg<7MPOE-e79rviCYP}RO~
z{gy^qhXoMu+C9!tM`Yd7K=*rCbF3X;n<h+~OH`|YIw-tvv~}+*HOenH<t0qpa4y8=
zHS>ZdEU$I902B_4N`Kn>%1fgo0Y`Gqk@ou2o|DYupHkvUM?9G>Oq(ubP_5n(a4sm=
zSie0NwAk3&#7VXnE9Y+cR78YGZ0=kSKu|xb4S$DwriFb<1z>Yhhyjpv9A9-m_myaT
zL(g*3@Li&q)A9VPEq8uL*&bj)mDsk61ggK}NN7uR?PWy%O-xFsCjs;lE_N)IMV>_D
zKJ<VL#yUlNktuhbP{;%YAPW0`^xgcQLag7?M(?gN<`w4afdQb^OCrcbyRM_~7ovJk
zNHh~b24E!gv^6YS+3S+7BYC33XFb+2(9pq;LKfhUKA1%l7^Y!o3AQ_nWw=G*URruh
zMutPA&9iE$Y~eK#oI*-qW-;T0g%$|!2;gj`Q;a5?4q3o2P@nm%-O&sQe%1rxYjsbU
z=Cxy{^=ua}TlJP3D30MR*uj5hdmoe^XJpi20TX^o;0UY4X?z)1s<H2RgJn{|6$#z<
zR;n`*f?W)!5|HZNntA7khsjPT59ziZB&B9$Lu%dtZpAW|tfCTdC6Y?b=`i<!I2Ng*
zNfkNTu9e33_iOjYx4+mM|MHjft$U@G=auN@>vPZ8&dgmG{g!t_+n1JiKi&Q4*X8vq
zpX}VuKUzy9i#hYNWpjYqoMKH>Ktt-x<rKPch!#4@&scBPKP}i01Gz9h`{B{i(UTAU
zA-4KAF_pZ}UMzRsq;W(hvxgQP(I6jBp>;^$Xi)Hp1+A%zQVaK+&J4<A;u~v5YD|{n
zlyG{H{@geu1nPyCNvpFYYBLlg1s69s5SfVLsK@JZx`(jG?sT~nm&;-I+LfWlT#gBk
cSBb^kZo4~HxXhgw;!~;V+4S6CBGKsbANlGKqW}N^

diff --git a/llvm/test/DebugInfo/Generic/inlined-local-type.ll b/llvm/test/DebugInfo/Generic/inlined-local-type.ll
new file mode 100644
index 0000000000000..a6f7d294de928
--- /dev/null
+++ b/llvm/test/DebugInfo/Generic/inlined-local-type.ll
@@ -0,0 +1,128 @@
+; RUN: %llc_dwarf -O0 -filetype=obj < %s | llvm-dwarfdump -debug-info - | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s
+; REQUIRES: object-emission
+
+; inline __attribute__((always_inline))
+; int removed() { struct A {int i;}; struct A a; return a.i++; }
+;
+; __attribute__((always_inline))
+; int not_removed() { struct B {int i;}; struct B b; return b.i++; }
+;
+; int foo() { return removed() + not_removed(); }}
+
+; Ensure that function-local types have the correct subprogram parent even if
+; those subprograms are inlined.
+
+; CHECK: DW_TAG_compile_unit
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_AT_abstract_origin	({{0x.*}} "not_removed")
+; CHECK:     DW_TAG_variable
+; CHECK:     NULL
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_AT_name	("removed")
+; CHECK: [[A:0x.*]]: DW_TAG_structure_type
+; CHECK:       DW_AT_name	("A")
+; CHECK:       DW_TAG_member
+; CHECK:       NULL
+; CHECK:     DW_TAG_variable
+; CHECK:       DW_AT_type	([[A]] "A")
+; CHECK:     NULL
+; CHECK:   DW_TAG_base_type
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_AT_name	("not_removed")
+; CHECK: [[B:0x.*]]: DW_TAG_structure_type
+; CHECK:       DW_AT_name	("B")
+; CHECK:       DW_TAG_member
+; CHECK:       NULL
+; CHECK:     DW_TAG_variable
+; CHECK:       DW_AT_type	([[B]] "B")
+; CHECK:     NULL
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_TAG_inlined_subroutine
+; CHECK:       DW_TAG_variable
+; CHECK:       NULL
+; CHECK:     DW_TAG_inlined_subroutine
+; CHECK:       DW_TAG_variable
+; CHECK:       NULL
+; CHECK:     NULL
+; CHECK:   NULL
+
+%struct.B = type { i32 }
+%struct.A = type { i32 }
+
+define dso_local i32 @not_removed() !dbg !12 {
+  %1 = alloca %struct.B, align 4
+  call void @llvm.dbg.declare(metadata %struct.B* %1, metadata !18, metadata !DIExpression()), !dbg !22
+  %2 = getelementptr inbounds %struct.B, %struct.B* %1, i32 0, i32 0, !dbg !23
+  %3 = load i32, i32* %2, align 4, !dbg !24
+  %4 = add nsw i32 %3, 1, !dbg !24
+  store i32 %4, i32* %2, align 4, !dbg !24
+  ret i32 %3, !dbg !25
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+define dso_local i32 @foo() !dbg !26 {
+  %1 = alloca %struct.A, align 4
+  %2 = alloca %struct.B, align 4
+  call void @llvm.dbg.declare(metadata %struct.A* %1, metadata !27, metadata !DIExpression()), !dbg !32
+  %3 = getelementptr inbounds %struct.A, %struct.A* %1, i32 0, i32 0, !dbg !34
+  %4 = load i32, i32* %3, align 4, !dbg !35
+  %5 = add nsw i32 %4, 1, !dbg !35
+  store i32 %5, i32* %3, align 4, !dbg !35
+  call void @llvm.dbg.declare(metadata %struct.B* %2, metadata !18, metadata !DIExpression()), !dbg !36
+  %6 = getelementptr inbounds %struct.B, %struct.B* %2, i32 0, i32 0, !dbg !38
+  %7 = load i32, i32* %6, align 4, !dbg !39
+  %8 = add nsw i32 %7, 1, !dbg !39
+  store i32 %8, i32* %6, align 4, !dbg !39
+  %9 = add nsw i32 %4, %7, !dbg !40
+  ret i32 %9, !dbg !41
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3, !4, !5, !6, !7, !8, !9, !10}
+!llvm.ident = !{!11}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "inlined-local-type.cpp", directory: "")
+!2 = !{i32 7, !"Dwarf Version", i32 4}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"wchar_size", i32 4}
+!5 = !{i32 1, !"branch-target-enforcement", i32 0}
+!6 = !{i32 1, !"sign-return-address", i32 0}
+!7 = !{i32 1, !"sign-return-address-all", i32 0}
+!8 = !{i32 1, !"sign-return-address-with-bkey", i32 0}
+!9 = !{i32 7, !"uwtable", i32 1}
+!10 = !{i32 7, !"frame-pointer", i32 1}
+!11 = !{!"clang version 14.0.0"}
+!12 = distinct !DISubprogram(name: "not_removed", scope: !13, file: !13, line: 5, type: !14, scopeLine: 5, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !43)
+!13 = !DIFile(filename: "inlined-local-type.cpp", directory: "")
+!14 = !DISubroutineType(types: !15)
+!15 = !{!16}
+!16 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!17 = !{}
+!18 = !DILocalVariable(name: "b", scope: !12, file: !13, line: 5, type: !19)
+!19 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "B", scope: !12, file: !13, line: 5, size: 32, elements: !20)
+!20 = !{!21}
+!21 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !19, file: !13, line: 5, baseType: !16, size: 32)
+!22 = !DILocation(line: 5, column: 49, scope: !12)
+!23 = !DILocation(line: 5, column: 61, scope: !12)
+!24 = !DILocation(line: 5, column: 62, scope: !12)
+!25 = !DILocation(line: 5, column: 52, scope: !12)
+!26 = distinct !DISubprogram(name: "foo", scope: !13, file: !13, line: 7, type: !14, scopeLine: 7, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !17)
+!27 = !DILocalVariable(name: "a", scope: !28, file: !13, line: 2, type: !29)
+!28 = distinct !DISubprogram(name: "removed", scope: !13, file: !13, line: 2, type: !14, scopeLine: 2, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !42)
+!29 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A", scope: !28, file: !13, line: 2, size: 32, elements: !30)
+!30 = !{!31}
+!31 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !29, file: !13, line: 2, baseType: !16, size: 32)
+!32 = !DILocation(line: 2, column: 45, scope: !28, inlinedAt: !33)
+!33 = distinct !DILocation(line: 7, column: 20, scope: !26)
+!34 = !DILocation(line: 2, column: 57, scope: !28, inlinedAt: !33)
+!35 = !DILocation(line: 2, column: 58, scope: !28, inlinedAt: !33)
+!36 = !DILocation(line: 5, column: 49, scope: !12, inlinedAt: !37)
+!37 = distinct !DILocation(line: 7, column: 32, scope: !26)
+!38 = !DILocation(line: 5, column: 61, scope: !12, inlinedAt: !37)
+!39 = !DILocation(line: 5, column: 62, scope: !12, inlinedAt: !37)
+!40 = !DILocation(line: 7, column: 30, scope: !26)
+!41 = !DILocation(line: 7, column: 13, scope: !26)
+!42 = !{!29}
+!43 = !{!19}
diff --git a/llvm/test/DebugInfo/Generic/lexical-block-retained-types.ll b/llvm/test/DebugInfo/Generic/lexical-block-retained-types.ll
new file mode 100644
index 0000000000000..64de1323c98c6
--- /dev/null
+++ b/llvm/test/DebugInfo/Generic/lexical-block-retained-types.ll
@@ -0,0 +1,55 @@
+; RUN: %llc_dwarf -O0 -filetype=obj < %s | llvm-dwarfdump - | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s
+
+; Test that retained unused (unreferenced) types emission.
+
+; Compiled from
+; $ clang -cc1 -debug-info-kind=unused-types test.cpp -emit-llvm
+
+; void test_unused() {
+;   struct Y {};
+;   {
+;     struct X {};
+;   }
+; }
+
+; CHECK: DW_TAG_compile_unit
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_AT_name	("test_unused")
+; CHECK:     DW_TAG_structure_type
+; CHECK:       DW_AT_name	("Y")
+
+; FIXME: here should be DW_TAG_lexical_block as a parent of structure 'X'.
+; But it's not possible to reliably emit a lexical block for which a LexicalScope
+; wasn't created, so we just fallback to the most close parent DIE
+; (see DwarfCompileUnit::getOrCreateLexicalBlockDIE() for details).
+
+; CHECK:     DW_TAG_structure_type
+; CHECK:       DW_AT_name	("X")
+; CHECK:     NULL
+; CHECK:   NULL
+
+define dso_local void @_Z11test_unusedv() !dbg !5 {
+entry:
+  ret void, !dbg !16
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!13, !14}
+!llvm.ident = !{!15}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 15.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, retainedTypes: !2, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "<stdin>", directory: "/")
+!2 = !{!3, !10}
+!3 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Y", scope: !5, file: !4, line: 2, size: 8, flags: DIFlagTypePassByValue, elements: !8)
+!4 = !DIFile(filename: "test.cpp", directory: "/")
+!5 = distinct !DISubprogram(name: "test_unused", linkageName: "_Z11test_unusedv", scope: !4, file: !4, line: 1, type: !6, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !9)
+!6 = !DISubroutineType(types: !7)
+!7 = !{null}
+!8 = !{}
+!9 = !{!3, !10}
+!10 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "X", scope: !11, file: !4, line: 4, size: 8, flags: DIFlagTypePassByValue, elements: !8)
+!11 = distinct !DILexicalBlock(scope: !5, file: !4, line: 3, column: 3)
+!13 = !{i32 2, !"Debug Info Version", i32 3}
+!14 = !{i32 1, !"wchar_size", i32 4}
+!15 = !{!"clang version 15.0.0"}
+!16 = !DILocation(line: 6, column: 1, scope: !5)
diff --git a/llvm/test/DebugInfo/Generic/lexical-block-types.ll b/llvm/test/DebugInfo/Generic/lexical-block-types.ll
new file mode 100644
index 0000000000000..aef59984351a7
--- /dev/null
+++ b/llvm/test/DebugInfo/Generic/lexical-block-types.ll
@@ -0,0 +1,425 @@
+; RUN: %llc_dwarf -O0 -filetype=obj < %s | llvm-dwarfdump -debug-info - | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s
+; REQUIRES: object-emission
+
+; inline __attribute__((always_inline))
+; void removed() {
+;   struct A1 { int i; };
+;   typedef int Int1;
+;   {
+;     struct I1 { Int1 j; };
+;     struct C1 { typedef char Char1; Char1 c; };
+;     A1 a1; a1.i++;
+;     {
+;       I1 i1; i1.j++;
+;       C1 c1; c1.c++;
+;     }
+;   }
+; }
+;
+; __attribute__((always_inline))
+; void not_removed() {
+;   struct A2 { int i; };
+;   typedef int Int2;
+;   {
+;     struct I2 { Int2 j; };
+;     struct C2 { typedef char Char2; Char2 c; };
+;     A2 a2; a2.i++;
+;     {
+;       I2 i2; i2.j++;
+;       C2 c2; c2.c++;
+;     }
+;   }
+; }
+;
+; void foo() {
+;   struct A3 { int i; };
+;   typedef int Int3;
+;   {
+;     struct I3 { Int3 j; };
+;     {
+;       struct C3 { typedef char Char3; Char3 c; };
+;       A3 a3; a3.i++;
+;       {
+;         I3 i3; i3.j++;
+;         C3 c3; c3.c++;
+;       }
+;     }
+;   }
+;   removed();
+;   not_removed();
+; }
+;
+; CHECK: DW_TAG_compile_unit
+
+; Out-of-line definition of `not_removed()` shouldn't contain any debug info for types.
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_AT_abstract_origin	{{.*}} "_Z11not_removedv"
+; CHECK:     DW_TAG_lexical_block
+; CHECK:       DW_TAG_variable
+; CHECK:         DW_AT_abstract_origin	{{.*}} "a2"
+; CHECK:       DW_TAG_lexical_block
+; CHECK:         DW_TAG_variable
+; CHECK:           DW_AT_abstract_origin	{{.*}} "i2"
+; CHECK:         DW_TAG_variable
+; CHECK:           DW_AT_abstract_origin	{{.*}} "c2"
+; CHECK:         NULL
+; CHECK:       NULL
+; CHECK:     NULL
+
+; Abstract definition of `removed()`.
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_AT_name	("removed")
+; CHECK:     DW_AT_inline	(DW_INL_inlined)
+
+; I1 and C1 defined in the first lexical block, typedef Char1 is a child of C1.
+; CHECK:     DW_TAG_lexical_block
+; CHECK:       DW_TAG_variable
+; CHECK:         DW_AT_name	("a1")
+; CHECK:         DW_AT_type	{{.*}} "A1"
+; CHECK:       DW_TAG_lexical_block
+; CHECK:         DW_TAG_variable
+; CHECK:           DW_AT_type	{{.*}} "I1"
+; CHECK:         DW_TAG_variable
+; CHECK:           DW_AT_type	{{.*}} "C1"
+; CHECK:         NULL
+; CHECK:       DW_TAG_structure_type
+; CHECK:         DW_AT_name	("I1")
+; CHECK:         DW_TAG_member
+; CHECK:           DW_AT_type	{{.*}} "Int1"
+; CHECK:         NULL
+; CHECK:       DW_TAG_structure_type
+; CHECK:         DW_AT_name	("C1")
+; CHECK:         DW_TAG_member
+; CHECK:           DW_AT_type	{{.*}} "Char1"
+; CHECK:         DW_TAG_typedef
+; CHECK:           DW_AT_name	("Char1")
+; CHECK:         NULL
+; CHECK:       NULL
+
+; A1 and typedef Int1 defined in the subprogram scope.
+; CHECK:     DW_TAG_structure_type
+; CHECK:       DW_AT_name	("A1")
+; CHECK:       DW_TAG_member
+; CHECK:       NULL
+; CHECK:     DW_TAG_typedef
+; CHECK:       DW_AT_name	("Int1")
+; CHECK:     NULL
+
+; CHECK:   DW_TAG_base_type
+; CHECK:   DW_TAG_base_type
+
+; Abstract definition of `not_removed()`.
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_AT_name	("not_removed")
+; CHECK:     DW_AT_inline	(DW_INL_inlined)
+
+; I2 and C2 defined in the first lexical block, typedef Char2 is a child of C2.
+; CHECK:     DW_TAG_lexical_block
+; CHECK:       DW_TAG_variable
+; CHECK:         DW_AT_name	("a2")
+; CHECK:         DW_AT_type	{{.*}} "A2"
+; CHECK:       DW_TAG_lexical_block
+; CHECK:         DW_TAG_variable
+; CHECK:           DW_AT_name	("i2")
+; CHECK:           DW_AT_type	{{.*}} "I2"
+; CHECK:         DW_TAG_variable
+; CHECK:           DW_AT_name	("c2")
+; CHECK:           DW_AT_type	{{.*}} "C2"
+; CHECK:         NULL
+; CHECK:       DW_TAG_structure_type
+; CHECK:         DW_AT_name	("I2")
+; CHECK:         DW_TAG_member
+; CHECK:           DW_AT_type	{{.*}} "Int2"
+; CHECK:         NULL
+; CHECK:       DW_TAG_structure_type
+; CHECK:         DW_AT_name	("C2")
+; CHECK:         DW_TAG_member
+; CHECK:           DW_AT_type	{{.*}} "Char2"
+; CHECK:         DW_TAG_typedef
+; CHECK:           DW_AT_name	("Char2")
+; CHECK:         NULL
+; CHECK:       NULL
+
+; A2 and typedef Int2 defined in subprogram scope.
+; CHECK:     DW_TAG_structure_type
+; CHECK:       DW_AT_name	("A2")
+; CHECK:       DW_TAG_member
+; CHECK:       NULL
+; CHECK:     DW_TAG_typedef
+; CHECK:       DW_AT_name	("Int2")
+; CHECK:     NULL
+
+; Definition of `foo()`.
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_AT_name	("foo")
+
+; CHECK:     DW_TAG_lexical_block
+; CHECK:       DW_TAG_lexical_block
+; CHECK:         DW_TAG_variable
+; CHECK:           DW_AT_name	("a3")
+; CHECK:           DW_AT_type	{{.*}} "A3"
+; CHECK:         DW_TAG_lexical_block
+; CHECK:           DW_TAG_variable
+; CHECK:             DW_AT_name	("i3")
+; CHECK:             DW_AT_type	{{.*}} "I3"
+; CHECK:           DW_TAG_variable
+; CHECK:             DW_AT_name	("c3")
+; CHECK:             DW_AT_type	{{.*}} "C3"
+; CHECK:           NULL
+
+; C3 has the inner lexical block scope, typedef Char3 is a child of C3.
+; CHECK:         DW_TAG_structure_type
+; CHECK:           DW_AT_name	("C3")
+; CHECK:           DW_TAG_member
+; CHECK:             DW_AT_type	{{.*}} "Char3"
+; CHECK:           DW_TAG_typedef
+; CHECK:             DW_AT_name	("Char3")
+; CHECK:           NULL
+; CHECK:         NULL
+
+; I3 has the outer lexical block scope.
+; CHECK:       DW_TAG_structure_type
+; CHECK:         DW_AT_name	("I3")
+; CHECK:         DW_TAG_member
+; CHECK:           DW_AT_type	{{.*}} "Int3"
+; CHECK:         NULL
+; CHECK:       NULL
+
+; CHECK:     DW_TAG_inlined_subroutine
+; CHECK:       DW_AT_abstract_origin	{{.*}} "_Z7removedv"
+; CHECK:       DW_TAG_lexical_block
+; CHECK:         DW_TAG_variable
+; CHECK:         DW_TAG_lexical_block
+; CHECK:           DW_TAG_variable
+; CHECK:           DW_TAG_variable
+; CHECK:           NULL
+; CHECK:         NULL
+; CHECK:       NULL
+
+; CHECK:     DW_TAG_inlined_subroutine
+; CHECK:       DW_AT_abstract_origin	{{.*}} "_Z11not_removedv"
+; CHECK:       DW_TAG_lexical_block
+; CHECK:         DW_TAG_variable
+; CHECK:         DW_TAG_lexical_block
+; CHECK:           DW_TAG_variable
+; CHECK:           DW_TAG_variable
+; CHECK:           NULL
+; CHECK:         NULL
+; CHECK:       NULL
+
+; A3 and Int3 defined within the subprogam scope.
+; CHECK:     DW_TAG_structure_type
+; CHECK:       DW_AT_name	("A3")
+; CHECK:       DW_TAG_member
+; CHECK:       NULL
+; CHECK:     DW_TAG_typedef
+; CHECK:       DW_AT_name	("Int3")
+; CHECK:     NULL
+; CHECK:   NULL
+
+%struct.A2 = type { i32 }
+%struct.I2 = type { i32 }
+%struct.C2 = type { i8 }
+%struct.A1 = type { i32 }
+%struct.I1 = type { i32 }
+%struct.C1 = type { i8 }
+%struct.A3 = type { i32 }
+%struct.I3 = type { i32 }
+%struct.C3 = type { i8 }
+
+define dso_local void @_Z11not_removedv() !dbg !8 {
+entry:
+  %a2 = alloca %struct.A2, align 4
+  %i2 = alloca %struct.I2, align 4
+  %c2 = alloca %struct.C2, align 1
+  call void @llvm.dbg.declare(metadata %struct.A2* %a2, metadata !12, metadata !DIExpression()), !dbg !18
+  %i = getelementptr inbounds %struct.A2, %struct.A2* %a2, i32 0, i32 0, !dbg !19
+  %0 = load i32, i32* %i, align 4, !dbg !20
+  %inc = add nsw i32 %0, 1, !dbg !20
+  store i32 %inc, i32* %i, align 4, !dbg !20
+  call void @llvm.dbg.declare(metadata %struct.I2* %i2, metadata !21, metadata !DIExpression()), !dbg !27
+  %j = getelementptr inbounds %struct.I2, %struct.I2* %i2, i32 0, i32 0, !dbg !28
+  %1 = load i32, i32* %j, align 4, !dbg !29
+  %inc1 = add nsw i32 %1, 1, !dbg !29
+  store i32 %inc1, i32* %j, align 4, !dbg !29
+  call void @llvm.dbg.declare(metadata %struct.C2* %c2, metadata !30, metadata !DIExpression()), !dbg !36
+  %c = getelementptr inbounds %struct.C2, %struct.C2* %c2, i32 0, i32 0, !dbg !37
+  %2 = load i8, i8* %c, align 1, !dbg !38
+  %inc2 = add i8 %2, 1, !dbg !38
+  store i8 %inc2, i8* %c, align 1, !dbg !38
+  ret void, !dbg !39
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+define dso_local void @_Z3foov() !dbg !40 {
+entry:
+  %a1.i = alloca %struct.A1, align 4
+  %i1.i = alloca %struct.I1, align 4
+  %c1.i = alloca %struct.C1, align 1
+  %a2.i = alloca %struct.A2, align 4
+  %i2.i = alloca %struct.I2, align 4
+  %c2.i = alloca %struct.C2, align 1
+  %a3 = alloca %struct.A3, align 4
+  %i3 = alloca %struct.I3, align 4
+  %c3 = alloca %struct.C3, align 1
+  call void @llvm.dbg.declare(metadata %struct.A3* %a3, metadata !41, metadata !DIExpression()), !dbg !47
+  %i = getelementptr inbounds %struct.A3, %struct.A3* %a3, i32 0, i32 0, !dbg !48
+  %0 = load i32, i32* %i, align 4, !dbg !49
+  %inc = add nsw i32 %0, 1, !dbg !49
+  store i32 %inc, i32* %i, align 4, !dbg !49
+  call void @llvm.dbg.declare(metadata %struct.I3* %i3, metadata !50, metadata !DIExpression()), !dbg !56
+  %j = getelementptr inbounds %struct.I3, %struct.I3* %i3, i32 0, i32 0, !dbg !57
+  %1 = load i32, i32* %j, align 4, !dbg !58
+  %inc1 = add nsw i32 %1, 1, !dbg !58
+  store i32 %inc1, i32* %j, align 4, !dbg !58
+  call void @llvm.dbg.declare(metadata %struct.C3* %c3, metadata !59, metadata !DIExpression()), !dbg !64
+  %c = getelementptr inbounds %struct.C3, %struct.C3* %c3, i32 0, i32 0, !dbg !65
+  %2 = load i8, i8* %c, align 1, !dbg !66
+  %inc2 = add i8 %2, 1, !dbg !66
+  store i8 %inc2, i8* %c, align 1, !dbg !66
+  call void @llvm.dbg.declare(metadata %struct.A1* %a1.i, metadata !67, metadata !DIExpression()), !dbg !73
+  %i.i3 = getelementptr inbounds %struct.A1, %struct.A1* %a1.i, i32 0, i32 0, !dbg !75
+  %3 = load i32, i32* %i.i3, align 4, !dbg !76
+  %inc.i4 = add nsw i32 %3, 1, !dbg !76
+  store i32 %inc.i4, i32* %i.i3, align 4, !dbg !76
+  call void @llvm.dbg.declare(metadata %struct.I1* %i1.i, metadata !77, metadata !DIExpression()), !dbg !83
+  %j.i5 = getelementptr inbounds %struct.I1, %struct.I1* %i1.i, i32 0, i32 0, !dbg !84
+  %4 = load i32, i32* %j.i5, align 4, !dbg !85
+  %inc1.i6 = add nsw i32 %4, 1, !dbg !85
+  store i32 %inc1.i6, i32* %j.i5, align 4, !dbg !85
+  call void @llvm.dbg.declare(metadata %struct.C1* %c1.i, metadata !86, metadata !DIExpression()), !dbg !91
+  %c.i7 = getelementptr inbounds %struct.C1, %struct.C1* %c1.i, i32 0, i32 0, !dbg !92
+  %5 = load i8, i8* %c.i7, align 1, !dbg !93
+  %inc2.i8 = add i8 %5, 1, !dbg !93
+  store i8 %inc2.i8, i8* %c.i7, align 1, !dbg !93
+  call void @llvm.dbg.declare(metadata %struct.A2* %a2.i, metadata !12, metadata !DIExpression()), !dbg !94
+  %i.i = getelementptr inbounds %struct.A2, %struct.A2* %a2.i, i32 0, i32 0, !dbg !96
+  %6 = load i32, i32* %i.i, align 4, !dbg !97
+  %inc.i = add nsw i32 %6, 1, !dbg !97
+  store i32 %inc.i, i32* %i.i, align 4, !dbg !97
+  call void @llvm.dbg.declare(metadata %struct.I2* %i2.i, metadata !21, metadata !DIExpression()), !dbg !98
+  %j.i = getelementptr inbounds %struct.I2, %struct.I2* %i2.i, i32 0, i32 0, !dbg !99
+  %7 = load i32, i32* %j.i, align 4, !dbg !100
+  %inc1.i = add nsw i32 %7, 1, !dbg !100
+  store i32 %inc1.i, i32* %j.i, align 4, !dbg !100
+  call void @llvm.dbg.declare(metadata %struct.C2* %c2.i, metadata !30, metadata !DIExpression()), !dbg !101
+  %c.i = getelementptr inbounds %struct.C2, %struct.C2* %c2.i, i32 0, i32 0, !dbg !102
+  %8 = load i8, i8* %c.i, align 1, !dbg !103
+  %inc2.i = add i8 %8, 1, !dbg !103
+  store i8 %inc2.i, i8* %c.i, align 1, !dbg !103
+  ret void, !dbg !104
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "test.cpp", directory: "/")
+!2 = !{i32 7, !"Dwarf Version", i32 4}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"wchar_size", i32 4}
+!5 = !{i32 7, !"uwtable", i32 1}
+!6 = !{i32 7, !"frame-pointer", i32 2}
+!7 = !{!"clang version 14.0.0"}
+!8 = distinct !DISubprogram(name: "not_removed", linkageName: "_Z11not_removedv", scope: !1, file: !1, line: 17, type: !9, scopeLine: 17, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !105)
+!9 = !DISubroutineType(types: !10)
+!10 = !{null}
+!11 = !{}
+!12 = !DILocalVariable(name: "a2", scope: !13, file: !1, line: 23, type: !14)
+!13 = distinct !DILexicalBlock(scope: !8, file: !1, line: 20, column: 3)
+!14 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A2", scope: !8, file: !1, line: 18, size: 32, flags: DIFlagTypePassByValue, elements: !15)
+!15 = !{!16}
+!16 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !14, file: !1, line: 18, baseType: !17, size: 32)
+!17 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!18 = !DILocation(line: 23, column: 8, scope: !13)
+!19 = !DILocation(line: 23, column: 15, scope: !13)
+!20 = !DILocation(line: 23, column: 16, scope: !13)
+!21 = !DILocalVariable(name: "i2", scope: !22, file: !1, line: 25, type: !23)
+!22 = distinct !DILexicalBlock(scope: !13, file: !1, line: 24, column: 5)
+!23 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "I2", scope: !13, file: !1, line: 21, size: 32, flags: DIFlagTypePassByValue, elements: !24)
+!24 = !{!25}
+!25 = !DIDerivedType(tag: DW_TAG_member, name: "j", scope: !23, file: !1, line: 21, baseType: !26, size: 32)
+!26 = !DIDerivedType(tag: DW_TAG_typedef, name: "Int2", scope: !8, file: !1, line: 19, baseType: !17)
+!27 = !DILocation(line: 25, column: 10, scope: !22)
+!28 = !DILocation(line: 25, column: 17, scope: !22)
+!29 = !DILocation(line: 25, column: 18, scope: !22)
+!30 = !DILocalVariable(name: "c2", scope: !22, file: !1, line: 26, type: !31)
+!31 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "C2", scope: !13, file: !1, line: 22, size: 8, flags: DIFlagTypePassByValue, elements: !32)
+!32 = !{!33}
+!33 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !31, file: !1, line: 22, baseType: !34, size: 8)
+!34 = !DIDerivedType(tag: DW_TAG_typedef, name: "Char2", scope: !31, file: !1, line: 22, baseType: !35)
+!35 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!36 = !DILocation(line: 26, column: 10, scope: !22)
+!37 = !DILocation(line: 26, column: 17, scope: !22)
+!38 = !DILocation(line: 26, column: 18, scope: !22)
+!39 = !DILocation(line: 29, column: 1, scope: !8)
+!40 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !1, file: !1, line: 31, type: !9, scopeLine: 31, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !107)
+!41 = !DILocalVariable(name: "a3", scope: !42, file: !1, line: 38, type: !44)
+!42 = distinct !DILexicalBlock(scope: !43, file: !1, line: 36, column: 5)
+!43 = distinct !DILexicalBlock(scope: !40, file: !1, line: 34, column: 3)
+!44 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A3", scope: !40, file: !1, line: 32, size: 32, flags: DIFlagTypePassByValue, elements: !45)
+!45 = !{!46}
+!46 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !44, file: !1, line: 32, baseType: !17, size: 32)
+!47 = !DILocation(line: 38, column: 10, scope: !42)
+!48 = !DILocation(line: 38, column: 17, scope: !42)
+!49 = !DILocation(line: 38, column: 18, scope: !42)
+!50 = !DILocalVariable(name: "i3", scope: !51, file: !1, line: 40, type: !52)
+!51 = distinct !DILexicalBlock(scope: !42, file: !1, line: 39, column: 7)
+!52 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "I3", scope: !43, file: !1, line: 35, size: 32, flags: DIFlagTypePassByValue, elements: !53)
+!53 = !{!54}
+!54 = !DIDerivedType(tag: DW_TAG_member, name: "j", scope: !52, file: !1, line: 35, baseType: !55, size: 32)
+!55 = !DIDerivedType(tag: DW_TAG_typedef, name: "Int3", scope: !40, file: !1, line: 33, baseType: !17)
+!56 = !DILocation(line: 40, column: 12, scope: !51)
+!57 = !DILocation(line: 40, column: 19, scope: !51)
+!58 = !DILocation(line: 40, column: 20, scope: !51)
+!59 = !DILocalVariable(name: "c3", scope: !51, file: !1, line: 41, type: !60)
+!60 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "C3", scope: !42, file: !1, line: 37, size: 8, flags: DIFlagTypePassByValue, elements: !61)
+!61 = !{!62}
+!62 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !60, file: !1, line: 37, baseType: !63, size: 8)
+!63 = !DIDerivedType(tag: DW_TAG_typedef, name: "Char3", scope: !60, file: !1, line: 37, baseType: !35)
+!64 = !DILocation(line: 41, column: 12, scope: !51)
+!65 = !DILocation(line: 41, column: 19, scope: !51)
+!66 = !DILocation(line: 41, column: 20, scope: !51)
+!67 = !DILocalVariable(name: "a1", scope: !68, file: !1, line: 8, type: !70)
+!68 = distinct !DILexicalBlock(scope: !69, file: !1, line: 5, column: 3)
+!69 = distinct !DISubprogram(name: "removed", linkageName: "_Z7removedv", scope: !1, file: !1, line: 2, type: !9, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !110)
+!70 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A1", scope: !69, file: !1, line: 3, size: 32, flags: DIFlagTypePassByValue, elements: !71, identifier: "_ZTSZ7removedvE2A1")
+!71 = !{!72}
+!72 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !70, file: !1, line: 3, baseType: !17, size: 32)
+!73 = !DILocation(line: 8, column: 8, scope: !68, inlinedAt: !74)
+!74 = distinct !DILocation(line: 45, column: 3, scope: !40)
+!75 = !DILocation(line: 8, column: 15, scope: !68, inlinedAt: !74)
+!76 = !DILocation(line: 8, column: 16, scope: !68, inlinedAt: !74)
+!77 = !DILocalVariable(name: "i1", scope: !78, file: !1, line: 10, type: !79)
+!78 = distinct !DILexicalBlock(scope: !68, file: !1, line: 9, column: 5)
+!79 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "I1", scope: !68, file: !1, line: 6, size: 32, flags: DIFlagTypePassByValue, elements: !80, identifier: "_ZTSZ7removedvE2I1")
+!80 = !{!81}
+!81 = !DIDerivedType(tag: DW_TAG_member, name: "j", scope: !79, file: !1, line: 6, baseType: !82, size: 32)
+!82 = !DIDerivedType(tag: DW_TAG_typedef, name: "Int1", scope: !69, file: !1, line: 4, baseType: !17)
+!83 = !DILocation(line: 10, column: 10, scope: !78, inlinedAt: !74)
+!84 = !DILocation(line: 10, column: 17, scope: !78, inlinedAt: !74)
+!85 = !DILocation(line: 10, column: 18, scope: !78, inlinedAt: !74)
+!86 = !DILocalVariable(name: "c1", scope: !78, file: !1, line: 11, type: !87)
+!87 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "C1", scope: !68, file: !1, line: 7, size: 8, flags: DIFlagTypePassByValue, elements: !88, identifier: "_ZTSZ7removedvE2C1")
+!88 = !{!89}
+!89 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !87, file: !1, line: 7, baseType: !90, size: 8)
+!90 = !DIDerivedType(tag: DW_TAG_typedef, name: "Char1", scope: !87, file: !1, line: 7, baseType: !35)
+!91 = !DILocation(line: 11, column: 10, scope: !78, inlinedAt: !74)
+!92 = !DILocation(line: 11, column: 17, scope: !78, inlinedAt: !74)
+!93 = !DILocation(line: 11, column: 18, scope: !78, inlinedAt: !74)
+!94 = !DILocation(line: 23, column: 8, scope: !13, inlinedAt: !95)
+!95 = distinct !DILocation(line: 46, column: 3, scope: !40)
+!96 = !DILocation(line: 23, column: 15, scope: !13, inlinedAt: !95)
+!97 = !DILocation(line: 23, column: 16, scope: !13, inlinedAt: !95)
+!98 = !DILocation(line: 25, column: 10, scope: !22, inlinedAt: !95)
+!99 = !DILocation(line: 25, column: 17, scope: !22, inlinedAt: !95)
+!100 = !DILocation(line: 25, column: 18, scope: !22, inlinedAt: !95)
+!101 = !DILocation(line: 26, column: 10, scope: !22, inlinedAt: !95)
+!102 = !DILocation(line: 26, column: 17, scope: !22, inlinedAt: !95)
+!103 = !DILocation(line: 26, column: 18, scope: !22, inlinedAt: !95)
+!104 = !DILocation(line: 47, column: 1, scope: !40)
+!105 = !{!14, !23, !26, !31}
+!107 = !{!44, !52, !55, !60}
+!110 = !{!70, !79, !82, !87}
diff --git a/llvm/test/DebugInfo/Generic/local-type-as-template-parameter.ll b/llvm/test/DebugInfo/Generic/local-type-as-template-parameter.ll
new file mode 100644
index 0000000000000..3fc9af7ccc0f7
--- /dev/null
+++ b/llvm/test/DebugInfo/Generic/local-type-as-template-parameter.ll
@@ -0,0 +1,161 @@
+; REQUIERES: system-linux
+; RUN: %llc_dwarf -O0 -filetype=obj < %s              \
+; RUN:  | llvm-dwarfdump --show-children --name=foo - \
+; RUN:  | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s
+
+; The test ensures that AsmPrinter doesn't crashed compiling this.
+; It also demostrates misplacement for a local type (see PR55680 for details).
+
+; The test compiled from:
+
+; template<typename T>
+; struct A {
+;   A(T &in) : a(in) {}
+;   T a;
+; };
+;
+; __attribute__((always_inline))
+; void foo() {
+;   struct B { int i; };
+;   B objB;
+;   A<B> objA(objB);
+; }
+;
+; int main() {
+;   foo();
+; }
+
+; Concrete out-of-line tree of foo().
+; CHECK: DW_TAG_subprogram
+; CHECK:   DW_AT_abstract_origin {{.*}} "_Z3foov"
+
+; FIXME: 'struct B' should be in the abstract tree below, not here.
+; CHECK:   DW_TAG_structure_type
+; CHECK:     DW_AT_name	("B")
+; CHECK:     DW_TAG_member
+; CHECK:     NULL
+;
+; CHECK:   DW_TAG_variable
+; CHECK:     DW_AT_abstract_origin {{.*}} "objB"
+; CHECK:   DW_TAG_variable
+; CHECK:     DW_AT_abstract_origin {{.*}} "objA"
+
+; CHECK:   NULL
+
+; Abstract tree of foo().
+; CHECK: DW_TAG_subprogram
+; CHECK:   DW_AT_name	("foo")
+; CHECK:   DW_AT_inline	(DW_INL_inlined)
+
+; CHECK:   DW_TAG_variable
+; CHECK:     DW_AT_name	("objB")
+; CHECK:   DW_TAG_variable
+; CHECK:     DW_AT_name	("objA")
+
+; CHECK:   NULL
+
+; CHECK: DW_TAG_inlined_subroutine
+; CHECK:   DW_AT_abstract_origin {{.*}} "_Z3foov"
+; CHECK:   DW_TAG_variable
+; CHECK:     DW_AT_abstract_origin {{.*}} "objB"
+; CHECK:   DW_TAG_variable
+; CHECK:     DW_AT_abstract_origin {{.*}} "objA"
+; CHECK:   NULL
+
+%struct.B = type { i32 }
+%struct.A = type { %struct.B }
+
+define dso_local void @_Z3foov() !dbg !7 {
+entry:
+  %objB = alloca %struct.B, align 4
+  %objA = alloca %struct.A, align 4
+  call void @llvm.dbg.declare(metadata ptr %objB, metadata !30, metadata !DIExpression()), !dbg !31
+  call void @llvm.dbg.declare(metadata ptr %objA, metadata !32, metadata !DIExpression()), !dbg !33
+  call void @_ZN1AIZ3foovE1BEC2ERS0_(ptr noundef nonnull align 4 dereferenceable(4) %objA, ptr noundef nonnull align 4 dereferenceable(4) %objB), !dbg !33
+  ret void, !dbg !34
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+define internal void @_ZN1AIZ3foovE1BEC2ERS0_(ptr noundef nonnull align 4 dereferenceable(4) %this, ptr noundef nonnull align 4 dereferenceable(4) %in) unnamed_addr align 2 !dbg !35 {
+entry:
+  %this.addr = alloca ptr, align 8
+  %in.addr = alloca ptr, align 8
+  store ptr %this, ptr %this.addr, align 8
+  call void @llvm.dbg.declare(metadata ptr %this.addr, metadata !36, metadata !DIExpression()), !dbg !38
+  store ptr %in, ptr %in.addr, align 8
+  call void @llvm.dbg.declare(metadata ptr %in.addr, metadata !39, metadata !DIExpression()), !dbg !40
+  %this1 = load ptr, ptr %this.addr, align 8
+  %a = getelementptr inbounds %struct.A, ptr %this1, i32 0, i32 0, !dbg !41
+  %0 = load ptr, ptr %in.addr, align 8, !dbg !42
+  call void @llvm.memcpy.p0.p0.i64(ptr align 4 %a, ptr align 4 %0, i64 4, i1 false), !dbg !41
+  ret void, !dbg !43
+}
+
+define dso_local noundef i32 @main() !dbg !44 {
+entry:
+  %objB.i = alloca %struct.B, align 4
+  %objA.i = alloca %struct.A, align 4
+  call void @llvm.dbg.declare(metadata ptr %objB.i, metadata !30, metadata !DIExpression()), !dbg !47
+  call void @llvm.dbg.declare(metadata ptr %objA.i, metadata !32, metadata !DIExpression()), !dbg !49
+  call void @_ZN1AIZ3foovE1BEC2ERS0_(ptr noundef nonnull align 4 dereferenceable(4) %objA.i, ptr noundef nonnull align 4 dereferenceable(4) %objB.i), !dbg !49
+  ret i32 0, !dbg !50
+}
+
+declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!22, !23, !24, !25, !26, !27, !28}
+!llvm.ident = !{!29}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 15.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, retainedTypes: !2, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "test.cpp", directory: "/", checksumkind: CSK_MD5, checksum: "aec7fd397e86f8655ef7f4bb4233b849")
+!2 = !{!3}
+!3 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A<B>", file: !1, line: 2, size: 32, flags: DIFlagTypePassByValue | DIFlagNonTrivial, elements: !4, templateParams: !20)
+!4 = !{!5, !15}
+!5 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !3, file: !1, line: 4, baseType: !6, size: 32)
+!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "B", scope: !7, file: !1, line: 9, size: 32, flags: DIFlagTypePassByValue, elements: !12)
+!7 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !1, file: !1, line: 8, type: !8, scopeLine: 8, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{null}
+!10 = !{}
+!11 = !{!6}
+!12 = !{!13}
+!13 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !6, file: !1, line: 9, baseType: !14, size: 32)
+!14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!15 = !DISubprogram(name: "A", scope: !3, file: !1, line: 3, type: !16, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagLocalToUnit)
+!16 = !DISubroutineType(types: !17)
+!17 = !{null, !18, !19}
+!18 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !3, size: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
+!19 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !6, size: 64)
+!20 = !{!21}
+!21 = !DITemplateTypeParameter(name: "T", type: !6)
+!22 = !{i32 7, !"Dwarf Version", i32 5}
+!23 = !{i32 2, !"Debug Info Version", i32 3}
+!24 = !{i32 1, !"wchar_size", i32 4}
+!25 = !{i32 7, !"PIC Level", i32 2}
+!26 = !{i32 7, !"PIE Level", i32 2}
+!27 = !{i32 7, !"uwtable", i32 2}
+!28 = !{i32 7, !"frame-pointer", i32 2}
+!29 = !{!"clang version 15.0.0"}
+!30 = !DILocalVariable(name: "objB", scope: !7, file: !1, line: 10, type: !6)
+!31 = !DILocation(line: 10, column: 5, scope: !7)
+!32 = !DILocalVariable(name: "objA", scope: !7, file: !1, line: 11, type: !3)
+!33 = !DILocation(line: 11, column: 8, scope: !7)
+!34 = !DILocation(line: 12, column: 1, scope: !7)
+!35 = distinct !DISubprogram(name: "A", linkageName: "_ZN1AIZ3foovE1BEC2ERS0_", scope: !3, file: !1, line: 3, type: !16, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !0, declaration: !15, retainedNodes: !10)
+!36 = !DILocalVariable(name: "this", arg: 1, scope: !35, type: !37, flags: DIFlagArtificial | DIFlagObjectPointer)
+!37 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !3, size: 64)
+!38 = !DILocation(line: 0, scope: !35)
+!39 = !DILocalVariable(name: "in", arg: 2, scope: !35, file: !1, line: 3, type: !19)
+!40 = !DILocation(line: 3, column: 8, scope: !35)
+!41 = !DILocation(line: 3, column: 14, scope: !35)
+!42 = !DILocation(line: 3, column: 16, scope: !35)
+!43 = !DILocation(line: 3, column: 21, scope: !35)
+!44 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 14, type: !45, scopeLine: 14, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !10)
+!45 = !DISubroutineType(types: !46)
+!46 = !{!14}
+!47 = !DILocation(line: 10, column: 5, scope: !7, inlinedAt: !48)
+!48 = distinct !DILocation(line: 15, column: 3, scope: !44)
+!49 = !DILocation(line: 11, column: 8, scope: !7, inlinedAt: !48)
+!50 = !DILocation(line: 16, column: 1, scope: !44)
diff --git a/llvm/test/DebugInfo/Generic/verifier-invalid-disubprogram.ll b/llvm/test/DebugInfo/Generic/verifier-invalid-disubprogram.ll
index 6d4d0e93d38f9..54ce1c56c6b30 100644
--- a/llvm/test/DebugInfo/Generic/verifier-invalid-disubprogram.ll
+++ b/llvm/test/DebugInfo/Generic/verifier-invalid-disubprogram.ll
@@ -38,7 +38,7 @@ define void @invalid_subprogram_declaration() !dbg !9 { ret void }
 define void @invalid_retained_nodes_list() !dbg !10 { ret void }
 !10 = distinct !DISubprogram(retainedNodes: !0)
 
-; CHECK: invalid retained nodes, expected DILocalVariable, DILabel or DIImportedEntity
+; CHECK: invalid retained nodes, expected DILocalVariable, DILabel, DIImportedEntity or DIType
 define void @invalid_retained_nodes_expected() !dbg !11 { ret void }
 !11 = distinct !DISubprogram(retainedNodes: !{!0})
 
diff --git a/llvm/test/DebugInfo/X86/set.ll b/llvm/test/DebugInfo/X86/set.ll
index 292c7c6e4a577..7dbec4520ed6c 100644
--- a/llvm/test/DebugInfo/X86/set.ll
+++ b/llvm/test/DebugInfo/X86/set.ll
@@ -68,11 +68,11 @@ attributes #1 = { nofree nosync nounwind readnone speculatable willreturn }
 !llvm.module.flags = !{!18, !19, !20}
 
 !0 = !{!"versions- cm3: d5.10.0 llvm: 9.0"}
-!1 = distinct !DICompileUnit(language: DW_LANG_Modula3, file: !2, producer: "cm3", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !3)
+!1 = distinct !DICompileUnit(language: DW_LANG_Modula3, file: !2, producer: "cm3", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug)
 !2 = !DIFile(filename: "Main.m3", directory: "/home/cm3/settest/src")
 !3 = !{!4}
 !4 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum", scope: !5, file: !2, line: 11, size: 8, align: 8, elements: !9)
-!5 = distinct !DISubprogram(name: "Test", linkageName: "Main__Test", scope: !2, file: !2, line: 11, type: !6, scopeLine: 11, spFlags: DISPFlagDefinition, unit: !1, retainedNodes: !8)
+!5 = distinct !DISubprogram(name: "Test", linkageName: "Main__Test", scope: !2, file: !2, line: 11, type: !6, scopeLine: 11, spFlags: DISPFlagDefinition, unit: !1, retainedNodes: !3)
 !6 = !DISubroutineType(types: !7)
 !7 = !{null}
 !8 = !{}
diff --git a/llvm/test/DebugInfo/Generic/split-dwarf-local-import.ll b/llvm/test/DebugInfo/X86/split-dwarf-local-import.ll
similarity index 98%
rename from llvm/test/DebugInfo/Generic/split-dwarf-local-import.ll
rename to llvm/test/DebugInfo/X86/split-dwarf-local-import.ll
index 1a41f3f348ca0..676ef23b679e8 100644
--- a/llvm/test/DebugInfo/Generic/split-dwarf-local-import.ll
+++ b/llvm/test/DebugInfo/X86/split-dwarf-local-import.ll
@@ -1,4 +1,3 @@
-; REQUIRES: x86-registered-target
 ; RUN: %llc_dwarf -O1 -filetype=obj -split-dwarf-file=%t.dwo < %s | llvm-dwarfdump -debug-info - | FileCheck %s --implicit-check-not "{{DW_TAG|NULL}}"
 
 ; CHECK-LABEL: debug_info contents
diff --git a/llvm/test/DebugInfo/Generic/split-dwarf-local-import2.ll b/llvm/test/DebugInfo/X86/split-dwarf-local-import2.ll
similarity index 98%
rename from llvm/test/DebugInfo/Generic/split-dwarf-local-import2.ll
rename to llvm/test/DebugInfo/X86/split-dwarf-local-import2.ll
index 0057f675f9b31..71f80992d1b04 100644
--- a/llvm/test/DebugInfo/Generic/split-dwarf-local-import2.ll
+++ b/llvm/test/DebugInfo/X86/split-dwarf-local-import2.ll
@@ -1,4 +1,3 @@
-; REQUIRES: x86-registered-target
 ; RUN: %llc_dwarf -split-dwarf-file=%t.dwo < %s | FileCheck %s
 
 ; Ensure function-local DW_TAG_imported_declaration get skipped if its parent subprogram was not emitted.
diff --git a/llvm/test/DebugInfo/Generic/split-dwarf-local-import3.ll b/llvm/test/DebugInfo/X86/split-dwarf-local-import3.ll
similarity index 100%
rename from llvm/test/DebugInfo/Generic/split-dwarf-local-import3.ll
rename to llvm/test/DebugInfo/X86/split-dwarf-local-import3.ll

From 21e1651c0ceb40eef4f1c3226c2e19a4ebe356c5 Mon Sep 17 00:00:00 2001
From: Siva Chandra Reddy <sivachandra@google.com>
Date: Fri, 16 Jun 2023 23:23:33 +0000
Subject: [PATCH 069/130] [libc] Remove the requirement of a platform-flush
 operation in File abstraction.

The libc flush operation is not supposed to trigger a platform level
flush operation. See "Notes" on this Linux man page:
    https://man7.org/linux/man-pages/man3/fflush.3.html

Reviewed By: michaelrj

Differential Revision: https://reviews.llvm.org/D153182
---
 libc/src/__support/File/file.cpp           |  1 -
 libc/src/__support/File/file.h             | 16 ++++++----------
 libc/src/__support/File/gpu/file.cpp       |  4 ++--
 libc/src/__support/File/linux/file.cpp     | 12 +-----------
 libc/src/stdio/fopencookie.cpp             |  8 ++------
 libc/test/src/__support/File/file_test.cpp |  3 +--
 libc/test/src/stdio/ftell_test.cpp         |  1 -
 7 files changed, 12 insertions(+), 33 deletions(-)

diff --git a/libc/src/__support/File/file.cpp b/libc/src/__support/File/file.cpp
index 326123d8b7482..65bfb585da7b1 100644
--- a/libc/src/__support/File/file.cpp
+++ b/libc/src/__support/File/file.cpp
@@ -335,7 +335,6 @@ int File::flush_unlocked() {
       return buf_result.error;
     }
     pos = 0;
-    return platform_flush(this);
   }
   // TODO: Add POSIX behavior for input streams.
   return 0;
diff --git a/libc/src/__support/File/file.h b/libc/src/__support/File/file.h
index 46bad008fcf7f..3a4bc89a7b311 100644
--- a/libc/src/__support/File/file.h
+++ b/libc/src/__support/File/file.h
@@ -56,7 +56,6 @@ class File {
   // file position indicator.
   using SeekFunc = ErrorOr<long>(File *, long, int);
   using CloseFunc = int(File *);
-  using FlushFunc = int(File *);
   // CleanupFunc is a function which does the equivalent of this:
   //
   // void my_file_cleanup(File *f) {
@@ -103,7 +102,6 @@ class File {
   ReadFunc *platform_read;
   SeekFunc *platform_seek;
   CloseFunc *platform_close;
-  FlushFunc *platform_flush;
   CleanupFunc *platform_cleanup;
 
   Mutex mutex;
@@ -202,15 +200,13 @@ class File {
   // is zero. This way, we will not have to employ the semantics of
   // the set_buffer method and allocate a buffer.
   constexpr File(WriteFunc *wf, ReadFunc *rf, SeekFunc *sf, CloseFunc *cf,
-                 FlushFunc *ff, CleanupFunc *clf, uint8_t *buffer,
-                 size_t buffer_size, int buffer_mode, bool owned,
-                 ModeFlags modeflags)
+                 CleanupFunc *clf, uint8_t *buffer, size_t buffer_size,
+                 int buffer_mode, bool owned, ModeFlags modeflags)
       : platform_write(wf), platform_read(rf), platform_seek(sf),
-        platform_close(cf), platform_flush(ff), platform_cleanup(clf),
-        mutex(false, false, false), ungetc_buf(0), buf(buffer),
-        bufsize(buffer_size), bufmode(buffer_mode), own_buf(owned),
-        mode(modeflags), pos(0), prev_op(FileOp::NONE), read_limit(0),
-        eof(false), err(false) {
+        platform_close(cf), platform_cleanup(clf), mutex(false, false, false),
+        ungetc_buf(0), buf(buffer), bufsize(buffer_size), bufmode(buffer_mode),
+        own_buf(owned), mode(modeflags), pos(0), prev_op(FileOp::NONE),
+        read_limit(0), eof(false), err(false) {
     if constexpr (ENABLE_BUFFER)
       adjust_buf();
   }
diff --git a/libc/src/__support/File/gpu/file.cpp b/libc/src/__support/File/gpu/file.cpp
index f31bcb53292cb..e2a0c1579ffa3 100644
--- a/libc/src/__support/File/gpu/file.cpp
+++ b/libc/src/__support/File/gpu/file.cpp
@@ -26,8 +26,8 @@ class GPUFile : public File {
 
 public:
   constexpr GPUFile(uintptr_t file, File::ModeFlags modeflags)
-      : File(&write_func, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,
-             0, _IONBF, false, modeflags),
+      : File(&write_func, nullptr, nullptr, nullptr, nullptr, nullptr, 0,
+             _IONBF, false, modeflags),
         file(file) {}
 
   uintptr_t get_file() const { return file; }
diff --git a/libc/src/__support/File/linux/file.cpp b/libc/src/__support/File/linux/file.cpp
index 7fb7cb587c86f..704bbd0d6035e 100644
--- a/libc/src/__support/File/linux/file.cpp
+++ b/libc/src/__support/File/linux/file.cpp
@@ -24,7 +24,6 @@ FileIOResult write_func(File *, const void *, size_t);
 FileIOResult read_func(File *, void *, size_t);
 ErrorOr<long> seek_func(File *, long, int);
 int close_func(File *);
-int flush_func(File *);
 
 } // anonymous namespace
 
@@ -34,7 +33,7 @@ class LinuxFile : public File {
 public:
   constexpr LinuxFile(int file_descriptor, uint8_t *buffer, size_t buffer_size,
                       int buffer_mode, bool owned, File::ModeFlags modeflags)
-      : File(&write_func, &read_func, &seek_func, &close_func, flush_func,
+      : File(&write_func, &read_func, &seek_func, &close_func,
              &cleanup_file<LinuxFile>, buffer, buffer_size, buffer_mode, owned,
              modeflags),
         fd(file_descriptor) {}
@@ -91,15 +90,6 @@ int close_func(File *f) {
   return 0;
 }
 
-int flush_func(File *f) {
-  auto *lf = reinterpret_cast<LinuxFile *>(f);
-  int ret = __llvm_libc::syscall_impl(SYS_fsync, lf->get_fd());
-  if (ret < 0) {
-    return -ret;
-  }
-  return 0;
-}
-
 } // anonymous namespace
 
 ErrorOr<File *> openfile(const char *path, const char *mode) {
diff --git a/libc/src/stdio/fopencookie.cpp b/libc/src/stdio/fopencookie.cpp
index da72ed3318105..4c8dbe2e796a7 100644
--- a/libc/src/stdio/fopencookie.cpp
+++ b/libc/src/stdio/fopencookie.cpp
@@ -26,15 +26,13 @@ class CookieFile : public __llvm_libc::File {
   static FileIOResult cookie_read(File *f, void *data, size_t size);
   static ErrorOr<long> cookie_seek(File *f, long offset, int whence);
   static int cookie_close(File *f);
-  static int cookie_flush(File *);
 
 public:
   CookieFile(void *c, cookie_io_functions_t cops, uint8_t *buffer,
              size_t bufsize, File::ModeFlags mode)
       : File(&cookie_write, &cookie_read, &CookieFile::cookie_seek,
-             &cookie_close, &cookie_flush, &cleanup_file<CookieFile>, buffer,
-             bufsize, 0 /* default buffering mode */,
-             true /* File owns buffer */, mode),
+             &cookie_close, &cleanup_file<CookieFile>, buffer, bufsize,
+             0 /* default buffering mode */, true /* File owns buffer */, mode),
         cookie(c), ops(cops) {}
 };
 
@@ -74,8 +72,6 @@ int CookieFile::cookie_close(File *f) {
   return cookie_file->ops.close(cookie_file->cookie);
 }
 
-int CookieFile::cookie_flush(File *) { return 0; }
-
 } // anonymous namespace
 
 LLVM_LIBC_FUNCTION(::FILE *, fopencookie,
diff --git a/libc/test/src/__support/File/file_test.cpp b/libc/test/src/__support/File/file_test.cpp
index c8cb1e3daea26..1446f66a01598 100644
--- a/libc/test/src/__support/File/file_test.cpp
+++ b/libc/test/src/__support/File/file_test.cpp
@@ -33,13 +33,12 @@ class StringFile : public File {
                                 size_t len);
   static ErrorOr<long> str_seek(__llvm_libc::File *f, long offset, int whence);
   static int str_close(__llvm_libc::File *f) { return 0; }
-  static int str_flush(__llvm_libc::File *f) { return 0; }
 
 public:
   explicit StringFile(char *buffer, size_t buflen, int bufmode, bool owned,
                       ModeFlags modeflags)
       : __llvm_libc::File(&str_write, &str_read, &str_seek, &str_close,
-                          &str_flush, &__llvm_libc::cleanup_file<StringFile>,
+                          &__llvm_libc::cleanup_file<StringFile>,
                           reinterpret_cast<uint8_t *>(buffer), buflen, bufmode,
                           owned, modeflags),
         pos(0), eof_marker(0), write_append(false) {
diff --git a/libc/test/src/stdio/ftell_test.cpp b/libc/test/src/stdio/ftell_test.cpp
index e5e4cc4aba4f1..fec881db4d888 100644
--- a/libc/test/src/stdio/ftell_test.cpp
+++ b/libc/test/src/stdio/ftell_test.cpp
@@ -7,7 +7,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "src/stdio/fclose.h"
-#include "src/stdio/fflush.h"
 #include "src/stdio/fopen.h"
 #include "src/stdio/fread.h"
 #include "src/stdio/fseek.h"

From 2e6bb8c9b82c185f0e96895b1f69045cfb0640b4 Mon Sep 17 00:00:00 2001
From: Scott Linder <Scott.Linder@amd.com>
Date: Mon, 19 Jun 2023 17:13:08 +0000
Subject: [PATCH 070/130] [DebugInfo] Support more than 2 operands in DWARF
 operations

Update DWARFExpression::Operation and LVOperation to support more than
2 operands.

Take the opportunity to use a SmallVector, which will handle at least 2
operands without allocation anyway, and removes the static limit
completely.

As there is no longer the concept of an "unused operand", remove
Operation::Encoding::SizeNA. Any use of it is now replaced with explicit
checks for how many operands an operation has.

There are still places where the limit remains 2, namely in the
DWARFLinker and in DIExpressions, but these can be updated in later
patches as-needed.

There are no explicit tests as this is nearly NFC: no new operation is
added which makes use of the additional operand capacity yet. A future
patch adding a new DWARF extension point will include operations which
require the support.

Reviewed By: Orlando, CarlosAlbertoEnciso

Differential Revision: https://reviews.llvm.org/D147270
---
 .../llvm/DebugInfo/DWARF/DWARFExpression.h    | 30 ++++++------
 .../DebugInfo/LogicalView/Core/LVLocation.h   | 17 ++-----
 .../DebugInfo/LogicalView/Core/LVReader.h     |  9 ++--
 .../DebugInfo/LogicalView/Core/LVSymbol.h     |  3 +-
 .../LogicalView/Readers/LVCodeViewReader.h    |  3 +-
 .../LogicalView/Readers/LVELFReader.h         |  3 +-
 llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp    |  4 +-
 llvm/lib/DWARFLinker/DWARFLinker.cpp          | 18 +++----
 llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp  | 48 ++++++++-----------
 .../DebugInfo/LogicalView/Core/LVLocation.cpp | 12 ++---
 .../DebugInfo/LogicalView/Core/LVSymbol.cpp   | 12 ++---
 .../LogicalView/Readers/LVCodeViewReader.cpp  |  2 +-
 .../LogicalView/Readers/LVCodeViewVisitor.cpp | 14 +++---
 .../LogicalView/Readers/LVELFReader.cpp       | 18 ++-----
 14 files changed, 83 insertions(+), 110 deletions(-)

diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h
index 7fde32711745d..6b80dbc038822 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h
@@ -24,8 +24,7 @@ class DWARFExpression {
 public:
   class iterator;
 
-  /// This class represents an Operation in the Expression. Each operation can
-  /// have up to 2 oprerands.
+  /// This class represents an Operation in the Expression.
   ///
   /// An Operation can be in Error state (check with isError()). This
   /// means that it couldn't be decoded successfully and if it is the
@@ -50,7 +49,6 @@ class DWARFExpression {
       SignedSize4 = SignBit | Size4,
       SignedSize8 = SignBit | Size8,
       SignedSizeLEB = SignBit | SizeLEB,
-      SizeNA = 0xFF ///< Unused operands get this encoding.
     };
 
     enum DwarfVersion : uint8_t {
@@ -64,14 +62,13 @@ class DWARFExpression {
     /// Description of the encoding of one expression Op.
     struct Description {
       DwarfVersion Version; ///< Dwarf version where the Op was introduced.
-      Encoding Op[2];       ///< Encoding for Op operands, or SizeNA.
-
-      Description(DwarfVersion Version = DwarfNA, Encoding Op1 = SizeNA,
-                  Encoding Op2 = SizeNA)
-          : Version(Version) {
-        Op[0] = Op1;
-        Op[1] = Op2;
-      }
+      SmallVector<Encoding> Op; ///< Encoding for Op operands.
+
+      template <typename... Ts>
+      Description(DwarfVersion Version, Ts... Op)
+          : Version(Version), Op{Op...} {}
+      Description() : Description(DwarfNA) {}
+      ~Description() = default;
     };
 
   private:
@@ -80,13 +77,18 @@ class DWARFExpression {
     Description Desc;
     bool Error = false;
     uint64_t EndOffset;
-    uint64_t Operands[2];
-    uint64_t OperandEndOffsets[2];
+    SmallVector<uint64_t> Operands;
+    SmallVector<uint64_t> OperandEndOffsets;
 
   public:
     const Description &getDescription() const { return Desc; }
     uint8_t getCode() const { return Opcode; }
+    uint64_t getNumOperands() const { return Operands.size(); }
+    ArrayRef<uint64_t> getRawOperands() const { return Operands; };
     uint64_t getRawOperand(unsigned Idx) const { return Operands[Idx]; }
+    ArrayRef<uint64_t> getOperandEndOffsets() const {
+      return OperandEndOffsets;
+    }
     uint64_t getOperandEndOffset(unsigned Idx) const {
       return OperandEndOffsets[Idx];
     }
@@ -165,7 +167,7 @@ class DWARFExpression {
 
   static bool prettyPrintRegisterOp(DWARFUnit *U, raw_ostream &OS,
                                     DIDumpOptions DumpOpts, uint8_t Opcode,
-                                    const uint64_t Operands[2]);
+                                    const ArrayRef<uint64_t> Operands);
 
 private:
   DataExtractor Data;
diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLocation.h b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLocation.h
index dc1df9f27a6c4..3b556f9927832 100644
--- a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLocation.h
+++ b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVLocation.h
@@ -33,22 +33,17 @@ class LVOperation final {
   //   OP_[GNU_]deref_type, OP_[GNU_]entry_value, OP_implicit_value,
   //   OP_[GNU_]implicit_pointer, OP_[GNU_]regval_type, OP_xderef_type.
   LVSmall Opcode = 0;
-  uint64_t Operands[2];
+  SmallVector<uint64_t> Operands;
 
 public:
   LVOperation() = delete;
-  LVOperation(LVSmall Opcode, LVUnsigned Operand1, LVUnsigned Operand2)
-      : Opcode(Opcode) {
-    Operands[0] = Operand1;
-    Operands[1] = Operand2;
-  }
+  LVOperation(LVSmall Opcode, ArrayRef<LVUnsigned> Operands)
+      : Opcode(Opcode), Operands(Operands) {}
   LVOperation(const LVOperation &) = delete;
   LVOperation &operator=(const LVOperation &) = delete;
   ~LVOperation() = default;
 
   LVSmall getOpcode() const { return Opcode; }
-  uint64_t getOperand1() const { return Operands[0]; }
-  uint64_t getOperand2() const { return Operands[1]; }
   std::string getOperandsDWARFInfo();
   std::string getOperandsCodeViewInfo();
 
@@ -154,8 +149,7 @@ class LVLocation : public LVObject {
 
   virtual void addObject(LVAddress LowPC, LVAddress HighPC,
                          LVUnsigned SectionOffset, uint64_t LocDescOffset) {}
-  virtual void addObject(LVSmall Opcode, LVUnsigned Operand1,
-                         LVUnsigned Operand2) {}
+  virtual void addObject(LVSmall Opcode, ArrayRef<LVUnsigned> Operands) {}
 
   static void print(LVLocations *Locations, raw_ostream &OS, bool Full = true);
   void printInterval(raw_ostream &OS, bool Full = true) const;
@@ -184,8 +178,7 @@ class LVLocationSymbol final : public LVLocation {
 
   void addObject(LVAddress LowPC, LVAddress HighPC, LVUnsigned SectionOffset,
                  uint64_t LocDescOffset) override;
-  void addObject(LVSmall Opcode, LVUnsigned Operand1,
-                 LVUnsigned Operand2) override;
+  void addObject(LVSmall Opcode, ArrayRef<LVUnsigned> Operands) override;
 
   void printRawExtra(raw_ostream &OS, bool Full = true) const override;
   void printExtra(raw_ostream &OS, bool Full = true) const override;
diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVReader.h b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVReader.h
index ea4c31af364ec..9ce26398e48df 100644
--- a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVReader.h
+++ b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVReader.h
@@ -228,10 +228,8 @@ class LVReader {
 #undef LV_CREATE_OBJECT
 
   // Operations creation.
-  LVOperation *createOperation(LVSmall OpCode, LVUnsigned Operand1,
-                               LVUnsigned Operand2) {
-    return new (AllocatedOperation.Allocate())
-        LVOperation(OpCode, Operand1, Operand2);
+  LVOperation *createOperation(LVSmall OpCode, ArrayRef<LVUnsigned> Operands) {
+    return new (AllocatedOperation.Allocate()) LVOperation(OpCode, Operands);
   }
 
   StringRef getFilename(LVObject *Object, size_t Index) const;
@@ -263,7 +261,8 @@ class LVReader {
   Error doPrint();
   Error doLoad();
 
-  virtual std::string getRegisterName(LVSmall Opcode, uint64_t Operands[2]) {
+  virtual std::string getRegisterName(LVSmall Opcode,
+                                      ArrayRef<uint64_t> Operands) {
     llvm_unreachable("Invalid instance reader.");
     return {};
   }
diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSymbol.h b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSymbol.h
index 3d4d65b2251f8..25bfa9eb77d8a 100644
--- a/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSymbol.h
+++ b/llvm/include/llvm/DebugInfo/LogicalView/Core/LVSymbol.h
@@ -126,8 +126,7 @@ class LVSymbol final : public LVElement {
   // Add a Location Entry.
   void addLocationConstant(dwarf::Attribute Attr, LVUnsigned Constant,
                            uint64_t LocDescOffset);
-  void addLocationOperands(LVSmall Opcode, uint64_t Operand1,
-                           uint64_t Operand2);
+  void addLocationOperands(LVSmall Opcode, ArrayRef<uint64_t> Operands);
   void addLocation(dwarf::Attribute Attr, LVAddress LowPC, LVAddress HighPC,
                    LVUnsigned SectionOffset, uint64_t LocDescOffset,
                    bool CallSiteLocation = false);
diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVCodeViewReader.h b/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVCodeViewReader.h
index 35d37b03c3bbc..8a32210bac3c9 100644
--- a/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVCodeViewReader.h
+++ b/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVCodeViewReader.h
@@ -215,7 +215,8 @@ class LVCodeViewReader final : public LVBinaryReader {
   static StringRef getSymbolKindName(SymbolKind Kind);
   static std::string formatRegisterId(RegisterId Register, CPUType CPU);
 
-  std::string getRegisterName(LVSmall Opcode, uint64_t Operands[2]) override;
+  std::string getRegisterName(LVSmall Opcode,
+                              ArrayRef<uint64_t> Operands) override;
 
   bool isSystemEntry(LVElement *Element, StringRef Name) const override;
 
diff --git a/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVELFReader.h b/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVELFReader.h
index 41d927c7e8219..0837b886a273a 100644
--- a/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVELFReader.h
+++ b/llvm/include/llvm/DebugInfo/LogicalView/Readers/LVELFReader.h
@@ -145,7 +145,8 @@ class LVELFReader final : public LVBinaryReader {
     return SymbolsWithLocations;
   }
 
-  std::string getRegisterName(LVSmall Opcode, uint64_t Operands[2]) override;
+  std::string getRegisterName(LVSmall Opcode,
+                              ArrayRef<uint64_t> Operands) override;
 
   void print(raw_ostream &OS) const;
 
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index 557291c97265f..f5a58f79ef4d5 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -2615,9 +2615,7 @@ void DwarfDebug::emitDebugLocEntry(ByteStreamer &Streamer,
            "3 operand ops not yet supported");
     Streamer.emitInt8(Op.getCode(), Comment != End ? *(Comment++) : "");
     Offset++;
-    for (unsigned I = 0; I < 2; ++I) {
-      if (Op.getDescription().Op[I] == Encoding::SizeNA)
-        continue;
+    for (unsigned I = 0; I < Op.getDescription().Op.size(); ++I) {
       if (Op.getDescription().Op[I] == Encoding::BaseTypeRef) {
         unsigned Length =
           Streamer.emitDIERef(*CU->ExprRefedBaseTypes[Op.getRawOperand(I)].Die);
diff --git a/llvm/lib/DWARFLinker/DWARFLinker.cpp b/llvm/lib/DWARFLinker/DWARFLinker.cpp
index 619df440b695e..2b1c6e5ada26d 100644
--- a/llvm/lib/DWARFLinker/DWARFLinker.cpp
+++ b/llvm/lib/DWARFLinker/DWARFLinker.cpp
@@ -1152,17 +1152,17 @@ void DWARFLinker::DIECloner::cloneExpression(
 
   uint64_t OpOffset = 0;
   for (auto &Op : Expression) {
-    auto Description = Op.getDescription();
+    auto Desc = Op.getDescription();
     // DW_OP_const_type is variable-length and has 3
-    // operands. DWARFExpression thus far only supports 2.
-    auto Op0 = Description.Op[0];
-    auto Op1 = Description.Op[1];
-    if ((Op0 == Encoding::BaseTypeRef && Op1 != Encoding::SizeNA) ||
-        (Op1 == Encoding::BaseTypeRef && Op0 != Encoding::Size1))
+    // operands. Thus far we only support 2.
+    if ((Desc.Op.size() == 2 && Desc.Op[0] == Encoding::BaseTypeRef) ||
+        (Desc.Op.size() == 2 && Desc.Op[1] == Encoding::BaseTypeRef &&
+         Desc.Op[0] != Encoding::Size1))
       Linker.reportWarning("Unsupported DW_OP encoding.", File);
 
-    if ((Op0 == Encoding::BaseTypeRef && Op1 == Encoding::SizeNA) ||
-        (Op1 == Encoding::BaseTypeRef && Op0 == Encoding::Size1)) {
+    if ((Desc.Op.size() == 1 && Desc.Op[0] == Encoding::BaseTypeRef) ||
+        (Desc.Op.size() == 2 && Desc.Op[1] == Encoding::BaseTypeRef &&
+         Desc.Op[0] == Encoding::Size1)) {
       // This code assumes that the other non-typeref operand fits into 1 byte.
       assert(OpOffset < Op.getEndOffset());
       uint32_t ULEBsize = Op.getEndOffset() - OpOffset - 1;
@@ -1171,7 +1171,7 @@ void DWARFLinker::DIECloner::cloneExpression(
       // Copy over the operation.
       OutputBuffer.push_back(Op.getCode());
       uint64_t RefOffset;
-      if (Op1 == Encoding::SizeNA) {
+      if (Desc.Op.size() == 1) {
         RefOffset = Op.getRawOperand(0);
       } else {
         OutputBuffer.push_back(Op.getRawOperand(0));
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp b/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp
index 109731e73c71d..7801170bc6e9e 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp
@@ -18,13 +18,11 @@ using namespace dwarf;
 
 namespace llvm {
 
-typedef std::vector<DWARFExpression::Operation::Description> DescVector;
-
-static DescVector getDescriptions() {
-  DescVector Descriptions;
-  typedef DWARFExpression::Operation Op;
-  typedef Op::Description Desc;
+typedef DWARFExpression::Operation Op;
+typedef Op::Description Desc;
 
+static std::vector<Desc> getOpDescriptions() {
+  std::vector<Desc> Descriptions;
   Descriptions.resize(0xff);
   Descriptions[DW_OP_addr] = Desc(Op::Dwarf2, Op::SizeAddr);
   Descriptions[DW_OP_deref] = Desc(Op::Dwarf2);
@@ -97,24 +95,25 @@ static DescVector getDescriptions() {
   Descriptions[DW_OP_GNU_addr_index] = Desc(Op::Dwarf4, Op::SizeLEB);
   Descriptions[DW_OP_GNU_const_index] = Desc(Op::Dwarf4, Op::SizeLEB);
   Descriptions[DW_OP_GNU_entry_value] = Desc(Op::Dwarf4, Op::SizeLEB);
-
   Descriptions[DW_OP_addrx] = Desc(Op::Dwarf5, Op::SizeLEB);
   Descriptions[DW_OP_constx] = Desc(Op::Dwarf5, Op::SizeLEB);
   Descriptions[DW_OP_convert] = Desc(Op::Dwarf5, Op::BaseTypeRef);
   Descriptions[DW_OP_entry_value] = Desc(Op::Dwarf5, Op::SizeLEB);
   Descriptions[DW_OP_regval_type] =
       Desc(Op::Dwarf5, Op::SizeLEB, Op::BaseTypeRef);
-
   return Descriptions;
 }
 
-static DWARFExpression::Operation::Description getOpDesc(unsigned OpCode) {
-  // FIXME: Make this constexpr once all compilers are smart enough to do it.
-  static DescVector Descriptions = getDescriptions();
+static Desc getDescImpl(ArrayRef<Desc> Descriptions, unsigned Opcode) {
   // Handle possible corrupted or unsupported operation.
-  if (OpCode >= Descriptions.size())
+  if (Opcode >= Descriptions.size())
     return {};
-  return Descriptions[OpCode];
+  return Descriptions[Opcode];
+}
+
+static Desc getOpDesc(unsigned Opcode) {
+  static std::vector<Desc> Descriptions = getOpDescriptions();
+  return getDescImpl(Descriptions, Opcode);
 }
 
 bool DWARFExpression::Operation::extract(DataExtractor Data,
@@ -127,13 +126,12 @@ bool DWARFExpression::Operation::extract(DataExtractor Data,
   if (Desc.Version == Operation::DwarfNA)
     return false;
 
-  for (unsigned Operand = 0; Operand < 2; ++Operand) {
+  Operands.resize(Desc.Op.size());
+  OperandEndOffsets.resize(Desc.Op.size());
+  for (unsigned Operand = 0; Operand < Desc.Op.size(); ++Operand) {
     unsigned Size = Desc.Op[Operand];
     unsigned Signed = Size & Operation::SignBit;
 
-    if (Size == Operation::SizeNA)
-      break;
-
     switch (Size & ~Operation::SignBit) {
     case Operation::Size1:
       Operands[Operand] = Data.getU8(&Offset);
@@ -208,9 +206,9 @@ bool DWARFExpression::Operation::extract(DataExtractor Data,
 
 static void prettyPrintBaseTypeRef(DWARFUnit *U, raw_ostream &OS,
                                    DIDumpOptions DumpOpts,
-                                   const uint64_t Operands[2],
+                                   ArrayRef<uint64_t> Operands,
                                    unsigned Operand) {
-  assert(Operand < 2 && "operand out of bounds");
+  assert(Operand < Operands.size() && "operand out of bounds");
   auto Die = U->getDIEForOffset(U->getOffset() + Operands[Operand]);
   if (Die && Die.getTag() == dwarf::DW_TAG_base_type) {
     OS << " (";
@@ -228,7 +226,7 @@ static void prettyPrintBaseTypeRef(DWARFUnit *U, raw_ostream &OS,
 bool DWARFExpression::prettyPrintRegisterOp(DWARFUnit *U, raw_ostream &OS,
                                             DIDumpOptions DumpOpts,
                                             uint8_t Opcode,
-                                            const uint64_t Operands[2]) {
+                                            ArrayRef<uint64_t> Operands) {
   if (!DumpOpts.GetNameForDWARFReg)
     return false;
 
@@ -278,13 +276,10 @@ bool DWARFExpression::Operation::print(raw_ostream &OS, DIDumpOptions DumpOpts,
     if (prettyPrintRegisterOp(U, OS, DumpOpts, Opcode, Operands))
       return true;
 
-  for (unsigned Operand = 0; Operand < 2; ++Operand) {
+  for (unsigned Operand = 0; Operand < Desc.Op.size(); ++Operand) {
     unsigned Size = Desc.Op[Operand];
     unsigned Signed = Size & Operation::SignBit;
 
-    if (Size == Operation::SizeNA)
-      break;
-
     if (Size == Operation::BaseTypeRef && U) {
       // For DW_OP_convert the operand may be 0 to indicate that conversion to
       // the generic type should be done. The same holds for DW_OP_reinterpret,
@@ -356,12 +351,9 @@ void DWARFExpression::print(raw_ostream &OS, DIDumpOptions DumpOpts,
 }
 
 bool DWARFExpression::Operation::verify(const Operation &Op, DWARFUnit *U) {
-  for (unsigned Operand = 0; Operand < 2; ++Operand) {
+  for (unsigned Operand = 0; Operand < Op.Desc.Op.size(); ++Operand) {
     unsigned Size = Op.Desc.Op[Operand];
 
-    if (Size == Operation::SizeNA)
-      break;
-
     if (Size == Operation::BaseTypeRef) {
       // For DW_OP_convert the operand may be 0 to indicate that conversion to
       // the generic type should be done, so don't look up a base type in that
diff --git a/llvm/lib/DebugInfo/LogicalView/Core/LVLocation.cpp b/llvm/lib/DebugInfo/LogicalView/Core/LVLocation.cpp
index 954ac0436b6ca..17b32a5f67b49 100644
--- a/llvm/lib/DebugInfo/LogicalView/Core/LVLocation.cpp
+++ b/llvm/lib/DebugInfo/LogicalView/Core/LVLocation.cpp
@@ -352,7 +352,7 @@ std::string LVOperation::getOperandsCodeViewInfo() {
   uint16_t OperationCode = getCodeViewOperationCode(Opcode);
 
   switch (OperationCode) {
-  // Operands: [Offset, 0].
+  // Operands: [Offset].
   case codeview::SymbolKind::S_DEFRANGE_FRAMEPOINTER_REL:
     Stream << "frame_pointer_rel " << int(Operands[0]);
     break;
@@ -360,7 +360,7 @@ std::string LVOperation::getOperandsCodeViewInfo() {
     Stream << "frame_pointer_rel_full_scope " << int(Operands[0]);
     break;
 
-  // Operands: [Register, 0].
+  // Operands: [Register].
   case codeview::SymbolKind::S_DEFRANGE_REGISTER:
     Stream << "register " << getReader().getRegisterName(Opcode, Operands);
     break;
@@ -375,7 +375,7 @@ std::string LVOperation::getOperandsCodeViewInfo() {
            << " offset " << int(Operands[1]);
     break;
 
-  // Operands: [Program, 0].
+  // Operands: [Program].
   case codeview::SymbolKind::S_DEFRANGE:
     Stream << "frame " << int(Operands[0]);
     break;
@@ -576,11 +576,11 @@ void LVLocationSymbol::addObject(LVAddress LowPC, LVAddress HighPC,
 }
 
 // Add a Location Record.
-void LVLocationSymbol::addObject(LVSmall Opcode, LVUnsigned Operand1,
-                                 LVUnsigned Operand2) {
+void LVLocationSymbol::addObject(LVSmall Opcode,
+                                 ArrayRef<LVUnsigned> Operands) {
   if (!Entries)
     Entries = std::make_unique<LVOperations>();
-  Entries->push_back(getReader().createOperation(Opcode, Operand1, Operand2));
+  Entries->push_back(getReader().createOperation(Opcode, Operands));
 }
 
 // Based on the DWARF attribute, define the location kind.
diff --git a/llvm/lib/DebugInfo/LogicalView/Core/LVSymbol.cpp b/llvm/lib/DebugInfo/LogicalView/Core/LVSymbol.cpp
index 7e86a2f4c1b8a..4608fe20cb6df 100644
--- a/llvm/lib/DebugInfo/LogicalView/Core/LVSymbol.cpp
+++ b/llvm/lib/DebugInfo/LogicalView/Core/LVSymbol.cpp
@@ -82,10 +82,10 @@ void LVSymbol::addLocation(dwarf::Attribute Attr, LVAddress LowPC,
 }
 
 // Add a Location Record.
-void LVSymbol::addLocationOperands(LVSmall Opcode, uint64_t Operand1,
-                                   uint64_t Operand2) {
+void LVSymbol::addLocationOperands(LVSmall Opcode,
+                                   ArrayRef<uint64_t> Operands) {
   if (CurrentLocation)
-    CurrentLocation->addObject(Opcode, Operand1, Operand2);
+    CurrentLocation->addObject(Opcode, Operands);
 }
 
 // Add a Location Entry.
@@ -97,8 +97,7 @@ void LVSymbol::addLocationConstant(dwarf::Attribute Attr, LVUnsigned Constant,
               /*SectionOffset=*/0, LocDescOffset);
 
   // Add records to Location Entry.
-  addLocationOperands(/*Opcode=*/LVLocationMemberOffset,
-                      /*Operand1=*/Constant, /*Operand2=*/0);
+  addLocationOperands(/*Opcode=*/LVLocationMemberOffset, {Constant});
 }
 
 LVLocations::iterator LVSymbol::addLocationGap(LVLocations::iterator Pos,
@@ -115,8 +114,7 @@ LVLocations::iterator LVSymbol::addLocationGap(LVLocations::iterator Pos,
   LVLocations::iterator Iter = Locations->insert(Pos, Gap);
 
   // Add gap to Location Entry.
-  Gap->addObject(/*op=*/dwarf::DW_OP_hi_user,
-                 /*opd1=*/0, /*opd2=*/0);
+  Gap->addObject(dwarf::DW_OP_hi_user, {});
 
   // Mark the entry as a gap.
   Gap->setIsGapEntry();
diff --git a/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewReader.cpp b/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewReader.cpp
index 5b099d47c45a4..d72fe2683f92c 100644
--- a/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewReader.cpp
+++ b/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewReader.cpp
@@ -1212,7 +1212,7 @@ Error LVCodeViewReader::loadTargetInfo(const PDBFile &Pdb) {
 }
 
 std::string LVCodeViewReader::getRegisterName(LVSmall Opcode,
-                                              uint64_t Operands[2]) {
+                                              ArrayRef<uint64_t> Operands) {
   // Get Compilation Unit CPU Type.
   CPUType CPU = getCompileUnitCPUType();
   // For CodeView the register always is in Operands[0];
diff --git a/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewVisitor.cpp b/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewVisitor.cpp
index c3ca6df91933f..e4f5f533262bf 100644
--- a/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewVisitor.cpp
+++ b/llvm/lib/DebugInfo/LogicalView/Readers/LVCodeViewVisitor.cpp
@@ -1065,7 +1065,7 @@ Error LVSymbolVisitor::visitKnownRecord(
 
     uint64_t Operand1 = DefRangeFramePointerRelFullScope.Offset;
     Symbol->addLocation(Attr, 0, 0, 0, 0);
-    Symbol->addLocationOperands(LVSmall(Attr), Operand1, /*Operand2=*/0);
+    Symbol->addLocationOperands(LVSmall(Attr), {Operand1});
   }
 
   return Error::success();
@@ -1103,7 +1103,7 @@ Error LVSymbolVisitor::visitKnownRecord(
         Reader->linearAddress(Range.ISectStart, Range.OffsetStart);
 
     Symbol->addLocation(Attr, Address, Address + Range.Range, 0, 0);
-    Symbol->addLocationOperands(LVSmall(Attr), Operand1, /*Operand2=*/0);
+    Symbol->addLocationOperands(LVSmall(Attr), {Operand1});
   }
 
   return Error::success();
@@ -1142,7 +1142,7 @@ Error LVSymbolVisitor::visitKnownRecord(
         Reader->linearAddress(Range.ISectStart, Range.OffsetStart);
 
     Symbol->addLocation(Attr, Address, Address + Range.Range, 0, 0);
-    Symbol->addLocationOperands(LVSmall(Attr), Operand1, Operand2);
+    Symbol->addLocationOperands(LVSmall(Attr), {Operand1, Operand2});
   }
 
   return Error::success();
@@ -1177,7 +1177,7 @@ Error LVSymbolVisitor::visitKnownRecord(CVSymbol &Record,
         Reader->linearAddress(Range.ISectStart, Range.OffsetStart);
 
     Symbol->addLocation(Attr, Address, Address + Range.Range, 0, 0);
-    Symbol->addLocationOperands(LVSmall(Attr), Operand1, /*Operand2=*/0);
+    Symbol->addLocationOperands(LVSmall(Attr), {Operand1});
   }
 
   return Error::success();
@@ -1215,7 +1215,7 @@ Error LVSymbolVisitor::visitKnownRecord(
         Reader->linearAddress(Range.ISectStart, Range.OffsetStart);
 
     Symbol->addLocation(Attr, Address, Address + Range.Range, 0, 0);
-    Symbol->addLocationOperands(LVSmall(Attr), Operand1, /*Operand2=*/0);
+    Symbol->addLocationOperands(LVSmall(Attr), {Operand1});
   }
 
   return Error::success();
@@ -1258,7 +1258,7 @@ Error LVSymbolVisitor::visitKnownRecord(CVSymbol &Record,
         Reader->linearAddress(Range.ISectStart, Range.OffsetStart);
 
     Symbol->addLocation(Attr, Address, Address + Range.Range, 0, 0);
-    Symbol->addLocationOperands(LVSmall(Attr), Operand1, /*Operand2=*/0);
+    Symbol->addLocationOperands(LVSmall(Attr), {Operand1, /*Operand2=*/0});
   }
 
   return Error::success();
@@ -1299,7 +1299,7 @@ Error LVSymbolVisitor::visitKnownRecord(CVSymbol &Record,
         Reader->linearAddress(Range.ISectStart, Range.OffsetStart);
 
     Symbol->addLocation(Attr, Address, Address + Range.Range, 0, 0);
-    Symbol->addLocationOperands(LVSmall(Attr), Operand1, /*Operand2=*/0);
+    Symbol->addLocationOperands(LVSmall(Attr), {Operand1, /*Operand2=*/0});
   }
 
   return Error::success();
diff --git a/llvm/lib/DebugInfo/LogicalView/Readers/LVELFReader.cpp b/llvm/lib/DebugInfo/LogicalView/Readers/LVELFReader.cpp
index 67002fb97153b..ab458341a0bd5 100644
--- a/llvm/lib/DebugInfo/LogicalView/Readers/LVELFReader.cpp
+++ b/llvm/lib/DebugInfo/LogicalView/Readers/LVELFReader.cpp
@@ -759,7 +759,8 @@ void LVELFReader::createLineAndFileRecords(
     }
 }
 
-std::string LVELFReader::getRegisterName(LVSmall Opcode, uint64_t Operands[2]) {
+std::string LVELFReader::getRegisterName(LVSmall Opcode,
+                                         ArrayRef<uint64_t> Operands) {
   // The 'prettyPrintRegisterOp' function uses the DWARFUnit to support
   // DW_OP_regval_type. At this point we are operating on a logical view
   // item, with no access to the underlying DWARF data used by LLVM.
@@ -973,19 +974,8 @@ void LVELFReader::processLocationList(dwarf::Attribute Attr,
                                       bool CallSiteLocation) {
 
   auto ProcessLocationExpression = [&](const DWARFExpression &Expression) {
-    // DW_OP_const_type is variable-length and has 3
-    // operands. DWARFExpression thus far only supports 2.
-    uint64_t Operands[2] = {0};
-    for (const DWARFExpression::Operation &Op : Expression) {
-      DWARFExpression::Operation::Description Description = Op.getDescription();
-      for (unsigned Operand = 0; Operand < 2; ++Operand) {
-        if (Description.Op[Operand] == DWARFExpression::Operation::SizeNA)
-          break;
-        Operands[Operand] = Op.getRawOperand(Operand);
-      }
-      CurrentSymbol->addLocationOperands(Op.getCode(), Operands[0],
-                                         Operands[1]);
-    }
+    for (const DWARFExpression::Operation &Op : Expression)
+      CurrentSymbol->addLocationOperands(Op.getCode(), Op.getRawOperands());
   };
 
   DWARFUnit *U = Die.getDwarfUnit();

From 407dcaf002f71abf940f3e903fa17791b150995d Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 19 Jun 2023 12:48:33 -0700
Subject: [PATCH 071/130] [Driver] Correct -fnoxray-link-deps to
 -fno-xray-link-deps

and removed unused CC1Option.
Also change -whole-archive to the canonical spelling and improve tests.
---
 clang/include/clang/Driver/Options.td      | 6 ++----
 clang/lib/Driver/ToolChains/CommonArgs.cpp | 4 ++--
 clang/lib/Driver/XRayArgs.cpp              | 2 +-
 clang/test/Driver/XRay/xray-nolinkdeps.cpp | 8 +++-----
 4 files changed, 8 insertions(+), 12 deletions(-)

diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 87df2266bd7e3..cf002e772df02 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -2266,10 +2266,8 @@ defm xray_function_index : BoolFOption<"xray-function-index",
           " expense of single-function patching performance">>;
 
 def fxray_link_deps : Flag<["-"], "fxray-link-deps">, Group<f_Group>,
-  Flags<[CC1Option]>,
-  HelpText<"Tells clang to add the link dependencies for XRay.">;
-def fnoxray_link_deps : Flag<["-"], "fnoxray-link-deps">, Group<f_Group>,
-  Flags<[CC1Option]>;
+  HelpText<"Link XRay runtime library when -fxray-instrument is specified (default)">;
+def fno_xray_link_deps : Flag<["-"], "fno-xray-link-deps">, Group<f_Group>;
 
 def fxray_instrumentation_bundle :
   Joined<["-"], "fxray-instrumentation-bundle=">,
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 165b1f9ae1300..2d9392893bc36 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -1236,11 +1236,11 @@ bool tools::addXRayRuntime(const ToolChain&TC, const ArgList &Args, ArgStringLis
     return false;
 
   if (TC.getXRayArgs().needsXRayRt()) {
-    CmdArgs.push_back("-whole-archive");
+    CmdArgs.push_back("--whole-archive");
     CmdArgs.push_back(TC.getCompilerRTArgString(Args, "xray"));
     for (const auto &Mode : TC.getXRayArgs().modeList())
       CmdArgs.push_back(TC.getCompilerRTArgString(Args, Mode));
-    CmdArgs.push_back("-no-whole-archive");
+    CmdArgs.push_back("--no-whole-archive");
     return true;
   }
 
diff --git a/clang/lib/Driver/XRayArgs.cpp b/clang/lib/Driver/XRayArgs.cpp
index f15a91f1aba44..9a4b28576a9b9 100644
--- a/clang/lib/Driver/XRayArgs.cpp
+++ b/clang/lib/Driver/XRayArgs.cpp
@@ -74,7 +74,7 @@ XRayArgs::XRayArgs(const ToolChain &TC, const ArgList &Args) {
         << XRayInstrument->getSpelling() << A->getSpelling();
 
   if (!Args.hasFlag(options::OPT_fxray_link_deps,
-                    options::OPT_fnoxray_link_deps, true))
+                    options::OPT_fno_xray_link_deps, true))
     XRayRT = false;
 
   auto Bundles =
diff --git a/clang/test/Driver/XRay/xray-nolinkdeps.cpp b/clang/test/Driver/XRay/xray-nolinkdeps.cpp
index 5461fc325a242..0c42c1af4f926 100644
--- a/clang/test/Driver/XRay/xray-nolinkdeps.cpp
+++ b/clang/test/Driver/XRay/xray-nolinkdeps.cpp
@@ -1,8 +1,6 @@
-// RUN: %clang -v -o /dev/null -fxray-instrument -fnoxray-link-deps %s -### \
+// RUN: %clang -### --target=x86_64-unknown-linux-gnu -fxray-instrument -fxray-link-deps -fno-xray-link-deps %s \
 // RUN:     2>&1 | FileCheck --check-prefix DISABLE %s
-// RUN: %clang -v -o /dev/null -fxray-instrument -fxray-link-deps %s -### \
+// RUN: %clang -### --target=x86_64-unknown-linux-gnu -fxray-instrument %s \
 // RUN:     2>&1 | FileCheck --check-prefix ENABLE %s
-// ENABLE: clang_rt.xray
+// ENABLE:      "--whole-archive" "{{.*}}clang_rt.xray{{.*}}"--no-whole-archive"
 // DISABLE-NOT: clang_rt.xray
-// REQUIRES: linux || freebsd
-// REQUIRES: amd64 || x86_64 || x86_64h || arm || aarch64 || arm64

From b3a08fa31766f4c88d9d7412414f615437fc4af4 Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Mon, 19 Jun 2023 21:08:35 +0100
Subject: [PATCH 072/130] [AMDGPU] Remove unused macro CNT_MASK

---
 llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 05247d3ff1603..3908d0dc08efa 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -57,8 +57,6 @@ namespace {
 // associated with the operand.  Used for determining whether
 // s_waitcnt instruction needs to be emitted.
 
-#define CNT_MASK(t) (1u << (t))
-
 enum InstCounterType { VM_CNT = 0, LGKM_CNT, EXP_CNT, VS_CNT, NUM_INST_CNTS };
 } // namespace
 

From 4b6d41cd1d73a72403679ec21732df3610fc0964 Mon Sep 17 00:00:00 2001
From: Jay Foad <jay.foad@amd.com>
Date: Mon, 19 Jun 2023 16:54:17 +0100
Subject: [PATCH 073/130] [AMDGPU] Do not release VGPRs if there may be pending
 scratch stores

Differential Revision: https://reviews.llvm.org/D153295
---
 llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp   |  69 +++++--
 .../GlobalISel/dynamic-alloca-uniform.ll      |   3 -
 .../GlobalISel/llvm.amdgcn.atomic.inc.ll      |  12 --
 .../GlobalISel/llvm.amdgcn.intersect_ray.ll   |   4 -
 .../GlobalISel/llvm.amdgcn.trig.preop.ll      |   2 -
 llvm/test/CodeGen/AMDGPU/amd.endpgm.ll        |   1 -
 llvm/test/CodeGen/AMDGPU/cc-update.ll         |   3 -
 llvm/test/CodeGen/AMDGPU/cluster_stores.ll    |   2 -
 .../CodeGen/AMDGPU/flat-scratch-i8-i16.ll     |  36 ----
 llvm/test/CodeGen/AMDGPU/flat-scratch-svs.ll  |  18 --
 llvm/test/CodeGen/AMDGPU/flat-scratch.ll      |   6 -
 .../CodeGen/AMDGPU/fp-min-max-flat-atomics.ll |   2 -
 .../CodeGen/AMDGPU/lds-atomic-fmin-fmax.ll    |   8 -
 .../AMDGPU/llvm.amdgcn.fdot2.bf16.bf16.ll     |   2 -
 .../AMDGPU/llvm.amdgcn.fdot2.f16.f16.ll       |   2 -
 .../AMDGPU/llvm.amdgcn.intersect_ray.ll       |   4 -
 .../AMDGPU/memory-legalizer-flat-agent.ll     | 120 ------------
 .../memory-legalizer-flat-nontemporal.ll      |   8 -
 .../memory-legalizer-flat-singlethread.ll     | 184 ------------------
 .../AMDGPU/memory-legalizer-flat-system.ll    | 120 ------------
 .../AMDGPU/memory-legalizer-flat-volatile.ll  |  12 --
 .../AMDGPU/memory-legalizer-flat-wavefront.ll | 182 -----------------
 .../AMDGPU/memory-legalizer-flat-workgroup.ll | 148 --------------
 .../memory-legalizer-private-nontemporal.ll   |   4 -
 .../memory-legalizer-private-volatile.ll      |   4 -
 llvm/test/CodeGen/AMDGPU/offset-split-flat.ll |  46 -----
 ...tack-pointer-offset-relative-frameindex.ll |   2 -
 .../AMDGPU/waitcnt-preexisting-vscnt.mir      |   5 -
 28 files changed, 52 insertions(+), 957 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index 3908d0dc08efa..ad317931ef0cd 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -86,19 +86,20 @@ struct RegisterEncoding {
 };
 
 enum WaitEventType {
-  VMEM_ACCESS,       // vector-memory read & write
-  VMEM_READ_ACCESS,  // vector-memory read
-  VMEM_WRITE_ACCESS, // vector-memory write
-  LDS_ACCESS,        // lds read & write
-  GDS_ACCESS,        // gds read & write
-  SQ_MESSAGE,        // send message
-  SMEM_ACCESS,       // scalar-memory read & write
-  EXP_GPR_LOCK,      // export holding on its data src
-  GDS_GPR_LOCK,      // GDS holding on its data and addr src
-  EXP_POS_ACCESS,    // write to export position
-  EXP_PARAM_ACCESS,  // write to export parameter
-  VMW_GPR_LOCK,      // vector-memory write holding on its data src
-  EXP_LDS_ACCESS,    // read by ldsdir counting as export
+  VMEM_ACCESS,          // vector-memory read & write
+  VMEM_READ_ACCESS,     // vector-memory read
+  VMEM_WRITE_ACCESS,    // vector-memory write that is not scratch
+  SCRATCH_WRITE_ACCESS, // vector-memory write that may be scratch
+  LDS_ACCESS,           // lds read & write
+  GDS_ACCESS,           // gds read & write
+  SQ_MESSAGE,           // send message
+  SMEM_ACCESS,          // scalar-memory read & write
+  EXP_GPR_LOCK,         // export holding on its data src
+  GDS_GPR_LOCK,         // GDS holding on its data and addr src
+  EXP_POS_ACCESS,       // write to export position
+  EXP_PARAM_ACCESS,     // write to export parameter
+  VMW_GPR_LOCK,         // vector-memory write holding on its data src
+  EXP_LDS_ACCESS,       // read by ldsdir counting as export
   NUM_WAIT_EVENTS,
 };
 
@@ -108,7 +109,7 @@ static const unsigned WaitEventMaskForInst[NUM_INST_CNTS] = {
         (1 << SQ_MESSAGE),
     (1 << EXP_GPR_LOCK) | (1 << GDS_GPR_LOCK) | (1 << VMW_GPR_LOCK) |
         (1 << EXP_PARAM_ACCESS) | (1 << EXP_POS_ACCESS) | (1 << EXP_LDS_ACCESS),
-    (1 << VMEM_WRITE_ACCESS)};
+    (1 << VMEM_WRITE_ACCESS) | (1 << SCRATCH_WRITE_ACCESS)};
 
 // The mapping is:
 //  0                .. SQ_MAX_PGM_VGPRS-1               real VGPRs
@@ -458,13 +459,19 @@ class SIInsertWaitcnts : public MachineFunctionPass {
     assert(SIInstrInfo::isVMEM(Inst) || SIInstrInfo::isFLAT(Inst));
     if (!ST->hasVscnt())
       return VMEM_ACCESS;
-    if (Inst.mayStore() && !SIInstrInfo::isAtomicRet(Inst))
+    if (Inst.mayStore() && !SIInstrInfo::isAtomicRet(Inst)) {
+      // FLAT and SCRATCH instructions may access scratch. Other VMEM
+      // instructions do not.
+      if (SIInstrInfo::isFLAT(Inst) && mayAccessScratchThroughFlat(Inst))
+        return SCRATCH_WRITE_ACCESS;
       return VMEM_WRITE_ACCESS;
+    }
     return VMEM_READ_ACCESS;
   }
 
   bool mayAccessVMEMThroughFlat(const MachineInstr &MI) const;
   bool mayAccessLDSThroughFlat(const MachineInstr &MI) const;
+  bool mayAccessScratchThroughFlat(const MachineInstr &MI) const;
   bool generateWaitcntInstBefore(MachineInstr &MI,
                                  WaitcntBrackets &ScoreBrackets,
                                  MachineInstr *OldWaitcntInstr,
@@ -1036,11 +1043,13 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(MachineInstr &MI,
   }
   // Identify S_ENDPGM instructions which may have to wait for outstanding VMEM
   // stores. In this case it can be useful to send a message to explicitly
-  // release all VGPRs before the stores have completed.
+  // release all VGPRs before the stores have completed, but it is only safe to
+  // do this if there are no outstanding scratch stores.
   else if (MI.getOpcode() == AMDGPU::S_ENDPGM ||
            MI.getOpcode() == AMDGPU::S_ENDPGM_SAVED) {
     if (ST->getGeneration() >= AMDGPUSubtarget::GFX11 &&
-        ScoreBrackets.getScoreRange(VS_CNT) != 0)
+        ScoreBrackets.getScoreRange(VS_CNT) != 0 &&
+        !ScoreBrackets.hasPendingEvent(SCRATCH_WRITE_ACCESS))
       ReleaseVGPRInsts.insert(&MI);
   }
   // Resolve vm waits before gs-done.
@@ -1396,6 +1405,32 @@ bool SIInsertWaitcnts::mayAccessLDSThroughFlat(const MachineInstr &MI) const {
   return false;
 }
 
+// This is a flat memory operation. Check to see if it has memory tokens for
+// either scratch or FLAT.
+bool SIInsertWaitcnts::mayAccessScratchThroughFlat(
+    const MachineInstr &MI) const {
+  assert(TII->isFLAT(MI));
+
+  // SCRATCH instructions always access scratch.
+  if (TII->isFLATScratch(MI))
+    return true;
+
+  // GLOBAL instructions never access scratch.
+  if (TII->isFLATGlobal(MI))
+    return false;
+
+  // If there are no memory operands then conservatively assume the flat
+  // operation may access scratch.
+  if (MI.memoperands_empty())
+    return true;
+
+  // See if any memory operand specifies an address space that involves scratch.
+  return any_of(MI.memoperands(), [](const MachineMemOperand *Memop) {
+    unsigned AS = Memop->getAddrSpace();
+    return AS == AMDGPUAS::PRIVATE_ADDRESS || AS == AMDGPUAS::FLAT_ADDRESS;
+  });
+}
+
 void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
                                                WaitcntBrackets *ScoreBrackets) {
   // Now look at the instruction opcode. If it is a memory access
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/dynamic-alloca-uniform.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/dynamic-alloca-uniform.ll
index 2e3b716750042..83608340e974e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/dynamic-alloca-uniform.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/dynamic-alloca-uniform.ll
@@ -54,7 +54,6 @@ define amdgpu_kernel void @kernel_dynamic_stackalloc_sgpr_align4(i32 %n) {
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_add_u32 s0, s32, s0
 ; GFX11-NEXT:    scratch_store_b32 off, v0, s0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %alloca = alloca i32, i32 %n, align 4, addrspace(5)
   store i32 0, ptr addrspace(5) %alloca
@@ -194,7 +193,6 @@ define amdgpu_kernel void @kernel_dynamic_stackalloc_sgpr_align16(i32 %n) {
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_add_u32 s0, s32, s0
 ; GFX11-NEXT:    scratch_store_b32 off, v0, s0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %alloca = alloca i32, i32 %n, align 16, addrspace(5)
   store i32 0, ptr addrspace(5) %alloca
@@ -337,7 +335,6 @@ define amdgpu_kernel void @kernel_dynamic_stackalloc_sgpr_align32(i32 %n) {
 ; GFX11-NEXT:    s_add_u32 s0, s32, s0
 ; GFX11-NEXT:    s_and_b32 s0, s0, 0xfffffc00
 ; GFX11-NEXT:    scratch_store_b32 off, v0, s0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %alloca = alloca i32, i32 %n, align 32, addrspace(5)
   store i32 0, ptr addrspace(5) %alloca
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.inc.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.inc.ll
index bd02a052f7b4c..17763b474a7fe 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.inc.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.atomic.inc.ll
@@ -1471,7 +1471,6 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32(ptr %out, ptr %ptr) #0 {
 ; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %result = call i32 @llvm.amdgcn.atomic.inc.i32.p0(ptr %ptr, i32 42, i32 0, i32 0, i1 false)
   store i32 %result, ptr %out
@@ -1551,7 +1550,6 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset(ptr %out, ptr %ptr) #0
 ; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr i32, ptr %ptr, i32 4
   %result = call i32 @llvm.amdgcn.atomic.inc.i32.p0(ptr %gep, i32 42, i32 0, i32 0, i1 false)
@@ -1577,7 +1575,6 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32(ptr %ptr) nounwind {
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-NEXT:    flat_atomic_inc_u32 v[0:1], v2
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %result = call i32 @llvm.amdgcn.atomic.inc.i32.p0(ptr %ptr, i32 42, i32 0, i32 0, i1 false)
   ret void
@@ -1637,7 +1634,6 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset(ptr %ptr) nounwind {
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-NEXT:    flat_atomic_inc_u32 v[0:1], v2 offset:16
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr i32, ptr %ptr, i32 4
   %result = call i32 @llvm.amdgcn.atomic.inc.i32.p0(ptr %gep, i32 42, i32 0, i32 0, i1 false)
@@ -1743,7 +1739,6 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i32_offset_addr64(ptr %out, ptr %
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v3
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i32, ptr %ptr, i32 %id
@@ -1824,7 +1819,6 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i32_offset_addr64(ptr %ptr) #0
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX11-NEXT:    v_mov_b32_e32 v2, 42
 ; GFX11-NEXT:    flat_atomic_inc_u32 v[0:1], v2 offset:20
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i32, ptr %ptr, i32 %id
@@ -2005,7 +1999,6 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64(ptr %out, ptr %ptr) #0 {
 ; GFX11-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    flat_store_b64 v[2:3], v[0:1]
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.atomic.inc.i64.p0(ptr %ptr, i64 42, i32 0, i32 0, i1 false)
   store i64 %result, ptr %out
@@ -2100,7 +2093,6 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset(ptr %out, ptr %ptr) #0
 ; GFX11-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    flat_store_b64 v[2:3], v[0:1]
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr i64, ptr %ptr, i32 4
   %result = call i64 @llvm.amdgcn.atomic.inc.i64.p0(ptr %gep, i64 42, i32 0, i32 0, i1 false)
@@ -2128,7 +2120,6 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64(ptr %ptr) nounwind {
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
 ; GFX11-NEXT:    flat_atomic_inc_u64 v[2:3], v[0:1]
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.atomic.inc.i64.p0(ptr %ptr, i64 42, i32 0, i32 0, i1 false)
   ret void
@@ -2193,7 +2184,6 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset(ptr %ptr) nounwind {
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
 ; GFX11-NEXT:    flat_atomic_inc_u64 v[2:3], v[0:1] offset:32
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr i64, ptr %ptr, i32 4
   %result = call i64 @llvm.amdgcn.atomic.inc.i64.p0(ptr %gep, i64 42, i32 0, i32 0, i1 false)
@@ -2311,7 +2301,6 @@ define amdgpu_kernel void @flat_atomic_inc_ret_i64_offset_addr64(ptr %out, ptr %
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, 0, v3, vcc_lo
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    flat_store_b64 v[2:3], v[0:1]
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i64, ptr %ptr, i32 %id
@@ -2397,7 +2386,6 @@ define amdgpu_kernel void @flat_atomic_inc_noret_i64_offset_addr64(ptr %ptr) #0
 ; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v0, v4
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX11-NEXT:    flat_atomic_inc_u64 v[0:1], v[2:3] offset:40
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %id = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.tid = getelementptr i64, ptr %ptr, i32 %id
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
index fc5f953e02cf0..d62454c8bd0b6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
@@ -698,7 +698,6 @@ define amdgpu_kernel void @image_bvh_intersect_ray_nsa_reassign(ptr %p_node_ptr,
 ; GFX11-NEXT:    image_bvh_intersect_ray v[0:3], [v9, v10, v[0:2], v[3:5], v[6:8]], s[4:7]
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b128 v[0:1], v[0:3]
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep_node_ptr = getelementptr inbounds i32, ptr %p_node_ptr, i32 %lid
@@ -802,7 +801,6 @@ define amdgpu_kernel void @image_bvh_intersect_ray_a16_nsa_reassign(ptr %p_node_
 ; GFX11-NEXT:    image_bvh_intersect_ray v[0:3], [v6, v7, v[0:2], v[3:5]], s[4:7] a16
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b128 v[0:1], v[0:3]
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep_node_ptr = getelementptr inbounds i32, ptr %p_node_ptr, i32 %lid
@@ -916,7 +914,6 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_nsa_reassign(ptr %p_ray, <4
 ; GFX11-NEXT:    image_bvh64_intersect_ray v[0:3], [v[9:10], v11, v[0:2], v[3:5], v[6:8]], s[0:3]
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b128 v[0:1], v[0:3]
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep_ray = getelementptr inbounds float, ptr %p_ray, i32 %lid
@@ -1016,7 +1013,6 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_a16_nsa_reassign(ptr %p_ray
 ; GFX11-NEXT:    image_bvh64_intersect_ray v[0:3], [v[6:7], v8, v[0:2], v[3:5]], s[0:3] a16
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b128 v[0:1], v[0:3]
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
   %gep_ray = getelementptr inbounds float, ptr %p_ray, i32 %lid
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.trig.preop.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.trig.preop.ll
index a5f5454cf6107..4754558695d59 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.trig.preop.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.trig.preop.ll
@@ -93,7 +93,6 @@ define amdgpu_kernel void @s_trig_preop_f64(double %a, i32 %b) {
 ; GFX11-NEXT:    v_trig_preop_f64 v[0:1], s[2:3], s0
 ; GFX11-NEXT:    flat_store_b64 v[0:1], v[0:1] dlc
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %result = call double @llvm.amdgcn.trig.preop.f64(double %a, i32 %b)
   store volatile double %result, ptr undef
@@ -126,7 +125,6 @@ define amdgpu_kernel void @s_trig_preop_f64_imm(double %a, i32 %b) {
 ; GFX11-NEXT:    v_trig_preop_f64 v[0:1], s[0:1], 7
 ; GFX11-NEXT:    flat_store_b64 v[0:1], v[0:1] dlc
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %result = call double @llvm.amdgcn.trig.preop.f64(double %a, i32 7)
   store volatile double %result, ptr undef
diff --git a/llvm/test/CodeGen/AMDGPU/amd.endpgm.ll b/llvm/test/CodeGen/AMDGPU/amd.endpgm.ll
index 5615a00629201..653fb8e95e7d9 100644
--- a/llvm/test/CodeGen/AMDGPU/amd.endpgm.ll
+++ b/llvm/test/CodeGen/AMDGPU/amd.endpgm.ll
@@ -87,7 +87,6 @@ define amdgpu_kernel void @test2(ptr %p, i32 %x) {
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 ; GFX11-NEXT:  .LBB2_2: ; %then
 ; GFX11-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/cc-update.ll b/llvm/test/CodeGen/AMDGPU/cc-update.ll
index fdb8bec944cb3..8a69069d75e2c 100644
--- a/llvm/test/CodeGen/AMDGPU/cc-update.ll
+++ b/llvm/test/CodeGen/AMDGPU/cc-update.ll
@@ -57,7 +57,6 @@ define amdgpu_kernel void @test_kern_stack() local_unnamed_addr #0 {
 ; GFX1100-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX1100-NEXT:    scratch_store_b32 off, v0, off offset:4 dlc
 ; GFX1100-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX1100-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX1100-NEXT:    s_endpgm
 entry:
   %x = alloca i32, align 4, addrspace(5)
@@ -309,7 +308,6 @@ define amdgpu_kernel void @test_force_fp_kern_stack() local_unnamed_addr #2 {
 ; GFX1100-NEXT:    s_mov_b32 s33, 0
 ; GFX1100-NEXT:    scratch_store_b32 off, v0, s33 offset:4 dlc
 ; GFX1100-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX1100-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX1100-NEXT:    s_endpgm
 entry:
   %x = alloca i32, align 4, addrspace(5)
@@ -585,7 +583,6 @@ define amdgpu_kernel void @test_sgpr_offset_kernel() #1 {
 ; GFX1100-NEXT:    s_waitcnt vmcnt(0)
 ; GFX1100-NEXT:    scratch_store_b32 off, v0, off offset:8 dlc
 ; GFX1100-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX1100-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX1100-NEXT:    s_endpgm
 entry:
   ; Occupy 4096 bytes of scratch, so the offset of the spill of %a does not
diff --git a/llvm/test/CodeGen/AMDGPU/cluster_stores.ll b/llvm/test/CodeGen/AMDGPU/cluster_stores.ll
index baccd757cd207..b2deff2623ae3 100644
--- a/llvm/test/CodeGen/AMDGPU/cluster_stores.ll
+++ b/llvm/test/CodeGen/AMDGPU/cluster_stores.ll
@@ -113,7 +113,6 @@ define amdgpu_kernel void @cluster_load_cluster_store(ptr noalias %lb, ptr noali
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v4 offset:16
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(3)
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v5 offset:24
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 bb:
   %ld0 = load i32, ptr %lb
@@ -243,7 +242,6 @@ define amdgpu_kernel void @cluster_load_valu_cluster_store(ptr noalias %lb, ptr
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v4 offset:16
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(3)
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v5 offset:24
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 bb:
   %ld0 = load i32, ptr %lb
diff --git a/llvm/test/CodeGen/AMDGPU/flat-scratch-i8-i16.ll b/llvm/test/CodeGen/AMDGPU/flat-scratch-i8-i16.ll
index 30b310a501f77..7a2bed327db05 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-scratch-i8-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-scratch-i8-i16.ll
@@ -23,7 +23,6 @@ define amdgpu_ps void @test_scratch_load_i8_zext_v(ptr addrspace(5) %in, ptr %ou
 ; GFX11-NEXT:    scratch_load_u8 v0, v0, off
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[1:2], v0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr inbounds i8, ptr addrspace(5) %in, i32 1
   %load = load i8, ptr addrspace(5) %gep, align 4
@@ -51,7 +50,6 @@ define amdgpu_ps void @test_scratch_load_i8_sext_v(ptr addrspace(5) %in, ptr %ou
 ; GFX11-NEXT:    scratch_load_i8 v0, v0, off
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[1:2], v0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr inbounds i8, ptr addrspace(5) %in, i32 1
   %load = load i8, ptr addrspace(5) %gep, align 4
@@ -79,7 +77,6 @@ define amdgpu_ps void @test_scratch_load_i16_zext_v(ptr addrspace(5) %in, ptr %o
 ; GFX11-NEXT:    scratch_load_u16 v0, v0, off
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[1:2], v0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr inbounds i16, ptr addrspace(5) %in, i32 1
   %load = load i16, ptr addrspace(5) %gep, align 4
@@ -107,7 +104,6 @@ define amdgpu_ps void @test_scratch_load_i16_sext_v(ptr addrspace(5) %in, ptr %o
 ; GFX11-NEXT:    scratch_load_i16 v0, v0, off
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[1:2], v0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr inbounds i16, ptr addrspace(5) %in, i32 1
   %load = load i16, ptr addrspace(5) %gep, align 4
@@ -136,7 +132,6 @@ define amdgpu_ps void @test_scratch_load_i8_zext_to_d16_lo_v(ptr addrspace(5) %i
 ; GFX11-NEXT:    scratch_load_d16_u8 v3, v0, off
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[1:2], v3
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 bb:
   %gep = getelementptr i8, ptr addrspace(5) %in, i64 1
@@ -167,7 +162,6 @@ define amdgpu_ps void @test_scratch_load_i8_sext_to_d16_lo_v(ptr addrspace(5) %i
 ; GFX11-NEXT:    scratch_load_d16_i8 v3, v0, off
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[1:2], v3
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 bb:
   %gep = getelementptr i8, ptr addrspace(5) %in, i64 1
@@ -198,7 +192,6 @@ define amdgpu_ps void @test_scratch_load_i16_to_d16_lo_v(ptr addrspace(5) %in, p
 ; GFX11-NEXT:    scratch_load_d16_b16 v3, v0, off
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[1:2], v3
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 bb:
   %gep = getelementptr i16, ptr addrspace(5) %in, i64 1
@@ -229,7 +222,6 @@ define amdgpu_ps void @test_scratch_load_i8_zext_to_d16_hi_v(ptr addrspace(5) %i
 ; GFX11-NEXT:    scratch_load_d16_hi_u8 v3, v0, off
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[1:2], v3
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 bb:
   %gep = getelementptr i8, ptr addrspace(5) %in, i64 1
@@ -260,7 +252,6 @@ define amdgpu_ps void @test_scratch_load_i8_sext_to_d16_hi_v(ptr addrspace(5) %i
 ; GFX11-NEXT:    scratch_load_d16_hi_i8 v3, v0, off
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[1:2], v3
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 bb:
   %gep = getelementptr i8, ptr addrspace(5) %in, i64 1
@@ -291,7 +282,6 @@ define amdgpu_ps void @test_scratch_load_i16_to_d16_hi_v(ptr addrspace(5) %in, p
 ; GFX11-NEXT:    scratch_load_d16_hi_b16 v3, v0, off
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[1:2], v3
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 bb:
   %gep = getelementptr i16, ptr addrspace(5) %in, i64 1
@@ -320,7 +310,6 @@ define amdgpu_ps void @test_scratch_store_b8_from_d16_hi_v(ptr %in, ptr addrspac
 ; GFX11-NEXT:    v_add_nc_u32_e32 v1, 4, v2
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    scratch_store_d16_hi_b8 v1, v0, off
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 bb:
   %load = load <4 x i8>, ptr %in
@@ -349,7 +338,6 @@ define amdgpu_ps void @test_scratch_store_b16_from_d16_hi_v(ptr %in, ptr addrspa
 ; GFX11-NEXT:    v_add_nc_u32_e32 v1, 2, v2
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    scratch_store_d16_hi_b16 v1, v0, off
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 bb:
   %load = load <2 x i16>, ptr %in
@@ -383,7 +371,6 @@ define amdgpu_ps void @test_scratch_load_i8_zext_s(ptr addrspace(5) inreg %in, p
 ; GFX11-NEXT:    scratch_load_u8 v2, off, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr inbounds i8, ptr addrspace(5) %in, i32 1
   %load = load i8, ptr addrspace(5) %gep, align 4
@@ -411,7 +398,6 @@ define amdgpu_ps void @test_scratch_load_i8_sext_s(ptr addrspace(5) inreg %in, p
 ; GFX11-NEXT:    scratch_load_i8 v2, off, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr inbounds i8, ptr addrspace(5) %in, i32 1
   %load = load i8, ptr addrspace(5) %gep, align 4
@@ -439,7 +425,6 @@ define amdgpu_ps void @test_scratch_load_i16_zext_s(ptr addrspace(5) inreg %in,
 ; GFX11-NEXT:    scratch_load_u16 v2, off, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr inbounds i16, ptr addrspace(5) %in, i32 1
   %load = load i16, ptr addrspace(5) %gep, align 4
@@ -467,7 +452,6 @@ define amdgpu_ps void @test_scratch_load_i16_sext_s(ptr addrspace(5) inreg %in,
 ; GFX11-NEXT:    scratch_load_i16 v2, off, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr inbounds i16, ptr addrspace(5) %in, i32 1
   %load = load i16, ptr addrspace(5) %gep, align 4
@@ -497,7 +481,6 @@ define amdgpu_ps void @test_scratch_load_i8_zext_to_d16_lo_s(ptr addrspace(5) in
 ; GFX11-NEXT:    scratch_load_d16_u8 v2, off, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 bb:
   %gep = getelementptr i8, ptr addrspace(5) %in, i64 1
@@ -529,7 +512,6 @@ define amdgpu_ps void @test_scratch_load_i8_sext_to_d16_lo_s(ptr addrspace(5) in
 ; GFX11-NEXT:    scratch_load_d16_i8 v2, off, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 bb:
   %gep = getelementptr i8, ptr addrspace(5) %in, i64 1
@@ -561,7 +543,6 @@ define amdgpu_ps void @test_scratch_load_i16_to_d16_lo_s(ptr addrspace(5) inreg
 ; GFX11-NEXT:    scratch_load_d16_b16 v2, off, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 bb:
   %gep = getelementptr i16, ptr addrspace(5) %in, i64 1
@@ -593,7 +574,6 @@ define amdgpu_ps void @test_scratch_load_i8_zext_to_d16_hi_s(ptr addrspace(5) in
 ; GFX11-NEXT:    scratch_load_d16_hi_u8 v2, off, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 bb:
   %gep = getelementptr i8, ptr addrspace(5) %in, i64 1
@@ -625,7 +605,6 @@ define amdgpu_ps void @test_scratch_load_i8_sext_to_d16_hi_s(ptr addrspace(5) in
 ; GFX11-NEXT:    scratch_load_d16_hi_i8 v2, off, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 bb:
   %gep = getelementptr i8, ptr addrspace(5) %in, i64 1
@@ -657,7 +636,6 @@ define amdgpu_ps void @test_scratch_load_i16_to_d16_hi_s(ptr addrspace(5) inreg
 ; GFX11-NEXT:    scratch_load_d16_hi_b16 v2, off, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 bb:
   %gep = getelementptr i16, ptr addrspace(5) %in, i64 1
@@ -686,7 +664,6 @@ define amdgpu_ps void @test_scratch_store_b8_from_d16_hi_s(ptr %in, ptr addrspac
 ; GFX11-NEXT:    s_add_i32 s0, s0, 4
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    scratch_store_d16_hi_b8 off, v0, s0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 bb:
   %load = load <4 x i8>, ptr %in
@@ -715,7 +692,6 @@ define amdgpu_ps void @test_scratch_store_b16_from_d16_hi_s(ptr %in, ptr addrspa
 ; GFX11-NEXT:    s_add_i32 s0, s0, 2
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    scratch_store_d16_hi_b16 off, v0, s0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 bb:
   %load = load <2 x i16>, ptr %in
@@ -752,7 +728,6 @@ define amdgpu_ps void @test_scratch_load_i8_zext_svs(ptr addrspace(5) inreg %in,
 ; GFX11-NEXT:    scratch_load_u8 v0, v0, off
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[1:2], v0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %voffset4 = mul i32 %voffset, 4
   %gep0 = getelementptr inbounds i8, ptr addrspace(5) %in, i32 %voffset4
@@ -785,7 +760,6 @@ define amdgpu_ps void @test_scratch_load_i8_sext_svs(ptr addrspace(5) inreg %in,
 ; GFX11-NEXT:    scratch_load_i8 v0, v0, off
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[1:2], v0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %voffset4 = mul i32 %voffset, 4
   %gep0 = getelementptr inbounds i8, ptr addrspace(5) %in, i32 %voffset4
@@ -818,7 +792,6 @@ define amdgpu_ps void @test_scratch_load_i16_zext_svs(ptr addrspace(5) inreg %in
 ; GFX11-NEXT:    scratch_load_u16 v0, v0, off
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[1:2], v0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %voffset4 = mul i32 %voffset, 4
   %gep0 = getelementptr inbounds i8, ptr addrspace(5) %in, i32 %voffset4
@@ -851,7 +824,6 @@ define amdgpu_ps void @test_scratch_load_i16_sext_svs(ptr addrspace(5) inreg %in
 ; GFX11-NEXT:    scratch_load_i16 v0, v0, off
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[1:2], v0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %voffset4 = mul i32 %voffset, 4
   %gep0 = getelementptr inbounds i8, ptr addrspace(5) %in, i32 %voffset4
@@ -885,7 +857,6 @@ define amdgpu_ps void @test_scratch_load_i8_zext_to_d16_lo_svs(ptr addrspace(5)
 ; GFX11-NEXT:    scratch_load_d16_u8 v3, v0, off
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[1:2], v3
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 bb:
   %voffset4 = mul i32 %voffset, 4
@@ -921,7 +892,6 @@ define amdgpu_ps void @test_scratch_load_i8_sext_to_d16_lo_svs(ptr addrspace(5)
 ; GFX11-NEXT:    scratch_load_d16_i8 v3, v0, off
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[1:2], v3
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 bb:
   %voffset4 = mul i32 %voffset, 4
@@ -957,7 +927,6 @@ define amdgpu_ps void @test_scratch_load_i16_to_d16_lo_svs(ptr addrspace(5) inre
 ; GFX11-NEXT:    scratch_load_d16_b16 v3, v0, off
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[1:2], v3
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 bb:
   %voffset4 = mul i32 %voffset, 4
@@ -993,7 +962,6 @@ define amdgpu_ps void @test_scratch_load_i8_zext_to_d16_hi_svs(ptr addrspace(5)
 ; GFX11-NEXT:    scratch_load_d16_hi_u8 v3, v0, off
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[1:2], v3
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 bb:
   %voffset4 = mul i32 %voffset, 4
@@ -1029,7 +997,6 @@ define amdgpu_ps void @test_scratch_load_i8_sext_to_d16_hi_svs(ptr addrspace(5)
 ; GFX11-NEXT:    scratch_load_d16_hi_i8 v3, v0, off
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[1:2], v3
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 bb:
   %voffset4 = mul i32 %voffset, 4
@@ -1065,7 +1032,6 @@ define amdgpu_ps void @test_scratch_load_i16_to_d16_hi_svs(ptr addrspace(5) inre
 ; GFX11-NEXT:    scratch_load_d16_hi_b16 v3, v0, off
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b32 v[1:2], v3
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 bb:
   %voffset4 = mul i32 %voffset, 4
@@ -1099,7 +1065,6 @@ define amdgpu_ps void @test_scratch_store_b8_from_d16_hi_svs(ptr %in, ptr addrsp
 ; GFX11-NEXT:    v_add3_u32 v1, s0, v1, 4
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    scratch_store_d16_hi_b8 v1, v0, off
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 bb:
   %load = load <4 x i8>, ptr %in
@@ -1133,7 +1098,6 @@ define amdgpu_ps void @test_scratch_store_b16_from_d16_hi_svs(ptr %in, ptr addrs
 ; GFX11-NEXT:    v_add3_u32 v1, s0, v1, 2
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    scratch_store_d16_hi_b16 v1, v0, off
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 bb:
   %load = load <2 x i16>, ptr %in
diff --git a/llvm/test/CodeGen/AMDGPU/flat-scratch-svs.ll b/llvm/test/CodeGen/AMDGPU/flat-scratch-svs.ll
index 74ba38cddff59..2e01c0064ef7c 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-scratch-svs.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-scratch-svs.ll
@@ -67,7 +67,6 @@ define amdgpu_kernel void @soff1_voff1(i32 %soff) {
 ; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-SDAG-NEXT:    scratch_store_b8 v0, v3, off dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX11-GISEL-LABEL: soff1_voff1:
@@ -86,7 +85,6 @@ define amdgpu_kernel void @soff1_voff1(i32 %soff) {
 ; GFX11-GISEL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-GISEL-NEXT:    scratch_store_b8 v0, v3, off dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
 bb:
   %soff1 = mul i32 %soff, 1
@@ -164,7 +162,6 @@ define amdgpu_kernel void @soff1_voff2(i32 %soff) {
 ; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-SDAG-NEXT:    scratch_store_b8 v0, v3, off dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX11-GISEL-LABEL: soff1_voff2:
@@ -184,7 +181,6 @@ define amdgpu_kernel void @soff1_voff2(i32 %soff) {
 ; GFX11-GISEL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-GISEL-NEXT:    scratch_store_b8 v0, v3, off dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
 bb:
   %soff1 = mul i32 %soff, 1
@@ -262,7 +258,6 @@ define amdgpu_kernel void @soff1_voff4(i32 %soff) {
 ; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-SDAG-NEXT:    scratch_store_b8 v0, v3, off dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX11-GISEL-LABEL: soff1_voff4:
@@ -282,7 +277,6 @@ define amdgpu_kernel void @soff1_voff4(i32 %soff) {
 ; GFX11-GISEL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-GISEL-NEXT:    scratch_store_b8 v0, v3, off dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
 bb:
   %soff1 = mul i32 %soff, 1
@@ -361,7 +355,6 @@ define amdgpu_kernel void @soff2_voff1(i32 %soff) {
 ; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-SDAG-NEXT:    scratch_store_b8 v0, v3, off dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX11-GISEL-LABEL: soff2_voff1:
@@ -381,7 +374,6 @@ define amdgpu_kernel void @soff2_voff1(i32 %soff) {
 ; GFX11-GISEL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-GISEL-NEXT:    scratch_store_b8 v0, v3, off dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
 bb:
   %soff2 = mul i32 %soff, 2
@@ -463,7 +455,6 @@ define amdgpu_kernel void @soff2_voff2(i32 %soff) {
 ; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-SDAG-NEXT:    scratch_store_b8 v0, v3, off dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX11-GISEL-LABEL: soff2_voff2:
@@ -485,7 +476,6 @@ define amdgpu_kernel void @soff2_voff2(i32 %soff) {
 ; GFX11-GISEL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-GISEL-NEXT:    scratch_store_b8 v0, v3, off dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
 bb:
   %soff2 = mul i32 %soff, 2
@@ -567,7 +557,6 @@ define amdgpu_kernel void @soff2_voff4(i32 %soff) {
 ; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-SDAG-NEXT:    scratch_store_b8 v0, v3, off dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX11-GISEL-LABEL: soff2_voff4:
@@ -589,7 +578,6 @@ define amdgpu_kernel void @soff2_voff4(i32 %soff) {
 ; GFX11-GISEL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-GISEL-NEXT:    scratch_store_b8 v0, v3, off dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
 bb:
   %soff2 = mul i32 %soff, 2
@@ -668,7 +656,6 @@ define amdgpu_kernel void @soff4_voff1(i32 %soff) {
 ; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-SDAG-NEXT:    scratch_store_b8 v0, v3, off dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX11-GISEL-LABEL: soff4_voff1:
@@ -688,7 +675,6 @@ define amdgpu_kernel void @soff4_voff1(i32 %soff) {
 ; GFX11-GISEL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-GISEL-NEXT:    scratch_store_b8 v0, v3, off dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
 bb:
   %soff4 = mul i32 %soff, 4
@@ -770,7 +756,6 @@ define amdgpu_kernel void @soff4_voff2(i32 %soff) {
 ; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-SDAG-NEXT:    scratch_store_b8 v0, v3, off dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX11-GISEL-LABEL: soff4_voff2:
@@ -792,7 +777,6 @@ define amdgpu_kernel void @soff4_voff2(i32 %soff) {
 ; GFX11-GISEL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-GISEL-NEXT:    scratch_store_b8 v0, v3, off dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
 bb:
   %soff4 = mul i32 %soff, 4
@@ -874,7 +858,6 @@ define amdgpu_kernel void @soff4_voff4(i32 %soff) {
 ; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-SDAG-NEXT:    scratch_store_b8 v0, v3, off dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX11-GISEL-LABEL: soff4_voff4:
@@ -896,7 +879,6 @@ define amdgpu_kernel void @soff4_voff4(i32 %soff) {
 ; GFX11-GISEL-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-GISEL-NEXT:    scratch_store_b8 v0, v3, off dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
 bb:
   %soff4 = mul i32 %soff, 4
diff --git a/llvm/test/CodeGen/AMDGPU/flat-scratch.ll b/llvm/test/CodeGen/AMDGPU/flat-scratch.ll
index 0da1119117a31..71140db0b3a80 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-scratch.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-scratch.ll
@@ -61,7 +61,6 @@ define amdgpu_kernel void @zero_init_kernel() {
 ; GFX11-NEXT:    scratch_store_b128 off, v[0:3], off offset:36
 ; GFX11-NEXT:    scratch_store_b128 off, v[0:3], off offset:20
 ; GFX11-NEXT:    scratch_store_b128 off, v[0:3], off offset:4
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 ;
 ; GFX9-PAL-LABEL: zero_init_kernel:
@@ -165,7 +164,6 @@ define amdgpu_kernel void @zero_init_kernel() {
 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], off offset:36
 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], off offset:20
 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], off offset:4
-; GFX11-PAL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-PAL-NEXT:    s_endpgm
   %alloca = alloca [32 x i16], align 2, addrspace(5)
   call void @llvm.memset.p5.i64(ptr addrspace(5) align 2 dereferenceable(64) %alloca, i8 0, i64 64, i1 false)
@@ -937,7 +935,6 @@ define amdgpu_kernel void @zero_init_small_offset_kernel() {
 ; GFX11-NEXT:    scratch_store_b128 off, v[0:3], off offset:276
 ; GFX11-NEXT:    scratch_store_b128 off, v[0:3], off offset:292
 ; GFX11-NEXT:    scratch_store_b128 off, v[0:3], off offset:308
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 ;
 ; GFX9-PAL-LABEL: zero_init_small_offset_kernel:
@@ -1051,7 +1048,6 @@ define amdgpu_kernel void @zero_init_small_offset_kernel() {
 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], off offset:276
 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], off offset:292
 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], off offset:308
-; GFX11-PAL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-PAL-NEXT:    s_endpgm
   %padding = alloca [64 x i32], align 4, addrspace(5)
   %alloca = alloca [32 x i16], align 2, addrspace(5)
@@ -1935,7 +1931,6 @@ define amdgpu_kernel void @zero_init_large_offset_kernel() {
 ; GFX11-NEXT:    scratch_store_b128 off, v[0:3], s0 offset:16
 ; GFX11-NEXT:    scratch_store_b128 off, v[0:3], s0 offset:32
 ; GFX11-NEXT:    scratch_store_b128 off, v[0:3], s0 offset:48
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 ;
 ; GFX9-PAL-LABEL: zero_init_large_offset_kernel:
@@ -2054,7 +2049,6 @@ define amdgpu_kernel void @zero_init_large_offset_kernel() {
 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], s0 offset:16
 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], s0 offset:32
 ; GFX11-PAL-NEXT:    scratch_store_b128 off, v[0:3], s0 offset:48
-; GFX11-PAL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-PAL-NEXT:    s_endpgm
   %padding = alloca [4096 x i32], align 4, addrspace(5)
   %alloca = alloca [32 x i16], align 2, addrspace(5)
diff --git a/llvm/test/CodeGen/AMDGPU/fp-min-max-flat-atomics.ll b/llvm/test/CodeGen/AMDGPU/fp-min-max-flat-atomics.ll
index b50bcd9da6ac7..3d6efc9cd2371 100644
--- a/llvm/test/CodeGen/AMDGPU/fp-min-max-flat-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp-min-max-flat-atomics.ll
@@ -16,7 +16,6 @@ define amdgpu_cs void @flat_atomic_fmin_f32_noret(ptr %ptr, float %data) {
 ; GFX11-LABEL: flat_atomic_fmin_f32_noret:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    flat_atomic_min_f32 v[0:1], v2
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %ret = call float @llvm.amdgcn.flat.atomic.fmin.f32.p1.f32(ptr %ptr, float %data)
   ret void
@@ -31,7 +30,6 @@ define amdgpu_cs void @flat_atomic_fmax_f32_noret(ptr %ptr, float %data) {
 ; GFX11-LABEL: flat_atomic_fmax_f32_noret:
 ; GFX11:       ; %bb.0:
 ; GFX11-NEXT:    flat_atomic_max_f32 v[0:1], v2
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %ret = call float @llvm.amdgcn.flat.atomic.fmax.f32.p1.f32(ptr %ptr, float %data)
   ret void
diff --git a/llvm/test/CodeGen/AMDGPU/lds-atomic-fmin-fmax.ll b/llvm/test/CodeGen/AMDGPU/lds-atomic-fmin-fmax.ll
index 99ed605e7ed52..dee5b724934a0 100644
--- a/llvm/test/CodeGen/AMDGPU/lds-atomic-fmin-fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/lds-atomic-fmin-fmax.ll
@@ -173,7 +173,6 @@ define amdgpu_kernel void @lds_ds_fmin(ptr addrspace(5) %out, ptr addrspace(3) %
 ; GFX11-NEXT:    ds_min_rtn_f32 v0, v3, v1
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    scratch_store_b32 off, v0, s0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 ;
 ; G_SI-LABEL: lds_ds_fmin:
@@ -333,7 +332,6 @@ define amdgpu_kernel void @lds_ds_fmin(ptr addrspace(5) %out, ptr addrspace(3) %
 ; G_GFX11-NEXT:    ds_min_rtn_f32 v0, v3, v0
 ; G_GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; G_GFX11-NEXT:    scratch_store_b32 off, v0, s0
-; G_GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; G_GFX11-NEXT:    s_endpgm
   %idx.add = add nuw i32 %idx, 4
   %shl0 = shl i32 %idx.add, 3
@@ -501,7 +499,6 @@ define amdgpu_kernel void @lds_ds_fmax(ptr addrspace(5) %out, ptr addrspace(3) %
 ; GFX11-NEXT:    ds_max_rtn_f32 v0, v3, v1
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    scratch_store_b32 off, v0, s0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 ;
 ; G_SI-LABEL: lds_ds_fmax:
@@ -661,7 +658,6 @@ define amdgpu_kernel void @lds_ds_fmax(ptr addrspace(5) %out, ptr addrspace(3) %
 ; G_GFX11-NEXT:    ds_max_rtn_f32 v0, v3, v0
 ; G_GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; G_GFX11-NEXT:    scratch_store_b32 off, v0, s0
-; G_GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; G_GFX11-NEXT:    s_endpgm
   %idx.add = add nuw i32 %idx, 4
   %shl0 = shl i32 %idx.add, 3
@@ -860,7 +856,6 @@ define amdgpu_kernel void @lds_ds_fmin_f64(ptr addrspace(5) %out, ptr addrspace(
 ; GFX11-NEXT:    ds_min_rtn_f64 v[0:1], v5, v[2:3]
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    scratch_store_b64 off, v[0:1], s0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 ;
 ; G_SI-LABEL: lds_ds_fmin_f64:
@@ -1049,7 +1044,6 @@ define amdgpu_kernel void @lds_ds_fmin_f64(ptr addrspace(5) %out, ptr addrspace(
 ; G_GFX11-NEXT:    ds_min_rtn_f64 v[0:1], v5, v[2:3]
 ; G_GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; G_GFX11-NEXT:    scratch_store_b64 off, v[0:1], s0
-; G_GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; G_GFX11-NEXT:    s_endpgm
   %idx.add = add nuw i32 %idx, 4
   %shl0 = shl i32 %idx.add, 3
@@ -1248,7 +1242,6 @@ define amdgpu_kernel void @lds_ds_fmax_f64(ptr addrspace(5) %out, ptr addrspace(
 ; GFX11-NEXT:    ds_max_rtn_f64 v[0:1], v5, v[2:3]
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    scratch_store_b64 off, v[0:1], s0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 ;
 ; G_SI-LABEL: lds_ds_fmax_f64:
@@ -1437,7 +1430,6 @@ define amdgpu_kernel void @lds_ds_fmax_f64(ptr addrspace(5) %out, ptr addrspace(
 ; G_GFX11-NEXT:    ds_max_rtn_f64 v[0:1], v5, v[2:3]
 ; G_GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; G_GFX11-NEXT:    scratch_store_b64 off, v[0:1], s0
-; G_GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; G_GFX11-NEXT:    s_endpgm
   %idx.add = add nuw i32 %idx, 4
   %shl0 = shl i32 %idx.add, 3
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.bf16.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.bf16.bf16.ll
index b5d5be5fa6555..6c23291797a40 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.bf16.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.bf16.bf16.ll
@@ -42,7 +42,6 @@ define amdgpu_kernel void @test_llvm_amdgcn_fdot2_bf16_bf16_dpp(
 ; SDAG-GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; SDAG-GFX11-NEXT:    v_dot2_bf16_bf16_e64_dpp v0, v2, v0, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; SDAG-GFX11-NEXT:    scratch_store_b16 off, v0, s0
-; SDAG-GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; SDAG-GFX11-NEXT:    s_endpgm
 ;
 ; GISEL-GFX11-LABEL: test_llvm_amdgcn_fdot2_bf16_bf16_dpp:
@@ -55,7 +54,6 @@ define amdgpu_kernel void @test_llvm_amdgcn_fdot2_bf16_bf16_dpp(
 ; GISEL-GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GISEL-GFX11-NEXT:    v_dot2_bf16_bf16_e64_dpp v0, v0, v1, v2 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GISEL-GFX11-NEXT:    scratch_store_b16 off, v0, s0
-; GISEL-GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GISEL-GFX11-NEXT:    s_endpgm
     ptr addrspace(5) %r,
     ptr addrspace(5) %a,
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.f16.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.f16.f16.ll
index dde7df0f79480..8ead5bfe084d0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.f16.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fdot2.f16.f16.ll
@@ -42,7 +42,6 @@ define amdgpu_kernel void @test_llvm_amdgcn_fdot2_f16_f16_dpp(
 ; SDAG-GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; SDAG-GFX11-NEXT:    v_dot2_f16_f16_e64_dpp v0, v2, v0, v1 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; SDAG-GFX11-NEXT:    scratch_store_b16 off, v0, s0
-; SDAG-GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; SDAG-GFX11-NEXT:    s_endpgm
 ;
 ; GISEL-GFX11-LABEL: test_llvm_amdgcn_fdot2_f16_f16_dpp:
@@ -55,7 +54,6 @@ define amdgpu_kernel void @test_llvm_amdgcn_fdot2_f16_f16_dpp(
 ; GISEL-GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GISEL-GFX11-NEXT:    v_dot2_f16_f16_e64_dpp v0, v0, v1, v2 quad_perm:[1,0,0,0] row_mask:0xf bank_mask:0xf bound_ctrl:1
 ; GISEL-GFX11-NEXT:    scratch_store_b16 off, v0, s0
-; GISEL-GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GISEL-GFX11-NEXT:    s_endpgm
     ptr addrspace(5) %r,
     ptr addrspace(5) %a,
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
index 10b77e967a31d..4b1cfaf7ef858 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.intersect_ray.ll
@@ -236,7 +236,6 @@ define amdgpu_kernel void @image_bvh_intersect_ray_nsa_reassign(ptr %p_node_ptr,
 ; GFX11-NEXT:    image_bvh_intersect_ray v[0:3], [v9, v10, v[6:8], v[3:5], v[0:2]], s[4:7]
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b128 v[0:1], v[0:3]
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 main_body:
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -326,7 +325,6 @@ define amdgpu_kernel void @image_bvh_intersect_ray_a16_nsa_reassign(ptr %p_node_
 ; GFX11-NEXT:    image_bvh_intersect_ray v[0:3], [v6, v7, v[3:5], v[0:2]], s[4:7] a16
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b128 v[0:1], v[0:3]
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 main_body:
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -422,7 +420,6 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_nsa_reassign(ptr %p_ray, <4
 ; GFX11-NEXT:    image_bvh64_intersect_ray v[0:3], [v[9:10], v11, v[6:8], v[3:5], v[0:2]], s[0:3]
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b128 v[0:1], v[0:3]
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 main_body:
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -508,7 +505,6 @@ define amdgpu_kernel void @image_bvh64_intersect_ray_a16_nsa_reassign(ptr %p_ray
 ; GFX11-NEXT:    image_bvh64_intersect_ray v[0:3], [v[6:7], v8, v[3:5], v[0:2]], s[0:3] a16
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    flat_store_b128 v[0:1], v[0:3]
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 main_body:
   %lid = tail call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll
index f5fad4b7a1bcd..db46b9686a0c3 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-agent.ll
@@ -124,7 +124,6 @@ define amdgpu_kernel void @flat_agent_unordered_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_unordered_load:
@@ -136,7 +135,6 @@ define amdgpu_kernel void @flat_agent_unordered_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -259,7 +257,6 @@ define amdgpu_kernel void @flat_agent_monotonic_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_monotonic_load:
@@ -271,7 +268,6 @@ define amdgpu_kernel void @flat_agent_monotonic_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -405,7 +401,6 @@ define amdgpu_kernel void @flat_agent_acquire_load(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_acquire_load:
@@ -419,7 +414,6 @@ define amdgpu_kernel void @flat_agent_acquire_load(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -565,7 +559,6 @@ define amdgpu_kernel void @flat_agent_seq_cst_load(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_seq_cst_load:
@@ -581,7 +574,6 @@ define amdgpu_kernel void @flat_agent_seq_cst_load(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -686,7 +678,6 @@ define amdgpu_kernel void @flat_agent_unordered_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_unordered_store:
@@ -698,7 +689,6 @@ define amdgpu_kernel void @flat_agent_unordered_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -802,7 +792,6 @@ define amdgpu_kernel void @flat_agent_monotonic_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_monotonic_store:
@@ -814,7 +803,6 @@ define amdgpu_kernel void @flat_agent_monotonic_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -932,7 +920,6 @@ define amdgpu_kernel void @flat_agent_release_store(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_release_store:
@@ -946,7 +933,6 @@ define amdgpu_kernel void @flat_agent_release_store(
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -1064,7 +1050,6 @@ define amdgpu_kernel void @flat_agent_seq_cst_store(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_seq_cst_store:
@@ -1078,7 +1063,6 @@ define amdgpu_kernel void @flat_agent_seq_cst_store(
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -1182,7 +1166,6 @@ define amdgpu_kernel void @flat_agent_monotonic_atomicrmw(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_monotonic_atomicrmw:
@@ -1194,7 +1177,6 @@ define amdgpu_kernel void @flat_agent_monotonic_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -1453,7 +1435,6 @@ define amdgpu_kernel void @flat_agent_release_atomicrmw(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_release_atomicrmw:
@@ -1467,7 +1448,6 @@ define amdgpu_kernel void @flat_agent_release_atomicrmw(
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -1914,7 +1894,6 @@ define amdgpu_kernel void @flat_agent_acquire_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_acquire_ret_atomicrmw:
@@ -1930,7 +1909,6 @@ define amdgpu_kernel void @flat_agent_acquire_ret_atomicrmw(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -2078,7 +2056,6 @@ define amdgpu_kernel void @flat_agent_acq_rel_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_acq_rel_ret_atomicrmw:
@@ -2096,7 +2073,6 @@ define amdgpu_kernel void @flat_agent_acq_rel_ret_atomicrmw(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -2244,7 +2220,6 @@ define amdgpu_kernel void @flat_agent_seq_cst_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_seq_cst_ret_atomicrmw:
@@ -2262,7 +2237,6 @@ define amdgpu_kernel void @flat_agent_seq_cst_ret_atomicrmw(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -2367,7 +2341,6 @@ define amdgpu_kernel void @flat_agent_monotonic_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_monotonic_monotonic_cmpxchg:
@@ -2377,7 +2350,6 @@ define amdgpu_kernel void @flat_agent_monotonic_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -2636,7 +2608,6 @@ define amdgpu_kernel void @flat_agent_release_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_release_monotonic_cmpxchg:
@@ -2648,7 +2619,6 @@ define amdgpu_kernel void @flat_agent_release_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -4619,7 +4589,6 @@ define amdgpu_kernel void @flat_agent_monotonic_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_monotonic_monotonic_ret_cmpxchg:
@@ -4631,7 +4600,6 @@ define amdgpu_kernel void @flat_agent_monotonic_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -4775,7 +4743,6 @@ define amdgpu_kernel void @flat_agent_acquire_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_acquire_monotonic_ret_cmpxchg:
@@ -4789,7 +4756,6 @@ define amdgpu_kernel void @flat_agent_acquire_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -4936,7 +4902,6 @@ define amdgpu_kernel void @flat_agent_release_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_release_monotonic_ret_cmpxchg:
@@ -4950,7 +4915,6 @@ define amdgpu_kernel void @flat_agent_release_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5108,7 +5072,6 @@ define amdgpu_kernel void @flat_agent_acq_rel_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_acq_rel_monotonic_ret_cmpxchg:
@@ -5124,7 +5087,6 @@ define amdgpu_kernel void @flat_agent_acq_rel_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5282,7 +5244,6 @@ define amdgpu_kernel void @flat_agent_seq_cst_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_seq_cst_monotonic_ret_cmpxchg:
@@ -5298,7 +5259,6 @@ define amdgpu_kernel void @flat_agent_seq_cst_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5442,7 +5402,6 @@ define amdgpu_kernel void @flat_agent_monotonic_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_monotonic_acquire_ret_cmpxchg:
@@ -5456,7 +5415,6 @@ define amdgpu_kernel void @flat_agent_monotonic_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5600,7 +5558,6 @@ define amdgpu_kernel void @flat_agent_acquire_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_acquire_acquire_ret_cmpxchg:
@@ -5614,7 +5571,6 @@ define amdgpu_kernel void @flat_agent_acquire_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5772,7 +5728,6 @@ define amdgpu_kernel void @flat_agent_release_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_release_acquire_ret_cmpxchg:
@@ -5788,7 +5743,6 @@ define amdgpu_kernel void @flat_agent_release_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5946,7 +5900,6 @@ define amdgpu_kernel void @flat_agent_acq_rel_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_acq_rel_acquire_ret_cmpxchg:
@@ -5962,7 +5915,6 @@ define amdgpu_kernel void @flat_agent_acq_rel_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -6120,7 +6072,6 @@ define amdgpu_kernel void @flat_agent_seq_cst_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_seq_cst_acquire_ret_cmpxchg:
@@ -6136,7 +6087,6 @@ define amdgpu_kernel void @flat_agent_seq_cst_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -6294,7 +6244,6 @@ define amdgpu_kernel void @flat_agent_monotonic_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_monotonic_seq_cst_ret_cmpxchg:
@@ -6310,7 +6259,6 @@ define amdgpu_kernel void @flat_agent_monotonic_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -6468,7 +6416,6 @@ define amdgpu_kernel void @flat_agent_acquire_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_acquire_seq_cst_ret_cmpxchg:
@@ -6484,7 +6431,6 @@ define amdgpu_kernel void @flat_agent_acquire_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -6642,7 +6588,6 @@ define amdgpu_kernel void @flat_agent_release_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_release_seq_cst_ret_cmpxchg:
@@ -6658,7 +6603,6 @@ define amdgpu_kernel void @flat_agent_release_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -6816,7 +6760,6 @@ define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_acq_rel_seq_cst_ret_cmpxchg:
@@ -6832,7 +6775,6 @@ define amdgpu_kernel void @flat_agent_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -6990,7 +6932,6 @@ define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_seq_cst_seq_cst_ret_cmpxchg:
@@ -7006,7 +6947,6 @@ define amdgpu_kernel void @flat_agent_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -7131,7 +7071,6 @@ define amdgpu_kernel void @flat_agent_one_as_unordered_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_unordered_load:
@@ -7143,7 +7082,6 @@ define amdgpu_kernel void @flat_agent_one_as_unordered_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -7266,7 +7204,6 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_monotonic_load:
@@ -7278,7 +7215,6 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -7419,7 +7355,6 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_acquire_load:
@@ -7434,7 +7369,6 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -7587,7 +7521,6 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_load:
@@ -7604,7 +7537,6 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -7709,7 +7641,6 @@ define amdgpu_kernel void @flat_agent_one_as_unordered_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_unordered_store:
@@ -7721,7 +7652,6 @@ define amdgpu_kernel void @flat_agent_one_as_unordered_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -7825,7 +7755,6 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_monotonic_store:
@@ -7837,7 +7766,6 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -7955,7 +7883,6 @@ define amdgpu_kernel void @flat_agent_one_as_release_store(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_release_store:
@@ -7969,7 +7896,6 @@ define amdgpu_kernel void @flat_agent_one_as_release_store(
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -8087,7 +8013,6 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_store(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_store:
@@ -8101,7 +8026,6 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_store(
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -8205,7 +8129,6 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_atomicrmw(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_monotonic_atomicrmw:
@@ -8217,7 +8140,6 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -8472,7 +8394,6 @@ define amdgpu_kernel void @flat_agent_one_as_release_atomicrmw(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_release_atomicrmw:
@@ -8486,7 +8407,6 @@ define amdgpu_kernel void @flat_agent_one_as_release_atomicrmw(
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -8931,7 +8851,6 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_acquire_ret_atomicrmw:
@@ -8948,7 +8867,6 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_ret_atomicrmw(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -9102,7 +9020,6 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_ret_atomicrmw:
@@ -9121,7 +9038,6 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_ret_atomicrmw(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -9275,7 +9191,6 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_ret_atomicrmw:
@@ -9294,7 +9209,6 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_ret_atomicrmw(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -9399,7 +9313,6 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_monotonic_monotonic_cmpxchg:
@@ -9409,7 +9322,6 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -9664,7 +9576,6 @@ define amdgpu_kernel void @flat_agent_one_as_release_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_release_monotonic_cmpxchg:
@@ -9676,7 +9587,6 @@ define amdgpu_kernel void @flat_agent_one_as_release_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -11599,7 +11509,6 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_monotonic_monotonic_ret_cmpxchg:
@@ -11611,7 +11520,6 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -11762,7 +11670,6 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_acquire_monotonic_ret_cmpxchg:
@@ -11777,7 +11684,6 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -11924,7 +11830,6 @@ define amdgpu_kernel void @flat_agent_one_as_release_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_release_monotonic_ret_cmpxchg:
@@ -11938,7 +11843,6 @@ define amdgpu_kernel void @flat_agent_one_as_release_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -12103,7 +12007,6 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg:
@@ -12120,7 +12023,6 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -12285,7 +12187,6 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg:
@@ -12302,7 +12203,6 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -12453,7 +12353,6 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_monotonic_acquire_ret_cmpxchg:
@@ -12468,7 +12367,6 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -12619,7 +12517,6 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_acquire_acquire_ret_cmpxchg:
@@ -12634,7 +12531,6 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -12799,7 +12695,6 @@ define amdgpu_kernel void @flat_agent_one_as_release_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_release_acquire_ret_cmpxchg:
@@ -12816,7 +12711,6 @@ define amdgpu_kernel void @flat_agent_one_as_release_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -12981,7 +12875,6 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_acquire_ret_cmpxchg:
@@ -12998,7 +12891,6 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -13163,7 +13055,6 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_acquire_ret_cmpxchg:
@@ -13180,7 +13071,6 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -13345,7 +13235,6 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg:
@@ -13362,7 +13251,6 @@ define amdgpu_kernel void @flat_agent_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -13527,7 +13415,6 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_acquire_seq_cst_ret_cmpxchg:
@@ -13544,7 +13431,6 @@ define amdgpu_kernel void @flat_agent_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -13709,7 +13595,6 @@ define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_release_seq_cst_ret_cmpxchg:
@@ -13726,7 +13611,6 @@ define amdgpu_kernel void @flat_agent_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -13891,7 +13775,6 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg:
@@ -13908,7 +13791,6 @@ define amdgpu_kernel void @flat_agent_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -14073,7 +13955,6 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg:
@@ -14090,7 +13971,6 @@ define amdgpu_kernel void @flat_agent_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
index 6ec3f659e84a2..6d48e36d91b14 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-nontemporal.ll
@@ -124,7 +124,6 @@ define amdgpu_kernel void @flat_nontemporal_load_0(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_nontemporal_load_0:
@@ -136,7 +135,6 @@ define amdgpu_kernel void @flat_nontemporal_load_0(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -274,7 +272,6 @@ define amdgpu_kernel void @flat_nontemporal_load_1(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_nontemporal_load_1:
@@ -289,7 +286,6 @@ define amdgpu_kernel void @flat_nontemporal_load_1(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -414,7 +410,6 @@ define amdgpu_kernel void @flat_nontemporal_store_0(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2 glc slc dlc
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_nontemporal_store_0:
@@ -426,7 +421,6 @@ define amdgpu_kernel void @flat_nontemporal_store_0(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2 glc slc dlc
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -564,7 +558,6 @@ define amdgpu_kernel void @flat_nontemporal_store_1(
 ; GFX11-WGP-NEXT:    v_add_co_ci_u32_e64 v1, null, s3, 0, s0
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2 glc slc dlc
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_nontemporal_store_1:
@@ -579,7 +572,6 @@ define amdgpu_kernel void @flat_nontemporal_store_1(
 ; GFX11-CU-NEXT:    v_add_co_ci_u32_e64 v1, null, s3, 0, s0
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2 glc slc dlc
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-singlethread.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-singlethread.ll
index cfeb62f7d0233..8720186ab02f3 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-singlethread.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-singlethread.ll
@@ -124,7 +124,6 @@ define amdgpu_kernel void @flat_singlethread_unordered_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_unordered_load:
@@ -136,7 +135,6 @@ define amdgpu_kernel void @flat_singlethread_unordered_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -259,7 +257,6 @@ define amdgpu_kernel void @flat_singlethread_monotonic_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_monotonic_load:
@@ -271,7 +268,6 @@ define amdgpu_kernel void @flat_singlethread_monotonic_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -394,7 +390,6 @@ define amdgpu_kernel void @flat_singlethread_acquire_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_acquire_load:
@@ -406,7 +401,6 @@ define amdgpu_kernel void @flat_singlethread_acquire_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -529,7 +523,6 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_seq_cst_load:
@@ -541,7 +534,6 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -646,7 +638,6 @@ define amdgpu_kernel void @flat_singlethread_unordered_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_unordered_store:
@@ -658,7 +649,6 @@ define amdgpu_kernel void @flat_singlethread_unordered_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -762,7 +752,6 @@ define amdgpu_kernel void @flat_singlethread_monotonic_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_monotonic_store:
@@ -774,7 +763,6 @@ define amdgpu_kernel void @flat_singlethread_monotonic_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -878,7 +866,6 @@ define amdgpu_kernel void @flat_singlethread_release_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_release_store:
@@ -890,7 +877,6 @@ define amdgpu_kernel void @flat_singlethread_release_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -994,7 +980,6 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_seq_cst_store:
@@ -1006,7 +991,6 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -1110,7 +1094,6 @@ define amdgpu_kernel void @flat_singlethread_monotonic_atomicrmw(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_monotonic_atomicrmw:
@@ -1122,7 +1105,6 @@ define amdgpu_kernel void @flat_singlethread_monotonic_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -1226,7 +1208,6 @@ define amdgpu_kernel void @flat_singlethread_acquire_atomicrmw(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_acquire_atomicrmw:
@@ -1238,7 +1219,6 @@ define amdgpu_kernel void @flat_singlethread_acquire_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -1342,7 +1322,6 @@ define amdgpu_kernel void @flat_singlethread_release_atomicrmw(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_release_atomicrmw:
@@ -1354,7 +1333,6 @@ define amdgpu_kernel void @flat_singlethread_release_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -1458,7 +1436,6 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_atomicrmw(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_acq_rel_atomicrmw:
@@ -1470,7 +1447,6 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -1574,7 +1550,6 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_atomicrmw(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_seq_cst_atomicrmw:
@@ -1586,7 +1561,6 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -1708,7 +1682,6 @@ define amdgpu_kernel void @flat_singlethread_acquire_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_acquire_ret_atomicrmw:
@@ -1722,7 +1695,6 @@ define amdgpu_kernel void @flat_singlethread_acquire_ret_atomicrmw(
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -1845,7 +1817,6 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_acq_rel_ret_atomicrmw:
@@ -1859,7 +1830,6 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_ret_atomicrmw(
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -1982,7 +1952,6 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_seq_cst_ret_atomicrmw:
@@ -1996,7 +1965,6 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_ret_atomicrmw(
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -2101,7 +2069,6 @@ define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_monotonic_monotonic_cmpxchg:
@@ -2111,7 +2078,6 @@ define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -2216,7 +2182,6 @@ define amdgpu_kernel void @flat_singlethread_acquire_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_acquire_monotonic_cmpxchg:
@@ -2226,7 +2191,6 @@ define amdgpu_kernel void @flat_singlethread_acquire_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -2331,7 +2295,6 @@ define amdgpu_kernel void @flat_singlethread_release_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_release_monotonic_cmpxchg:
@@ -2341,7 +2304,6 @@ define amdgpu_kernel void @flat_singlethread_release_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -2446,7 +2408,6 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_acq_rel_monotonic_cmpxchg:
@@ -2456,7 +2417,6 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -2561,7 +2521,6 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_seq_cst_monotonic_cmpxchg:
@@ -2571,7 +2530,6 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -2676,7 +2634,6 @@ define amdgpu_kernel void @flat_singlethread_monotonic_acquire_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_monotonic_acquire_cmpxchg:
@@ -2686,7 +2643,6 @@ define amdgpu_kernel void @flat_singlethread_monotonic_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -2791,7 +2747,6 @@ define amdgpu_kernel void @flat_singlethread_acquire_acquire_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_acquire_acquire_cmpxchg:
@@ -2801,7 +2756,6 @@ define amdgpu_kernel void @flat_singlethread_acquire_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -2906,7 +2860,6 @@ define amdgpu_kernel void @flat_singlethread_release_acquire_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_release_acquire_cmpxchg:
@@ -2916,7 +2869,6 @@ define amdgpu_kernel void @flat_singlethread_release_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -3021,7 +2973,6 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_acq_rel_acquire_cmpxchg:
@@ -3031,7 +2982,6 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -3136,7 +3086,6 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_seq_cst_acquire_cmpxchg:
@@ -3146,7 +3095,6 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -3251,7 +3199,6 @@ define amdgpu_kernel void @flat_singlethread_monotonic_seq_cst_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_monotonic_seq_cst_cmpxchg:
@@ -3261,7 +3208,6 @@ define amdgpu_kernel void @flat_singlethread_monotonic_seq_cst_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -3366,7 +3312,6 @@ define amdgpu_kernel void @flat_singlethread_acquire_seq_cst_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_acquire_seq_cst_cmpxchg:
@@ -3376,7 +3321,6 @@ define amdgpu_kernel void @flat_singlethread_acquire_seq_cst_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -3481,7 +3425,6 @@ define amdgpu_kernel void @flat_singlethread_release_seq_cst_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_release_seq_cst_cmpxchg:
@@ -3491,7 +3434,6 @@ define amdgpu_kernel void @flat_singlethread_release_seq_cst_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -3596,7 +3538,6 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_seq_cst_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_acq_rel_seq_cst_cmpxchg:
@@ -3606,7 +3547,6 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_seq_cst_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -3711,7 +3651,6 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_seq_cst_seq_cst_cmpxchg:
@@ -3721,7 +3660,6 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -3852,7 +3790,6 @@ define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_monotonic_monotonic_ret_cmpxchg:
@@ -3864,7 +3801,6 @@ define amdgpu_kernel void @flat_singlethread_monotonic_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -3997,7 +3933,6 @@ define amdgpu_kernel void @flat_singlethread_acquire_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_acquire_monotonic_ret_cmpxchg:
@@ -4009,7 +3944,6 @@ define amdgpu_kernel void @flat_singlethread_acquire_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -4142,7 +4076,6 @@ define amdgpu_kernel void @flat_singlethread_release_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_release_monotonic_ret_cmpxchg:
@@ -4154,7 +4087,6 @@ define amdgpu_kernel void @flat_singlethread_release_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -4287,7 +4219,6 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_acq_rel_monotonic_ret_cmpxchg:
@@ -4299,7 +4230,6 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -4432,7 +4362,6 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_seq_cst_monotonic_ret_cmpxchg:
@@ -4444,7 +4373,6 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -4577,7 +4505,6 @@ define amdgpu_kernel void @flat_singlethread_monotonic_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_monotonic_acquire_ret_cmpxchg:
@@ -4589,7 +4516,6 @@ define amdgpu_kernel void @flat_singlethread_monotonic_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -4722,7 +4648,6 @@ define amdgpu_kernel void @flat_singlethread_acquire_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_acquire_acquire_ret_cmpxchg:
@@ -4734,7 +4659,6 @@ define amdgpu_kernel void @flat_singlethread_acquire_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -4867,7 +4791,6 @@ define amdgpu_kernel void @flat_singlethread_release_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_release_acquire_ret_cmpxchg:
@@ -4879,7 +4802,6 @@ define amdgpu_kernel void @flat_singlethread_release_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5012,7 +4934,6 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_acq_rel_acquire_ret_cmpxchg:
@@ -5024,7 +4945,6 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5157,7 +5077,6 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_seq_cst_acquire_ret_cmpxchg:
@@ -5169,7 +5088,6 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5302,7 +5220,6 @@ define amdgpu_kernel void @flat_singlethread_monotonic_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_monotonic_seq_cst_ret_cmpxchg:
@@ -5314,7 +5231,6 @@ define amdgpu_kernel void @flat_singlethread_monotonic_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5447,7 +5363,6 @@ define amdgpu_kernel void @flat_singlethread_acquire_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_acquire_seq_cst_ret_cmpxchg:
@@ -5459,7 +5374,6 @@ define amdgpu_kernel void @flat_singlethread_acquire_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5592,7 +5506,6 @@ define amdgpu_kernel void @flat_singlethread_release_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_release_seq_cst_ret_cmpxchg:
@@ -5604,7 +5517,6 @@ define amdgpu_kernel void @flat_singlethread_release_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5737,7 +5649,6 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_acq_rel_seq_cst_ret_cmpxchg:
@@ -5749,7 +5660,6 @@ define amdgpu_kernel void @flat_singlethread_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5882,7 +5792,6 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_seq_cst_seq_cst_ret_cmpxchg:
@@ -5894,7 +5803,6 @@ define amdgpu_kernel void @flat_singlethread_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -6019,7 +5927,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_unordered_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_unordered_load:
@@ -6031,7 +5938,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_unordered_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -6154,7 +6060,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_load:
@@ -6166,7 +6071,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -6289,7 +6193,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_load:
@@ -6301,7 +6204,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -6424,7 +6326,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_load:
@@ -6436,7 +6337,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -6541,7 +6441,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_unordered_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_unordered_store:
@@ -6553,7 +6452,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_unordered_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -6657,7 +6555,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_store:
@@ -6669,7 +6566,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -6773,7 +6669,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_release_store:
@@ -6785,7 +6680,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -6889,7 +6783,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_store:
@@ -6901,7 +6794,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -7005,7 +6897,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_atomicrmw(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_atomicrmw:
@@ -7017,7 +6908,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -7121,7 +7011,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_atomicrmw(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_atomicrmw:
@@ -7133,7 +7022,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -7237,7 +7125,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_atomicrmw(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_release_atomicrmw:
@@ -7249,7 +7136,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -7353,7 +7239,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_atomicrmw(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_acq_rel_atomicrmw:
@@ -7365,7 +7250,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -7469,7 +7353,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_atomicrmw(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_atomicrmw:
@@ -7481,7 +7364,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -7603,7 +7485,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_ret_atomicrmw:
@@ -7617,7 +7498,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_ret_atomicrmw(
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -7740,7 +7620,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_acq_rel_ret_atomicrmw:
@@ -7754,7 +7633,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_ret_atomicrmw(
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -7877,7 +7755,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_ret_atomicrmw:
@@ -7891,7 +7768,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_ret_atomicrmw(
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -7996,7 +7872,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_monotonic_cmpxchg:
@@ -8006,7 +7881,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -8111,7 +7985,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_monotonic_cmpxchg:
@@ -8121,7 +7994,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -8226,7 +8098,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_release_monotonic_cmpxchg:
@@ -8236,7 +8107,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -8341,7 +8211,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_acq_rel_monotonic_cmpxchg:
@@ -8351,7 +8220,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -8456,7 +8324,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_monotonic_cmpxchg:
@@ -8466,7 +8333,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -8571,7 +8437,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_acquire_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_acquire_cmpxchg:
@@ -8581,7 +8446,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -8686,7 +8550,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_acquire_cmpxchg:
@@ -8696,7 +8559,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -8801,7 +8663,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_release_acquire_cmpxchg:
@@ -8811,7 +8672,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -8916,7 +8776,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_acq_rel_acquire_cmpxchg:
@@ -8926,7 +8785,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -9031,7 +8889,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_acquire_cmpxchg:
@@ -9041,7 +8898,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -9146,7 +9002,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_seq_cst_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_seq_cst_cmpxchg:
@@ -9156,7 +9011,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_seq_cst_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -9261,7 +9115,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_seq_cst_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_seq_cst_cmpxchg:
@@ -9271,7 +9124,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_seq_cst_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -9376,7 +9228,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_seq_cst_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_release_seq_cst_cmpxchg:
@@ -9386,7 +9237,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_seq_cst_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -9491,7 +9341,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg:
@@ -9501,7 +9350,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_seq_cst_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -9606,7 +9454,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg:
@@ -9616,7 +9463,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -9747,7 +9593,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_ret_cmpx
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_monotonic_ret_cmpxchg:
@@ -9759,7 +9604,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_monotonic_ret_cmpx
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -9892,7 +9736,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_ret_cmpxch
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_monotonic_ret_cmpxchg:
@@ -9904,7 +9747,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_monotonic_ret_cmpxch
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -10037,7 +9879,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_ret_cmpxch
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_release_monotonic_ret_cmpxchg:
@@ -10049,7 +9890,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_monotonic_ret_cmpxch
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -10182,7 +10022,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxch
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxchg:
@@ -10194,7 +10033,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_monotonic_ret_cmpxch
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -10327,7 +10165,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxch
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxchg:
@@ -10339,7 +10176,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_monotonic_ret_cmpxch
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -10472,7 +10308,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_acquire_ret_cmpxch
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_acquire_ret_cmpxchg:
@@ -10484,7 +10319,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_acquire_ret_cmpxch
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -10617,7 +10451,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_acquire_ret_cmpxchg:
@@ -10629,7 +10462,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -10762,7 +10594,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_release_acquire_ret_cmpxchg:
@@ -10774,7 +10605,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -10907,7 +10737,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg:
@@ -10919,7 +10748,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -11052,7 +10880,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg:
@@ -11064,7 +10891,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -11197,7 +11023,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxch
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxchg:
@@ -11209,7 +11034,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_monotonic_seq_cst_ret_cmpxch
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -11342,7 +11166,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg:
@@ -11354,7 +11177,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -11487,7 +11309,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_release_seq_cst_ret_cmpxchg:
@@ -11499,7 +11320,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -11632,7 +11452,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg:
@@ -11644,7 +11463,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -11777,7 +11595,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg:
@@ -11789,7 +11606,6 @@ define amdgpu_kernel void @flat_singlethread_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll
index dd6fd62134686..a55436d15384d 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-system.ll
@@ -124,7 +124,6 @@ define amdgpu_kernel void @flat_system_unordered_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_unordered_load:
@@ -136,7 +135,6 @@ define amdgpu_kernel void @flat_system_unordered_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -259,7 +257,6 @@ define amdgpu_kernel void @flat_system_monotonic_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_monotonic_load:
@@ -271,7 +268,6 @@ define amdgpu_kernel void @flat_system_monotonic_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -407,7 +403,6 @@ define amdgpu_kernel void @flat_system_acquire_load(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_acquire_load:
@@ -421,7 +416,6 @@ define amdgpu_kernel void @flat_system_acquire_load(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -569,7 +563,6 @@ define amdgpu_kernel void @flat_system_seq_cst_load(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_seq_cst_load:
@@ -585,7 +578,6 @@ define amdgpu_kernel void @flat_system_seq_cst_load(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -690,7 +682,6 @@ define amdgpu_kernel void @flat_system_unordered_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_unordered_store:
@@ -702,7 +693,6 @@ define amdgpu_kernel void @flat_system_unordered_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -806,7 +796,6 @@ define amdgpu_kernel void @flat_system_monotonic_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_monotonic_store:
@@ -818,7 +807,6 @@ define amdgpu_kernel void @flat_system_monotonic_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -938,7 +926,6 @@ define amdgpu_kernel void @flat_system_release_store(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_release_store:
@@ -952,7 +939,6 @@ define amdgpu_kernel void @flat_system_release_store(
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -1072,7 +1058,6 @@ define amdgpu_kernel void @flat_system_seq_cst_store(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_seq_cst_store:
@@ -1086,7 +1071,6 @@ define amdgpu_kernel void @flat_system_seq_cst_store(
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -1190,7 +1174,6 @@ define amdgpu_kernel void @flat_system_monotonic_atomicrmw(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_monotonic_atomicrmw:
@@ -1202,7 +1185,6 @@ define amdgpu_kernel void @flat_system_monotonic_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -1465,7 +1447,6 @@ define amdgpu_kernel void @flat_system_release_atomicrmw(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_release_atomicrmw:
@@ -1479,7 +1460,6 @@ define amdgpu_kernel void @flat_system_release_atomicrmw(
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -1936,7 +1916,6 @@ define amdgpu_kernel void @flat_system_acquire_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_acquire_ret_atomicrmw:
@@ -1952,7 +1931,6 @@ define amdgpu_kernel void @flat_system_acquire_ret_atomicrmw(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -2104,7 +2082,6 @@ define amdgpu_kernel void @flat_system_acq_rel_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_acq_rel_ret_atomicrmw:
@@ -2122,7 +2099,6 @@ define amdgpu_kernel void @flat_system_acq_rel_ret_atomicrmw(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -2274,7 +2250,6 @@ define amdgpu_kernel void @flat_system_seq_cst_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_seq_cst_ret_atomicrmw:
@@ -2292,7 +2267,6 @@ define amdgpu_kernel void @flat_system_seq_cst_ret_atomicrmw(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -2397,7 +2371,6 @@ define amdgpu_kernel void @flat_system_monotonic_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_monotonic_monotonic_cmpxchg:
@@ -2407,7 +2380,6 @@ define amdgpu_kernel void @flat_system_monotonic_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -2670,7 +2642,6 @@ define amdgpu_kernel void @flat_system_release_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_release_monotonic_cmpxchg:
@@ -2682,7 +2653,6 @@ define amdgpu_kernel void @flat_system_release_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -4697,7 +4667,6 @@ define amdgpu_kernel void @flat_system_monotonic_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_monotonic_monotonic_ret_cmpxchg:
@@ -4709,7 +4678,6 @@ define amdgpu_kernel void @flat_system_monotonic_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -4855,7 +4823,6 @@ define amdgpu_kernel void @flat_system_acquire_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_acquire_monotonic_ret_cmpxchg:
@@ -4869,7 +4836,6 @@ define amdgpu_kernel void @flat_system_acquire_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5018,7 +4984,6 @@ define amdgpu_kernel void @flat_system_release_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_release_monotonic_ret_cmpxchg:
@@ -5032,7 +4997,6 @@ define amdgpu_kernel void @flat_system_release_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5194,7 +5158,6 @@ define amdgpu_kernel void @flat_system_acq_rel_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_acq_rel_monotonic_ret_cmpxchg:
@@ -5210,7 +5173,6 @@ define amdgpu_kernel void @flat_system_acq_rel_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5372,7 +5334,6 @@ define amdgpu_kernel void @flat_system_seq_cst_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_seq_cst_monotonic_ret_cmpxchg:
@@ -5388,7 +5349,6 @@ define amdgpu_kernel void @flat_system_seq_cst_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5534,7 +5494,6 @@ define amdgpu_kernel void @flat_system_monotonic_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_monotonic_acquire_ret_cmpxchg:
@@ -5548,7 +5507,6 @@ define amdgpu_kernel void @flat_system_monotonic_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5694,7 +5652,6 @@ define amdgpu_kernel void @flat_system_acquire_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_acquire_acquire_ret_cmpxchg:
@@ -5708,7 +5665,6 @@ define amdgpu_kernel void @flat_system_acquire_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5870,7 +5826,6 @@ define amdgpu_kernel void @flat_system_release_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_release_acquire_ret_cmpxchg:
@@ -5886,7 +5841,6 @@ define amdgpu_kernel void @flat_system_release_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -6048,7 +6002,6 @@ define amdgpu_kernel void @flat_system_acq_rel_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_acq_rel_acquire_ret_cmpxchg:
@@ -6064,7 +6017,6 @@ define amdgpu_kernel void @flat_system_acq_rel_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -6226,7 +6178,6 @@ define amdgpu_kernel void @flat_system_seq_cst_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_seq_cst_acquire_ret_cmpxchg:
@@ -6242,7 +6193,6 @@ define amdgpu_kernel void @flat_system_seq_cst_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -6404,7 +6354,6 @@ define amdgpu_kernel void @flat_system_monotonic_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_monotonic_seq_cst_ret_cmpxchg:
@@ -6420,7 +6369,6 @@ define amdgpu_kernel void @flat_system_monotonic_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -6582,7 +6530,6 @@ define amdgpu_kernel void @flat_system_acquire_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_acquire_seq_cst_ret_cmpxchg:
@@ -6598,7 +6545,6 @@ define amdgpu_kernel void @flat_system_acquire_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -6760,7 +6706,6 @@ define amdgpu_kernel void @flat_system_release_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_release_seq_cst_ret_cmpxchg:
@@ -6776,7 +6721,6 @@ define amdgpu_kernel void @flat_system_release_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -6938,7 +6882,6 @@ define amdgpu_kernel void @flat_system_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_acq_rel_seq_cst_ret_cmpxchg:
@@ -6954,7 +6897,6 @@ define amdgpu_kernel void @flat_system_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -7116,7 +7058,6 @@ define amdgpu_kernel void @flat_system_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_seq_cst_seq_cst_ret_cmpxchg:
@@ -7132,7 +7073,6 @@ define amdgpu_kernel void @flat_system_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl0_inv
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -7257,7 +7197,6 @@ define amdgpu_kernel void @flat_system_one_as_unordered_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_unordered_load:
@@ -7269,7 +7208,6 @@ define amdgpu_kernel void @flat_system_one_as_unordered_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -7392,7 +7330,6 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_monotonic_load:
@@ -7404,7 +7341,6 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -7547,7 +7483,6 @@ define amdgpu_kernel void @flat_system_one_as_acquire_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_acquire_load:
@@ -7562,7 +7497,6 @@ define amdgpu_kernel void @flat_system_one_as_acquire_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -7717,7 +7651,6 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_seq_cst_load:
@@ -7734,7 +7667,6 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -7839,7 +7771,6 @@ define amdgpu_kernel void @flat_system_one_as_unordered_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_unordered_store:
@@ -7851,7 +7782,6 @@ define amdgpu_kernel void @flat_system_one_as_unordered_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -7955,7 +7885,6 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_monotonic_store:
@@ -7967,7 +7896,6 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -8087,7 +8015,6 @@ define amdgpu_kernel void @flat_system_one_as_release_store(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_release_store:
@@ -8101,7 +8028,6 @@ define amdgpu_kernel void @flat_system_one_as_release_store(
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -8221,7 +8147,6 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_store(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_seq_cst_store:
@@ -8235,7 +8160,6 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_store(
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -8339,7 +8263,6 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_atomicrmw(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_monotonic_atomicrmw:
@@ -8351,7 +8274,6 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -8610,7 +8532,6 @@ define amdgpu_kernel void @flat_system_one_as_release_atomicrmw(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_release_atomicrmw:
@@ -8624,7 +8545,6 @@ define amdgpu_kernel void @flat_system_one_as_release_atomicrmw(
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -9079,7 +8999,6 @@ define amdgpu_kernel void @flat_system_one_as_acquire_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_acquire_ret_atomicrmw:
@@ -9096,7 +9015,6 @@ define amdgpu_kernel void @flat_system_one_as_acquire_ret_atomicrmw(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -9254,7 +9172,6 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_acq_rel_ret_atomicrmw:
@@ -9273,7 +9190,6 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_ret_atomicrmw(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -9431,7 +9347,6 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_seq_cst_ret_atomicrmw:
@@ -9450,7 +9365,6 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_ret_atomicrmw(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -9555,7 +9469,6 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_monotonic_monotonic_cmpxchg:
@@ -9565,7 +9478,6 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -9824,7 +9736,6 @@ define amdgpu_kernel void @flat_system_one_as_release_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_release_monotonic_cmpxchg:
@@ -9836,7 +9747,6 @@ define amdgpu_kernel void @flat_system_one_as_release_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -11803,7 +11713,6 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_monotonic_monotonic_ret_cmpxchg:
@@ -11815,7 +11724,6 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -11968,7 +11876,6 @@ define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_acquire_monotonic_ret_cmpxchg:
@@ -11983,7 +11890,6 @@ define amdgpu_kernel void @flat_system_one_as_acquire_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -12132,7 +12038,6 @@ define amdgpu_kernel void @flat_system_one_as_release_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_release_monotonic_ret_cmpxchg:
@@ -12146,7 +12051,6 @@ define amdgpu_kernel void @flat_system_one_as_release_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -12315,7 +12219,6 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_acq_rel_monotonic_ret_cmpxchg:
@@ -12332,7 +12235,6 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -12501,7 +12403,6 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_seq_cst_monotonic_ret_cmpxchg:
@@ -12518,7 +12419,6 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -12671,7 +12571,6 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_monotonic_acquire_ret_cmpxchg:
@@ -12686,7 +12585,6 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -12839,7 +12737,6 @@ define amdgpu_kernel void @flat_system_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_acquire_acquire_ret_cmpxchg:
@@ -12854,7 +12751,6 @@ define amdgpu_kernel void @flat_system_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -13023,7 +12919,6 @@ define amdgpu_kernel void @flat_system_one_as_release_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_release_acquire_ret_cmpxchg:
@@ -13040,7 +12935,6 @@ define amdgpu_kernel void @flat_system_one_as_release_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -13209,7 +13103,6 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_acq_rel_acquire_ret_cmpxchg:
@@ -13226,7 +13119,6 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -13395,7 +13287,6 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_seq_cst_acquire_ret_cmpxchg:
@@ -13412,7 +13303,6 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -13581,7 +13471,6 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_monotonic_seq_cst_ret_cmpxchg:
@@ -13598,7 +13487,6 @@ define amdgpu_kernel void @flat_system_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -13767,7 +13655,6 @@ define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_acquire_seq_cst_ret_cmpxchg:
@@ -13784,7 +13671,6 @@ define amdgpu_kernel void @flat_system_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -13953,7 +13839,6 @@ define amdgpu_kernel void @flat_system_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_release_seq_cst_ret_cmpxchg:
@@ -13970,7 +13855,6 @@ define amdgpu_kernel void @flat_system_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -14139,7 +14023,6 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg:
@@ -14156,7 +14039,6 @@ define amdgpu_kernel void @flat_system_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -14325,7 +14207,6 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl1_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg:
@@ -14342,7 +14223,6 @@ define amdgpu_kernel void @flat_system_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    buffer_gl1_inv
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-volatile.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-volatile.ll
index d8879dd39f07a..c07c6478da018 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-volatile.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-volatile.ll
@@ -73,7 +73,6 @@ define amdgpu_kernel void @flat_nontemporal_load_0(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_nontemporal_load_0:
@@ -86,7 +85,6 @@ define amdgpu_kernel void @flat_nontemporal_load_0(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -171,7 +169,6 @@ define amdgpu_kernel void @flat_nontemporal_load_1(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_nontemporal_load_1:
@@ -187,7 +184,6 @@ define amdgpu_kernel void @flat_nontemporal_load_1(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v0, s2
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -265,7 +261,6 @@ define amdgpu_kernel void @flat_nontemporal_store_0(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2 dlc
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_nontemporal_store_0:
@@ -278,7 +273,6 @@ define amdgpu_kernel void @flat_nontemporal_store_0(
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2 dlc
 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -363,7 +357,6 @@ define amdgpu_kernel void @flat_nontemporal_store_1(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2 dlc
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_nontemporal_store_1:
@@ -379,7 +372,6 @@ define amdgpu_kernel void @flat_nontemporal_store_1(
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2 dlc
 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -457,7 +449,6 @@ define amdgpu_kernel void @flat_volatile_workgroup_acquire_load(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_volatile_workgroup_acquire_load:
@@ -470,7 +461,6 @@ define amdgpu_kernel void @flat_volatile_workgroup_acquire_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -542,7 +532,6 @@ define amdgpu_kernel void @flat_volatile_workgroup_release_store(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_volatile_workgroup_release_store:
@@ -555,7 +544,6 @@ define amdgpu_kernel void @flat_volatile_workgroup_release_store(
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
    i32 %in, ptr %out) {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-wavefront.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-wavefront.ll
index 42f7e6707af0c..350ab8d48da81 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-wavefront.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-wavefront.ll
@@ -124,7 +124,6 @@ define amdgpu_kernel void @flat_wavefront_unordered_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_unordered_load:
@@ -136,7 +135,6 @@ define amdgpu_kernel void @flat_wavefront_unordered_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -259,7 +257,6 @@ define amdgpu_kernel void @flat_wavefront_monotonic_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_monotonic_load:
@@ -271,7 +268,6 @@ define amdgpu_kernel void @flat_wavefront_monotonic_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -394,7 +390,6 @@ define amdgpu_kernel void @flat_wavefront_acquire_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_acquire_load:
@@ -406,7 +401,6 @@ define amdgpu_kernel void @flat_wavefront_acquire_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -529,7 +523,6 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_load:
@@ -541,7 +534,6 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -646,7 +638,6 @@ define amdgpu_kernel void @flat_wavefront_unordered_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_unordered_store:
@@ -658,7 +649,6 @@ define amdgpu_kernel void @flat_wavefront_unordered_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -762,7 +752,6 @@ define amdgpu_kernel void @flat_wavefront_monotonic_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_monotonic_store:
@@ -774,7 +763,6 @@ define amdgpu_kernel void @flat_wavefront_monotonic_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -878,7 +866,6 @@ define amdgpu_kernel void @flat_wavefront_release_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_release_store:
@@ -890,7 +877,6 @@ define amdgpu_kernel void @flat_wavefront_release_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -994,7 +980,6 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_store:
@@ -1006,7 +991,6 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -1110,7 +1094,6 @@ define amdgpu_kernel void @flat_wavefront_monotonic_atomicrmw(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_monotonic_atomicrmw:
@@ -1122,7 +1105,6 @@ define amdgpu_kernel void @flat_wavefront_monotonic_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -1226,7 +1208,6 @@ define amdgpu_kernel void @flat_wavefront_acquire_atomicrmw(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_acquire_atomicrmw:
@@ -1238,7 +1219,6 @@ define amdgpu_kernel void @flat_wavefront_acquire_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -1342,7 +1322,6 @@ define amdgpu_kernel void @flat_wavefront_release_atomicrmw(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_release_atomicrmw:
@@ -1354,7 +1333,6 @@ define amdgpu_kernel void @flat_wavefront_release_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -1458,7 +1436,6 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_atomicrmw(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_acq_rel_atomicrmw:
@@ -1470,7 +1447,6 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -1574,7 +1550,6 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_atomicrmw(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_atomicrmw:
@@ -1586,7 +1561,6 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -1708,7 +1682,6 @@ define amdgpu_kernel void @flat_wavefront_acquire_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_acquire_ret_atomicrmw:
@@ -1722,7 +1695,6 @@ define amdgpu_kernel void @flat_wavefront_acquire_ret_atomicrmw(
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -1845,7 +1817,6 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_acq_rel_ret_atomicrmw:
@@ -1859,7 +1830,6 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_ret_atomicrmw(
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -1982,7 +1952,6 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_ret_atomicrmw:
@@ -1996,7 +1965,6 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_ret_atomicrmw(
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -2101,7 +2069,6 @@ define amdgpu_kernel void @flat_wavefront_monotonic_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_monotonic_monotonic_cmpxchg:
@@ -2111,7 +2078,6 @@ define amdgpu_kernel void @flat_wavefront_monotonic_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -2216,7 +2182,6 @@ define amdgpu_kernel void @flat_wavefront_acquire_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_acquire_monotonic_cmpxchg:
@@ -2226,7 +2191,6 @@ define amdgpu_kernel void @flat_wavefront_acquire_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -2331,7 +2295,6 @@ define amdgpu_kernel void @flat_wavefront_release_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_release_monotonic_cmpxchg:
@@ -2341,7 +2304,6 @@ define amdgpu_kernel void @flat_wavefront_release_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -2446,7 +2408,6 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_acq_rel_monotonic_cmpxchg:
@@ -2456,7 +2417,6 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -2561,7 +2521,6 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_monotonic_cmpxchg:
@@ -2571,7 +2530,6 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -2676,7 +2634,6 @@ define amdgpu_kernel void @flat_wavefront_monotonic_acquire_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_monotonic_acquire_cmpxchg:
@@ -2686,7 +2643,6 @@ define amdgpu_kernel void @flat_wavefront_monotonic_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -2791,7 +2747,6 @@ define amdgpu_kernel void @flat_wavefront_acquire_acquire_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_acquire_acquire_cmpxchg:
@@ -2801,7 +2756,6 @@ define amdgpu_kernel void @flat_wavefront_acquire_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -2906,7 +2860,6 @@ define amdgpu_kernel void @flat_wavefront_release_acquire_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_release_acquire_cmpxchg:
@@ -2916,7 +2869,6 @@ define amdgpu_kernel void @flat_wavefront_release_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -3021,7 +2973,6 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_acquire_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_acq_rel_acquire_cmpxchg:
@@ -3031,7 +2982,6 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -3136,7 +3086,6 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_acquire_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_acquire_cmpxchg:
@@ -3146,7 +3095,6 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -3251,7 +3199,6 @@ define amdgpu_kernel void @flat_wavefront_monotonic_seq_cst_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_monotonic_seq_cst_cmpxchg:
@@ -3261,7 +3208,6 @@ define amdgpu_kernel void @flat_wavefront_monotonic_seq_cst_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -3366,7 +3312,6 @@ define amdgpu_kernel void @flat_wavefront_acquire_seq_cst_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_acquire_seq_cst_cmpxchg:
@@ -3376,7 +3321,6 @@ define amdgpu_kernel void @flat_wavefront_acquire_seq_cst_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -3481,7 +3425,6 @@ define amdgpu_kernel void @flat_wavefront_release_seq_cst_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_release_seq_cst_cmpxchg:
@@ -3491,7 +3434,6 @@ define amdgpu_kernel void @flat_wavefront_release_seq_cst_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -3596,7 +3538,6 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_seq_cst_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_acq_rel_seq_cst_cmpxchg:
@@ -3606,7 +3547,6 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_seq_cst_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -3711,7 +3651,6 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_seq_cst_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_seq_cst_cmpxchg:
@@ -3721,7 +3660,6 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_seq_cst_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -3852,7 +3790,6 @@ define amdgpu_kernel void @flat_wavefront_monotonic_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_monotonic_monotonic_ret_cmpxchg:
@@ -3864,7 +3801,6 @@ define amdgpu_kernel void @flat_wavefront_monotonic_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -3997,7 +3933,6 @@ define amdgpu_kernel void @flat_wavefront_acquire_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_acquire_monotonic_ret_cmpxchg:
@@ -4009,7 +3944,6 @@ define amdgpu_kernel void @flat_wavefront_acquire_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -4142,7 +4076,6 @@ define amdgpu_kernel void @flat_wavefront_release_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_release_monotonic_ret_cmpxchg:
@@ -4154,7 +4087,6 @@ define amdgpu_kernel void @flat_wavefront_release_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -4287,7 +4219,6 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_acq_rel_monotonic_ret_cmpxchg:
@@ -4299,7 +4230,6 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -4432,7 +4362,6 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_monotonic_ret_cmpxchg:
@@ -4444,7 +4373,6 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -4577,7 +4505,6 @@ define amdgpu_kernel void @flat_wavefront_monotonic_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_monotonic_acquire_ret_cmpxchg:
@@ -4589,7 +4516,6 @@ define amdgpu_kernel void @flat_wavefront_monotonic_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -4722,7 +4648,6 @@ define amdgpu_kernel void @flat_wavefront_acquire_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_acquire_acquire_ret_cmpxchg:
@@ -4734,7 +4659,6 @@ define amdgpu_kernel void @flat_wavefront_acquire_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -4867,7 +4791,6 @@ define amdgpu_kernel void @flat_wavefront_release_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_release_acquire_ret_cmpxchg:
@@ -4879,7 +4802,6 @@ define amdgpu_kernel void @flat_wavefront_release_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5012,7 +4934,6 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_acq_rel_acquire_ret_cmpxchg:
@@ -5024,7 +4945,6 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5157,7 +5077,6 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_acquire_ret_cmpxchg:
@@ -5169,7 +5088,6 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5302,7 +5220,6 @@ define amdgpu_kernel void @flat_wavefront_monotonic_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_monotonic_seq_cst_ret_cmpxchg:
@@ -5314,7 +5231,6 @@ define amdgpu_kernel void @flat_wavefront_monotonic_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5447,7 +5363,6 @@ define amdgpu_kernel void @flat_wavefront_acquire_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_acquire_seq_cst_ret_cmpxchg:
@@ -5459,7 +5374,6 @@ define amdgpu_kernel void @flat_wavefront_acquire_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5592,7 +5506,6 @@ define amdgpu_kernel void @flat_wavefront_release_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_release_seq_cst_ret_cmpxchg:
@@ -5604,7 +5517,6 @@ define amdgpu_kernel void @flat_wavefront_release_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5737,7 +5649,6 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_acq_rel_seq_cst_ret_cmpxchg:
@@ -5749,7 +5660,6 @@ define amdgpu_kernel void @flat_wavefront_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5882,7 +5792,6 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_seq_cst_seq_cst_ret_cmpxchg:
@@ -5894,7 +5803,6 @@ define amdgpu_kernel void @flat_wavefront_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -6019,7 +5927,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_unordered_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_unordered_load:
@@ -6031,7 +5938,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_unordered_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -6154,7 +6060,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_load:
@@ -6166,7 +6071,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -6289,7 +6193,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_load:
@@ -6301,7 +6204,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -6424,7 +6326,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_load:
@@ -6436,7 +6337,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -6541,7 +6441,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_unordered_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_unordered_store:
@@ -6553,7 +6452,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_unordered_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -6657,7 +6555,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_store:
@@ -6669,7 +6566,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -6773,7 +6669,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_release_store:
@@ -6785,7 +6680,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -6889,7 +6783,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_store:
@@ -6901,7 +6794,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -7005,7 +6897,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_atomicrmw(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_atomicrmw:
@@ -7017,7 +6908,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -7121,7 +7011,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_atomicrmw(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_atomicrmw:
@@ -7133,7 +7022,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -7237,7 +7125,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_atomicrmw(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_release_atomicrmw:
@@ -7249,7 +7136,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -7353,7 +7239,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_atomicrmw(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_acq_rel_atomicrmw:
@@ -7365,7 +7250,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -7469,7 +7353,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_atomicrmw(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_atomicrmw:
@@ -7481,7 +7364,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -7603,7 +7485,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_ret_atomicrmw:
@@ -7617,7 +7498,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_ret_atomicrmw(
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -7740,7 +7620,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_acq_rel_ret_atomicrmw:
@@ -7754,7 +7633,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_ret_atomicrmw(
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -7877,7 +7755,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_ret_atomicrmw:
@@ -7891,7 +7768,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_ret_atomicrmw(
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -7996,7 +7872,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_monotonic_cmpxchg:
@@ -8006,7 +7881,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -8111,7 +7985,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_monotonic_cmpxchg:
@@ -8121,7 +7994,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -8226,7 +8098,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_release_monotonic_cmpxchg:
@@ -8236,7 +8107,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -8341,7 +8211,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_acq_rel_monotonic_cmpxchg:
@@ -8351,7 +8220,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -8456,7 +8324,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_monotonic_cmpxchg:
@@ -8466,7 +8333,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -8571,7 +8437,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_acquire_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_acquire_cmpxchg:
@@ -8581,7 +8446,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -8686,7 +8550,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_acquire_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_acquire_cmpxchg:
@@ -8696,7 +8559,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -8801,7 +8663,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_acquire_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_release_acquire_cmpxchg:
@@ -8811,7 +8672,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -8916,7 +8776,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_acquire_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_acq_rel_acquire_cmpxchg:
@@ -8926,7 +8785,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -9031,7 +8889,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_acquire_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_acquire_cmpxchg:
@@ -9041,7 +8898,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -9146,7 +9002,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_seq_cst_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_seq_cst_cmpxchg:
@@ -9156,7 +9011,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_seq_cst_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -9261,7 +9115,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_seq_cst_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_seq_cst_cmpxchg:
@@ -9271,7 +9124,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_seq_cst_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -9376,7 +9228,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_seq_cst_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_release_seq_cst_cmpxchg:
@@ -9386,7 +9237,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_seq_cst_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -9491,7 +9341,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg:
@@ -9501,7 +9350,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_seq_cst_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -9606,7 +9454,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg:
@@ -9616,7 +9463,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_seq_cst_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -9747,7 +9593,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg:
@@ -9759,7 +9604,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_monotonic_ret_cmpxchg
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -9892,7 +9736,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg:
@@ -9904,7 +9747,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -10037,7 +9879,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg:
@@ -10049,7 +9890,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -10182,7 +10022,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg:
@@ -10194,7 +10033,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -10327,7 +10165,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg:
@@ -10339,7 +10176,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -10472,7 +10308,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_acquire_ret_cmpxchg:
@@ -10484,7 +10319,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -10617,7 +10451,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_release_acquire_ret_cmpxchg:
@@ -10629,7 +10462,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -10762,7 +10594,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg:
@@ -10774,7 +10605,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -10907,7 +10737,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg:
@@ -10919,7 +10748,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -11052,7 +10880,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg:
@@ -11064,7 +10891,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
    ptr %out, i32 %in, i32 %old) {
 entry:
@@ -11197,7 +11023,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg:
@@ -11209,7 +11034,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
    ptr %out, i32 %in, i32 %old) {
 entry:
@@ -11342,7 +11166,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_release_seq_cst_ret_cmpxchg:
@@ -11354,7 +11177,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
    ptr %out, i32 %in, i32 %old) {
 entry:
@@ -11487,7 +11309,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg:
@@ -11499,7 +11320,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_acq_relc_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
    ptr %out, i32 %in, i32 %old) {
 entry:
@@ -11632,7 +11452,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg:
@@ -11644,7 +11463,6 @@ define amdgpu_kernel void @flat_wavefront_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-workgroup.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-workgroup.ll
index 9f5996ffc14e4..0fd0cffcbb28b 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-workgroup.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-flat-workgroup.ll
@@ -124,7 +124,6 @@ define amdgpu_kernel void @flat_workgroup_unordered_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_unordered_load:
@@ -136,7 +135,6 @@ define amdgpu_kernel void @flat_workgroup_unordered_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -259,7 +257,6 @@ define amdgpu_kernel void @flat_workgroup_monotonic_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_monotonic_load:
@@ -271,7 +268,6 @@ define amdgpu_kernel void @flat_workgroup_monotonic_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -403,7 +399,6 @@ define amdgpu_kernel void @flat_workgroup_acquire_load(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_acquire_load:
@@ -416,7 +411,6 @@ define amdgpu_kernel void @flat_workgroup_acquire_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -559,7 +553,6 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_load(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_seq_cst_load:
@@ -573,7 +566,6 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -678,7 +670,6 @@ define amdgpu_kernel void @flat_workgroup_unordered_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_unordered_store:
@@ -690,7 +681,6 @@ define amdgpu_kernel void @flat_workgroup_unordered_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -794,7 +784,6 @@ define amdgpu_kernel void @flat_workgroup_monotonic_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_monotonic_store:
@@ -806,7 +795,6 @@ define amdgpu_kernel void @flat_workgroup_monotonic_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -921,7 +909,6 @@ define amdgpu_kernel void @flat_workgroup_release_store(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_release_store:
@@ -934,7 +921,6 @@ define amdgpu_kernel void @flat_workgroup_release_store(
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -1049,7 +1035,6 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_store(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_seq_cst_store:
@@ -1062,7 +1047,6 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_store(
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -1166,7 +1150,6 @@ define amdgpu_kernel void @flat_workgroup_monotonic_atomicrmw(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_monotonic_atomicrmw:
@@ -1178,7 +1161,6 @@ define amdgpu_kernel void @flat_workgroup_monotonic_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -1309,7 +1291,6 @@ define amdgpu_kernel void @flat_workgroup_acquire_atomicrmw(
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -1424,7 +1405,6 @@ define amdgpu_kernel void @flat_workgroup_release_atomicrmw(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_release_atomicrmw:
@@ -1437,7 +1417,6 @@ define amdgpu_kernel void @flat_workgroup_release_atomicrmw(
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -1580,7 +1559,6 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_atomicrmw(
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -1723,7 +1701,6 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_atomicrmw(
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -1849,7 +1826,6 @@ define amdgpu_kernel void @flat_workgroup_acquire_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_acquire_ret_atomicrmw:
@@ -1863,7 +1839,6 @@ define amdgpu_kernel void @flat_workgroup_acquire_ret_atomicrmw(
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -2001,7 +1976,6 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_acq_rel_ret_atomicrmw:
@@ -2016,7 +1990,6 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_ret_atomicrmw(
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -2154,7 +2127,6 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_seq_cst_ret_atomicrmw:
@@ -2169,7 +2141,6 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_ret_atomicrmw(
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -2274,7 +2245,6 @@ define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_monotonic_monotonic_cmpxchg:
@@ -2284,7 +2254,6 @@ define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -2414,7 +2383,6 @@ define amdgpu_kernel void @flat_workgroup_acquire_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -2530,7 +2498,6 @@ define amdgpu_kernel void @flat_workgroup_release_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_release_monotonic_cmpxchg:
@@ -2541,7 +2508,6 @@ define amdgpu_kernel void @flat_workgroup_release_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -2683,7 +2649,6 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -2825,7 +2790,6 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -2955,7 +2919,6 @@ define amdgpu_kernel void @flat_workgroup_monotonic_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -3085,7 +3048,6 @@ define amdgpu_kernel void @flat_workgroup_acquire_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -3227,7 +3189,6 @@ define amdgpu_kernel void @flat_workgroup_release_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -3369,7 +3330,6 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -3511,7 +3471,6 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -3653,7 +3612,6 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_cmpxchg(
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -3784,7 +3742,6 @@ define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_monotonic_monotonic_ret_cmpxchg:
@@ -3796,7 +3753,6 @@ define amdgpu_kernel void @flat_workgroup_monotonic_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -3936,7 +3892,6 @@ define amdgpu_kernel void @flat_workgroup_acquire_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_acquire_monotonic_ret_cmpxchg:
@@ -3948,7 +3903,6 @@ define amdgpu_kernel void @flat_workgroup_acquire_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -4092,7 +4046,6 @@ define amdgpu_kernel void @flat_workgroup_release_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_release_monotonic_ret_cmpxchg:
@@ -4105,7 +4058,6 @@ define amdgpu_kernel void @flat_workgroup_release_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -4256,7 +4208,6 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_acq_rel_monotonic_ret_cmpxchg:
@@ -4269,7 +4220,6 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -4420,7 +4370,6 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_seq_cst_monotonic_ret_cmpxchg:
@@ -4433,7 +4382,6 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -4573,7 +4521,6 @@ define amdgpu_kernel void @flat_workgroup_monotonic_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_monotonic_acquire_ret_cmpxchg:
@@ -4585,7 +4532,6 @@ define amdgpu_kernel void @flat_workgroup_monotonic_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -4725,7 +4671,6 @@ define amdgpu_kernel void @flat_workgroup_acquire_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_acquire_acquire_ret_cmpxchg:
@@ -4737,7 +4682,6 @@ define amdgpu_kernel void @flat_workgroup_acquire_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -4888,7 +4832,6 @@ define amdgpu_kernel void @flat_workgroup_release_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_release_acquire_ret_cmpxchg:
@@ -4901,7 +4844,6 @@ define amdgpu_kernel void @flat_workgroup_release_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5052,7 +4994,6 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_acq_rel_acquire_ret_cmpxchg:
@@ -5065,7 +5006,6 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5216,7 +5156,6 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_seq_cst_acquire_ret_cmpxchg:
@@ -5229,7 +5168,6 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5380,7 +5318,6 @@ define amdgpu_kernel void @flat_workgroup_monotonic_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_monotonic_seq_cst_ret_cmpxchg:
@@ -5393,7 +5330,6 @@ define amdgpu_kernel void @flat_workgroup_monotonic_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5544,7 +5480,6 @@ define amdgpu_kernel void @flat_workgroup_acquire_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_acquire_seq_cst_ret_cmpxchg:
@@ -5557,7 +5492,6 @@ define amdgpu_kernel void @flat_workgroup_acquire_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5708,7 +5642,6 @@ define amdgpu_kernel void @flat_workgroup_release_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_release_seq_cst_ret_cmpxchg:
@@ -5721,7 +5654,6 @@ define amdgpu_kernel void @flat_workgroup_release_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -5872,7 +5804,6 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_acq_rel_seq_cst_ret_cmpxchg:
@@ -5885,7 +5816,6 @@ define amdgpu_kernel void @flat_workgroup_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -6036,7 +5966,6 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_seq_cst_seq_cst_ret_cmpxchg:
@@ -6049,7 +5978,6 @@ define amdgpu_kernel void @flat_workgroup_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -6174,7 +6102,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_unordered_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_unordered_load:
@@ -6186,7 +6113,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_unordered_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -6309,7 +6235,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_load:
@@ -6321,7 +6246,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -6450,7 +6374,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_load:
@@ -6462,7 +6385,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -6597,7 +6519,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_load(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_load:
@@ -6609,7 +6530,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_load(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %in, ptr %out) {
 entry:
@@ -6714,7 +6634,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_unordered_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_unordered_store:
@@ -6726,7 +6645,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_unordered_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -6830,7 +6748,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_store(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_store:
@@ -6842,7 +6759,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -6952,7 +6868,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_store(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_release_store:
@@ -6964,7 +6879,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -7074,7 +6988,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_store(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_store:
@@ -7086,7 +6999,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_store(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     i32 %in, ptr %out) {
 entry:
@@ -7190,7 +7102,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_atomicrmw(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_atomicrmw:
@@ -7202,7 +7113,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -7325,7 +7235,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -7435,7 +7344,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_atomicrmw(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-WGP-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_release_atomicrmw:
@@ -7447,7 +7355,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -7576,7 +7483,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -7705,7 +7611,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_atomicrmw(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v2, s0
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -7833,7 +7738,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_ret_atomicrmw:
@@ -7847,7 +7751,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_ret_atomicrmw(
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -7982,7 +7885,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_ret_atomicrmw:
@@ -7996,7 +7898,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_ret_atomicrmw(
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -8131,7 +8032,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_ret_atomicrmw(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_ret_atomicrmw:
@@ -8145,7 +8045,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_ret_atomicrmw(
 ; GFX11-CU-NEXT:    flat_atomic_swap_b32 v2, v[0:1], v2 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in) {
 entry:
@@ -8250,7 +8149,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-WGP-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_monotonic_cmpxchg:
@@ -8260,7 +8158,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -8382,7 +8279,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -8493,7 +8389,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_cmpxchg(
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_release_monotonic_cmpxchg:
@@ -8503,7 +8398,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -8631,7 +8525,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -8759,7 +8652,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -8881,7 +8773,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -9003,7 +8894,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -9131,7 +9021,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -9259,7 +9148,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -9387,7 +9275,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -9515,7 +9402,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -9643,7 +9529,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -9771,7 +9656,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -9899,7 +9783,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -10027,7 +9910,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_cmpxchg(
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
 ; GFX11-CU-NEXT:    v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:16
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -10158,7 +10040,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg:
@@ -10170,7 +10051,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonicmonotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -10309,7 +10189,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg:
@@ -10321,7 +10200,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -10460,7 +10338,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-WGP-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_release_monotonic_ret_cmpxchg:
@@ -10472,7 +10349,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -10617,7 +10493,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg:
@@ -10629,7 +10504,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -10774,7 +10648,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg:
@@ -10786,7 +10659,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_monotonic_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -10925,7 +10797,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg:
@@ -10937,7 +10808,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -11076,7 +10946,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_acquire_ret_cmpxchg:
@@ -11088,7 +10957,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -11233,7 +11101,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_release_acquire_ret_cmpxchg:
@@ -11245,7 +11112,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -11390,7 +11256,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg:
@@ -11402,7 +11267,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -11547,7 +11411,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg:
@@ -11559,7 +11422,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_acquire_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -11704,7 +11566,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg:
@@ -11716,7 +11577,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_monotonic_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -11861,7 +11721,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg:
@@ -11873,7 +11732,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_acquire_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -12018,7 +11876,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_release_seq_cst_ret_cmpxchg:
@@ -12030,7 +11887,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_release_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -12175,7 +12031,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg:
@@ -12187,7 +12042,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_acq_rel_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
@@ -12332,7 +12186,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX11-WGP-NEXT:    buffer_gl0_inv
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg:
@@ -12344,7 +12197,6 @@ define amdgpu_kernel void @flat_workgroup_one_as_seq_cst_seq_cst_ret_cmpxchg(
 ; GFX11-CU-NEXT:    flat_atomic_cmpswap_b32 v2, v[0:1], v[2:3] offset:16 glc
 ; GFX11-CU-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-CU-NEXT:    flat_store_b32 v[0:1], v2
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr %out, i32 %in, i32 %old) {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
index 7cff01e543de7..9cbba21a522c8 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
@@ -515,7 +515,6 @@ define amdgpu_kernel void @private_nontemporal_store_0(
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX11-WGP-NEXT:    scratch_store_b32 off, v0, s0 glc slc dlc
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: private_nontemporal_store_0:
@@ -528,7 +527,6 @@ define amdgpu_kernel void @private_nontemporal_store_0(
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX11-CU-NEXT:    scratch_store_b32 off, v0, s0 glc slc dlc
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(5) %out) {
 entry:
@@ -692,7 +690,6 @@ define amdgpu_kernel void @private_nontemporal_store_1(
 ; GFX11-WGP-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX11-WGP-NEXT:    scratch_store_b32 v0, v1, off glc slc dlc
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: private_nontemporal_store_1:
@@ -706,7 +703,6 @@ define amdgpu_kernel void @private_nontemporal_store_1(
 ; GFX11-CU-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX11-CU-NEXT:    scratch_store_b32 v0, v1, off glc slc dlc
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(5) %out) {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-volatile.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-volatile.ll
index 424b8ec764f82..2160ce5bd37c0 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-volatile.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-volatile.ll
@@ -359,7 +359,6 @@ define amdgpu_kernel void @private_volatile_store_0(
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX11-WGP-NEXT:    scratch_store_b32 off, v0, s0 dlc
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: private_volatile_store_0:
@@ -373,7 +372,6 @@ define amdgpu_kernel void @private_volatile_store_0(
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v0, s1
 ; GFX11-CU-NEXT:    scratch_store_b32 off, v0, s0 dlc
 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(5) %out) {
 entry:
@@ -489,7 +487,6 @@ define amdgpu_kernel void @private_volatile_store_1(
 ; GFX11-WGP-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX11-WGP-NEXT:    scratch_store_b32 v0, v1, off dlc
 ; GFX11-WGP-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-WGP-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-WGP-NEXT:    s_endpgm
 ;
 ; GFX11-CU-LABEL: private_volatile_store_1:
@@ -504,7 +501,6 @@ define amdgpu_kernel void @private_volatile_store_1(
 ; GFX11-CU-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX11-CU-NEXT:    scratch_store_b32 v0, v1, off dlc
 ; GFX11-CU-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-CU-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-CU-NEXT:    s_endpgm
     ptr addrspace(1) %in, ptr addrspace(5) %out) {
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/offset-split-flat.ll b/llvm/test/CodeGen/AMDGPU/offset-split-flat.ll
index 0beee2f066101..a8cbd05e429c7 100644
--- a/llvm/test/CodeGen/AMDGPU/offset-split-flat.ll
+++ b/llvm/test/CodeGen/AMDGPU/offset-split-flat.ll
@@ -1806,7 +1806,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_1(ptr %p) {
 ; GFX11-NEXT:    flat_load_u8 v0, v[0:1] offset:1 glc dlc
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr i8, ptr %p, i64 1
   %load = load volatile i8, ptr %gep, align 1
@@ -1847,7 +1846,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_11bit_max(ptr %p) {
 ; GFX11-NEXT:    flat_load_u8 v0, v[0:1] offset:2047 glc dlc
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr i8, ptr %p, i64 2047
   %load = load volatile i8, ptr %gep, align 1
@@ -1888,7 +1886,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_12bit_max(ptr %p) {
 ; GFX11-NEXT:    flat_load_u8 v0, v[0:1] offset:4095 glc dlc
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr i8, ptr %p, i64 4095
   %load = load volatile i8, ptr %gep, align 1
@@ -1933,7 +1930,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_13bit_max(ptr %p) {
 ; GFX11-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:4095 glc dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX9-GISEL-LABEL: flat_inst_salu_offset_13bit_max:
@@ -1960,7 +1956,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_13bit_max(ptr %p) {
 ; GFX11-GISEL-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
   %gep = getelementptr i8, ptr %p, i64 8191
   %load = load volatile i8, ptr %gep, align 1
@@ -2005,7 +2000,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_neg_11bit_max(ptr %p) {
 ; GFX11-SDAG-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX9-GISEL-LABEL: flat_inst_salu_offset_neg_11bit_max:
@@ -2032,7 +2026,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_neg_11bit_max(ptr %p) {
 ; GFX11-GISEL-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
   %gep = getelementptr i8, ptr %p, i64 -2048
   %load = load volatile i8, ptr %gep, align 1
@@ -2077,7 +2070,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_neg_12bit_max(ptr %p) {
 ; GFX11-SDAG-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX9-GISEL-LABEL: flat_inst_salu_offset_neg_12bit_max:
@@ -2104,7 +2096,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_neg_12bit_max(ptr %p) {
 ; GFX11-GISEL-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
   %gep = getelementptr i8, ptr %p, i64 -4096
   %load = load volatile i8, ptr %gep, align 1
@@ -2149,7 +2140,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_neg_13bit_max(ptr %p) {
 ; GFX11-SDAG-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX9-GISEL-LABEL: flat_inst_salu_offset_neg_13bit_max:
@@ -2176,7 +2166,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_neg_13bit_max(ptr %p) {
 ; GFX11-GISEL-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
   %gep = getelementptr i8, ptr %p, i64 -8192
   %load = load volatile i8, ptr %gep, align 1
@@ -2217,7 +2206,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_2x_11bit_max(ptr %p) {
 ; GFX11-NEXT:    flat_load_u8 v0, v[0:1] offset:4095 glc dlc
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
   %gep = getelementptr i8, ptr %p, i64 4095
   %load = load volatile i8, ptr %gep, align 1
@@ -2262,7 +2250,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_2x_12bit_max(ptr %p) {
 ; GFX11-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:4095 glc dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX9-GISEL-LABEL: flat_inst_salu_offset_2x_12bit_max:
@@ -2289,7 +2276,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_2x_12bit_max(ptr %p) {
 ; GFX11-GISEL-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
   %gep = getelementptr i8, ptr %p, i64 8191
   %load = load volatile i8, ptr %gep, align 1
@@ -2334,7 +2320,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_2x_13bit_max(ptr %p) {
 ; GFX11-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:4095 glc dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX9-GISEL-LABEL: flat_inst_salu_offset_2x_13bit_max:
@@ -2361,7 +2346,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_2x_13bit_max(ptr %p) {
 ; GFX11-GISEL-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
   %gep = getelementptr i8, ptr %p, i64 16383
   %load = load volatile i8, ptr %gep, align 1
@@ -2406,7 +2390,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_2x_neg_11bit_max(ptr %p) {
 ; GFX11-SDAG-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX9-GISEL-LABEL: flat_inst_salu_offset_2x_neg_11bit_max:
@@ -2433,7 +2416,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_2x_neg_11bit_max(ptr %p) {
 ; GFX11-GISEL-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
   %gep = getelementptr i8, ptr %p, i64 -4096
   %load = load volatile i8, ptr %gep, align 1
@@ -2478,7 +2460,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_2x_neg_12bit_max(ptr %p) {
 ; GFX11-SDAG-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX9-GISEL-LABEL: flat_inst_salu_offset_2x_neg_12bit_max:
@@ -2505,7 +2486,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_2x_neg_12bit_max(ptr %p) {
 ; GFX11-GISEL-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
   %gep = getelementptr i8, ptr %p, i64 -8192
   %load = load volatile i8, ptr %gep, align 1
@@ -2550,7 +2530,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_2x_neg_13bit_max(ptr %p) {
 ; GFX11-SDAG-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX9-GISEL-LABEL: flat_inst_salu_offset_2x_neg_13bit_max:
@@ -2577,7 +2556,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_2x_neg_13bit_max(ptr %p) {
 ; GFX11-GISEL-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
   %gep = getelementptr i8, ptr %p, i64 -16384
   %load = load volatile i8, ptr %gep, align 1
@@ -2622,7 +2600,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_11bit_split0(ptr %p) {
 ; GFX11-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:2047 glc dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX9-GISEL-LABEL: flat_inst_salu_offset_64bit_11bit_split0:
@@ -2649,7 +2626,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_11bit_split0(ptr %p) {
 ; GFX11-GISEL-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
   %gep = getelementptr i8, ptr %p, i64 8589936639
   %load = load volatile i8, ptr %gep, align 1
@@ -2694,7 +2670,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_11bit_split1(ptr %p) {
 ; GFX11-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:2048 glc dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX9-GISEL-LABEL: flat_inst_salu_offset_64bit_11bit_split1:
@@ -2721,7 +2696,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_11bit_split1(ptr %p) {
 ; GFX11-GISEL-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
   %gep = getelementptr i8, ptr %p, i64 8589936640
   %load = load volatile i8, ptr %gep, align 1
@@ -2766,7 +2740,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_12bit_split0(ptr %p) {
 ; GFX11-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:4095 glc dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX9-GISEL-LABEL: flat_inst_salu_offset_64bit_12bit_split0:
@@ -2793,7 +2766,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_12bit_split0(ptr %p) {
 ; GFX11-GISEL-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
   %gep = getelementptr i8, ptr %p, i64 8589938687
   %load = load volatile i8, ptr %gep, align 1
@@ -2839,7 +2811,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_12bit_split1(ptr %p) {
 ; GFX11-SDAG-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX9-GISEL-LABEL: flat_inst_salu_offset_64bit_12bit_split1:
@@ -2866,7 +2837,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_12bit_split1(ptr %p) {
 ; GFX11-GISEL-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
   %gep = getelementptr i8, ptr %p, i64 8589938688
   %load = load volatile i8, ptr %gep, align 1
@@ -2912,7 +2882,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_13bit_split0(ptr %p) {
 ; GFX11-SDAG-NEXT:    flat_load_u8 v0, v[0:1] offset:4095 glc dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX9-GISEL-LABEL: flat_inst_salu_offset_64bit_13bit_split0:
@@ -2939,7 +2908,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_13bit_split0(ptr %p) {
 ; GFX11-GISEL-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
   %gep = getelementptr i8, ptr %p, i64 8589942783
   %load = load volatile i8, ptr %gep, align 1
@@ -2985,7 +2953,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_13bit_split1(ptr %p) {
 ; GFX11-SDAG-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX9-GISEL-LABEL: flat_inst_salu_offset_64bit_13bit_split1:
@@ -3012,7 +2979,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_13bit_split1(ptr %p) {
 ; GFX11-GISEL-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
   %gep = getelementptr i8, ptr %p, i64 8589942784
   %load = load volatile i8, ptr %gep, align 1
@@ -3060,7 +3026,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_11bit_neg_high_split0(ptr
 ; GFX11-SDAG-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX9-GISEL-LABEL: flat_inst_salu_offset_64bit_11bit_neg_high_split0:
@@ -3087,7 +3052,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_11bit_neg_high_split0(ptr
 ; GFX11-GISEL-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
   %gep = getelementptr i8, ptr %p, i64 -9223372036854773761
   %load = load volatile i8, ptr %gep, align 1
@@ -3135,7 +3099,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_11bit_neg_high_split1(ptr
 ; GFX11-SDAG-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX9-GISEL-LABEL: flat_inst_salu_offset_64bit_11bit_neg_high_split1:
@@ -3162,7 +3125,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_11bit_neg_high_split1(ptr
 ; GFX11-GISEL-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
   %gep = getelementptr i8, ptr %p, i64 -9223372036854773760
   %load = load volatile i8, ptr %gep, align 1
@@ -3210,7 +3172,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_12bit_neg_high_split0(ptr
 ; GFX11-SDAG-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX9-GISEL-LABEL: flat_inst_salu_offset_64bit_12bit_neg_high_split0:
@@ -3237,7 +3198,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_12bit_neg_high_split0(ptr
 ; GFX11-GISEL-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
   %gep = getelementptr i8, ptr %p, i64 -9223372036854771713
   %load = load volatile i8, ptr %gep, align 1
@@ -3285,7 +3245,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_12bit_neg_high_split1(ptr
 ; GFX11-SDAG-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX9-GISEL-LABEL: flat_inst_salu_offset_64bit_12bit_neg_high_split1:
@@ -3312,7 +3271,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_12bit_neg_high_split1(ptr
 ; GFX11-GISEL-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
   %gep = getelementptr i8, ptr %p, i64 -9223372036854771712
   %load = load volatile i8, ptr %gep, align 1
@@ -3360,7 +3318,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_13bit_neg_high_split0(ptr
 ; GFX11-SDAG-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX9-GISEL-LABEL: flat_inst_salu_offset_64bit_13bit_neg_high_split0:
@@ -3387,7 +3344,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_13bit_neg_high_split0(ptr
 ; GFX11-GISEL-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
   %gep = getelementptr i8, ptr %p, i64 -9223372036854767617
   %load = load volatile i8, ptr %gep, align 1
@@ -3435,7 +3391,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_13bit_neg_high_split1(ptr
 ; GFX11-SDAG-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-SDAG-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-SDAG-NEXT:    s_endpgm
 ;
 ; GFX9-GISEL-LABEL: flat_inst_salu_offset_64bit_13bit_neg_high_split1:
@@ -3462,7 +3417,6 @@ define amdgpu_kernel void @flat_inst_salu_offset_64bit_13bit_neg_high_split1(ptr
 ; GFX11-GISEL-NEXT:    flat_load_u8 v0, v[0:1] glc dlc
 ; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; GFX11-GISEL-NEXT:    flat_store_b8 v[0:1], v0
-; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-GISEL-NEXT:    s_endpgm
   %gep = getelementptr i8, ptr %p, i64 -9223372036854767616
   %load = load volatile i8, ptr %gep, align 1
diff --git a/llvm/test/CodeGen/AMDGPU/stack-pointer-offset-relative-frameindex.ll b/llvm/test/CodeGen/AMDGPU/stack-pointer-offset-relative-frameindex.ll
index b243c499bf9e6..d52655a079161 100644
--- a/llvm/test/CodeGen/AMDGPU/stack-pointer-offset-relative-frameindex.ll
+++ b/llvm/test/CodeGen/AMDGPU/stack-pointer-offset-relative-frameindex.ll
@@ -104,7 +104,6 @@ define amdgpu_kernel void @kernel_background_evaluate(ptr addrspace(5) %kg, ptr
 ; MUBUF11-NEXT:    v_mad_u64_u32 v[0:1], null, v2, s0, 0x3039
 ; MUBUF11-NEXT:    scratch_store_b32 off, v0, s0
 ; MUBUF11-NEXT:  .LBB0_2: ; %shader_eval_surface.exit
-; MUBUF11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; MUBUF11-NEXT:    s_endpgm
 ;
 ; FLATSCR11-LABEL: kernel_background_evaluate:
@@ -132,7 +131,6 @@ define amdgpu_kernel void @kernel_background_evaluate(ptr addrspace(5) %kg, ptr
 ; FLATSCR11-NEXT:    v_mad_u64_u32 v[0:1], null, v2, s0, 0x3039
 ; FLATSCR11-NEXT:    scratch_store_b32 off, v0, s0
 ; FLATSCR11-NEXT:  .LBB0_2: ; %shader_eval_surface.exit
-; FLATSCR11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; FLATSCR11-NEXT:    s_endpgm
 entry:
   %sd = alloca < 1339 x i32>, align 8192, addrspace(5)
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-preexisting-vscnt.mir b/llvm/test/CodeGen/AMDGPU/waitcnt-preexisting-vscnt.mir
index 97dc6b60c0b4a..5f04e8cff880e 100644
--- a/llvm/test/CodeGen/AMDGPU/waitcnt-preexisting-vscnt.mir
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-preexisting-vscnt.mir
@@ -31,7 +31,6 @@ body:             |
     ; GFX11-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
     ; GFX11-NEXT: S_WAITCNT 7
     ; GFX11-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
-    ; GFX11-NEXT: S_SENDMSG 3, implicit $exec, implicit $m0
     ; GFX11-NEXT: S_ENDPGM 0
     GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
     S_WAITCNT_VSCNT undef $sgpr_null, 0
@@ -70,7 +69,6 @@ body:             |
     ; GFX11-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
     ; GFX11-NEXT: S_WAITCNT 7
     ; GFX11-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
-    ; GFX11-NEXT: S_SENDMSG 3, implicit $exec, implicit $m0
     ; GFX11-NEXT: S_ENDPGM 0
     GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
     S_WAITCNT_VSCNT undef $sgpr_null, 1
@@ -111,7 +109,6 @@ body:             |
     ; GFX11-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
     ; GFX11-NEXT: S_WAITCNT 7
     ; GFX11-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
-    ; GFX11-NEXT: S_SENDMSG 3, implicit $exec, implicit $m0
     ; GFX11-NEXT: S_ENDPGM 0
     GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
     S_WAITCNT 112
@@ -151,7 +148,6 @@ body:             |
     ; GFX11-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
     ; GFX11-NEXT: S_WAITCNT 7
     ; GFX11-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
-    ; GFX11-NEXT: S_SENDMSG 3, implicit $exec, implicit $m0
     ; GFX11-NEXT: S_ENDPGM 0
     GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
     S_WAITCNT_VSCNT undef $sgpr_null, 0
@@ -194,7 +190,6 @@ body:             |
     ; GFX11-NEXT: $vgpr0 = FLAT_LOAD_DWORD $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr
     ; GFX11-NEXT: S_WAITCNT 7
     ; GFX11-NEXT: FLAT_STORE_DWORD $vgpr0_vgpr1, $vgpr0, 0, 0, implicit $exec, implicit $flat_scr
-    ; GFX11-NEXT: S_SENDMSG 3, implicit $exec, implicit $m0
     ; GFX11-NEXT: S_ENDPGM 0
     GLOBAL_STORE_DWORD $vgpr0_vgpr1, $vgpr2, 0, 0, implicit $exec
     S_WAITCNT 0

From 58669354bf1f8eef39979e31915b1e212a3985c9 Mon Sep 17 00:00:00 2001
From: Scott Linder <Scott.Linder@amd.com>
Date: Wed, 17 May 2023 21:33:09 +0000
Subject: [PATCH 074/130] [DebugInfo] Add DW_OP_LLVM_user extension point

The extension codespace for DWARF expressions (DW_OP_LLVM_{lo,hi}_user)
has shrunk over time, as no extension is ever "retired" in practice. To
facilitate future extensions, this patch reserves one open opcode as an extension
point (0xfe), which is followed by a ULEB128-encoded SubOperation, and
then by the subop's operands.

There is some prior-art, namely DW_OP_AARCH64_operation
(see https://github.com/ARM-software/abi-aa/blob/edd7460d87493fff124b8b5713acf71ffc06ee91/aadwarf64/aadwarf64.rst#45dwarf-expression-operations).

This version makes some different tradeoffs, opting to use a ULEB128 for
the subop encoding for future-proofing.

Reviewed By: #debug-info, dblaikie

Differential Revision: https://reviews.llvm.org/D147271
---
 llvm/include/llvm/BinaryFormat/Dwarf.def      | 26 ++++++++++-
 llvm/include/llvm/BinaryFormat/Dwarf.h        |  9 ++++
 .../llvm/DebugInfo/DWARF/DWARFExpression.h    |  4 ++
 llvm/lib/BinaryFormat/Dwarf.cpp               | 34 ++++++++++++++
 llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp    |  1 +
 llvm/lib/DWARFLinker/DWARFLinker.cpp          |  1 +
 llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp  | 46 ++++++++++++++++++-
 .../llvm-dwarfdump/X86/DW_OP_LLVM_user.s      | 16 +++++++
 .../DWARFExpressionCompactPrinterTest.cpp     |  9 ++++
 9 files changed, 144 insertions(+), 2 deletions(-)
 create mode 100644 llvm/test/tools/llvm-dwarfdump/X86/DW_OP_LLVM_user.s

diff --git a/llvm/include/llvm/BinaryFormat/Dwarf.def b/llvm/include/llvm/BinaryFormat/Dwarf.def
index 1409568b8664f..d0357bec0bbf5 100644
--- a/llvm/include/llvm/BinaryFormat/Dwarf.def
+++ b/llvm/include/llvm/BinaryFormat/Dwarf.def
@@ -13,7 +13,8 @@
 // TODO: Add other DW-based macros.
 #if !(                                                                         \
     defined HANDLE_DW_TAG || defined HANDLE_DW_AT || defined HANDLE_DW_FORM || \
-    defined HANDLE_DW_OP || defined HANDLE_DW_LANG || defined HANDLE_DW_ATE || \
+    defined HANDLE_DW_OP || defined HANDLE_DW_OP_LLVM_USEROP ||                    \
+    defined HANDLE_DW_LANG || defined HANDLE_DW_ATE ||                         \
     defined HANDLE_DW_VIRTUALITY || defined HANDLE_DW_DEFAULTED ||             \
     defined HANDLE_DW_CC || defined HANDLE_DW_LNS || defined HANDLE_DW_LNE ||  \
     defined HANDLE_DW_LNCT || defined HANDLE_DW_MACRO ||                       \
@@ -52,6 +53,10 @@
 #define HANDLE_DW_OP(ID, NAME, VERSION, VENDOR)
 #endif
 
+#ifndef HANDLE_DW_OP_LLVM_USEROP
+#define HANDLE_DW_OP_LLVM_USEROP(ID, NAME)
+#endif
+
 #ifndef HANDLE_DW_LANG
 #define HANDLE_DW_LANG(ID, NAME, LOWER_BOUND, VERSION, VENDOR)
 #endif
@@ -874,6 +879,24 @@ HANDLE_DW_OP(0xf8, PGI_omp_thread_num, 0, PGI)
 HANDLE_DW_OP(0xfb, GNU_addr_index, 0, GNU)
 HANDLE_DW_OP(0xfc, GNU_const_index, 0, GNU)
 
+// DW_OP_LLVM_user has two operands:
+//   (1) An unsigned LEB128 "LLVM Vendor Extension Opcode".
+//   (2) Zero or more literal operands, the number and type of which are
+//       implied by the opcode (1).
+// DW_OP_LLVM_user acts as an extension multiplexer, opening up the encoding
+// space to accommodate an infinite number of extensions. This better reflects
+// the de-facto permanent allocation of extensions.
+HANDLE_DW_OP(0xe9, LLVM_user, 0, LLVM)
+// "LLVM Vendor Extension" operations under the DW_OP_LLVM_user encoding
+// scheme. This list is authoritative and exhaustive. Once an operation is
+// registered here it cannot be removed nor have its encoding changed. The
+// encoding space must skip zero (which is reserved) and have no gaps.
+//
+// The DW_OP_LLVM_user DW_OP_LLVM_nop operation has no effect on the
+// location stack or any of its values. It is defined as a placeholder for
+// testing purposes.
+HANDLE_DW_OP_LLVM_USEROP(0x0001, nop)
+
 // DWARF languages.
 HANDLE_DW_LANG(0x0001, C89, 0, 2, DWARF)
 HANDLE_DW_LANG(0x0002, C, 0, 2, DWARF)
@@ -1235,6 +1258,7 @@ HANDLE_DW_SECT(8, RNGLISTS)
 #undef HANDLE_DW_AT
 #undef HANDLE_DW_FORM
 #undef HANDLE_DW_OP
+#undef HANDLE_DW_OP_LLVM_USEROP
 #undef HANDLE_DW_LANG
 #undef HANDLE_DW_ATE
 #undef HANDLE_DW_VIRTUALITY
diff --git a/llvm/include/llvm/BinaryFormat/Dwarf.h b/llvm/include/llvm/BinaryFormat/Dwarf.h
index ed6abbc0d1143..c4d0232ac6c75 100644
--- a/llvm/include/llvm/BinaryFormat/Dwarf.h
+++ b/llvm/include/llvm/BinaryFormat/Dwarf.h
@@ -146,6 +146,11 @@ enum LocationAtom {
   DW_OP_LLVM_arg = 0x1005,              ///< Only used in LLVM metadata.
 };
 
+enum LlvmUserLocationAtom {
+#define HANDLE_DW_OP_LLVM_USEROP(ID, NAME) DW_OP_LLVM_##NAME = ID,
+#include "llvm/BinaryFormat/Dwarf.def"
+};
+
 enum TypeKind : uint8_t {
 #define HANDLE_DW_ATE(ID, NAME, VERSION, VENDOR) DW_ATE_##NAME = ID,
 #include "llvm/BinaryFormat/Dwarf.def"
@@ -631,6 +636,8 @@ StringRef ChildrenString(unsigned Children);
 StringRef AttributeString(unsigned Attribute);
 StringRef FormEncodingString(unsigned Encoding);
 StringRef OperationEncodingString(unsigned Encoding);
+StringRef SubOperationEncodingString(unsigned OpEncoding,
+                                     unsigned SubOpEncoding);
 StringRef AttributeEncodingString(unsigned Encoding);
 StringRef DecimalSignString(unsigned Sign);
 StringRef EndianityString(unsigned Endian);
@@ -674,6 +681,8 @@ StringRef RLEString(unsigned RLE);
 /// @{
 unsigned getTag(StringRef TagString);
 unsigned getOperationEncoding(StringRef OperationEncodingString);
+unsigned getSubOperationEncoding(unsigned OpEncoding,
+                                 StringRef SubOperationEncodingString);
 unsigned getVirtuality(StringRef VirtualityString);
 unsigned getLanguage(StringRef LanguageString);
 unsigned getCallingConvention(StringRef LanguageString);
diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h
index 6b80dbc038822..00228a32173f1 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFExpression.h
@@ -42,6 +42,9 @@ class DWARFExpression {
       SizeRefAddr = 6,
       SizeBlock = 7, ///< Preceding operand contains block size
       BaseTypeRef = 8,
+      /// The operand is a ULEB128 encoded SubOpcode. This is only valid
+      /// for the first operand of an operation.
+      SizeSubOpLEB = 9,
       WasmLocationArg = 30,
       SignBit = 0x80,
       SignedSize1 = SignBit | Size1,
@@ -83,6 +86,7 @@ class DWARFExpression {
   public:
     const Description &getDescription() const { return Desc; }
     uint8_t getCode() const { return Opcode; }
+    std::optional<unsigned> getSubCode() const;
     uint64_t getNumOperands() const { return Operands.size(); }
     ArrayRef<uint64_t> getRawOperands() const { return Operands; };
     uint64_t getRawOperand(unsigned Idx) const { return Operands[Idx]; }
diff --git a/llvm/lib/BinaryFormat/Dwarf.cpp b/llvm/lib/BinaryFormat/Dwarf.cpp
index 12e9c7374ce8f..e4e5b5dd8c0e0 100644
--- a/llvm/lib/BinaryFormat/Dwarf.cpp
+++ b/llvm/lib/BinaryFormat/Dwarf.cpp
@@ -172,6 +172,40 @@ unsigned llvm::dwarf::getOperationEncoding(StringRef OperationEncodingString) {
       .Default(0);
 }
 
+static StringRef LlvmUserOperationEncodingString(unsigned Encoding) {
+  switch (Encoding) {
+  default:
+    llvm_unreachable("unhandled DWARF operation with LLVM user op");
+#define HANDLE_DW_OP_LLVM_USEROP(ID, NAME)                                     \
+  case DW_OP_LLVM_##NAME:                                                      \
+    return "DW_OP_LLVM_" #NAME;
+#include "llvm/BinaryFormat/Dwarf.def"
+  }
+}
+
+static unsigned
+getLlvmUserOperationEncoding(StringRef LlvmUserOperationEncodingString) {
+  unsigned E = StringSwitch<unsigned>(LlvmUserOperationEncodingString)
+#define HANDLE_DW_OP_LLVM_USEROP(ID, NAME) .Case(#NAME, DW_OP_LLVM_##NAME)
+#include "llvm/BinaryFormat/Dwarf.def"
+                   .Default(0);
+  assert(E && "unhandled DWARF operation string with LLVM user op");
+  return E;
+}
+
+StringRef llvm::dwarf::SubOperationEncodingString(unsigned OpEncoding,
+                                                  unsigned SubOpEncoding) {
+  assert(OpEncoding == DW_OP_LLVM_user);
+  return LlvmUserOperationEncodingString(SubOpEncoding);
+}
+
+unsigned
+llvm::dwarf::getSubOperationEncoding(unsigned OpEncoding,
+                                     StringRef SubOperationEncodingString) {
+  assert(OpEncoding == DW_OP_LLVM_user);
+  return getLlvmUserOperationEncoding(SubOperationEncodingString);
+}
+
 unsigned llvm::dwarf::OperationVersion(dwarf::LocationAtom Op) {
   switch (Op) {
   default:
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index f5a58f79ef4d5..f0c13edaf4bd3 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -2613,6 +2613,7 @@ void DwarfDebug::emitDebugLocEntry(ByteStreamer &Streamer,
   for (const auto &Op : Expr) {
     assert(Op.getCode() != dwarf::DW_OP_const_type &&
            "3 operand ops not yet supported");
+    assert(!Op.getSubCode() && "SubOps not yet supported");
     Streamer.emitInt8(Op.getCode(), Comment != End ? *(Comment++) : "");
     Offset++;
     for (unsigned I = 0; I < Op.getDescription().Op.size(); ++I) {
diff --git a/llvm/lib/DWARFLinker/DWARFLinker.cpp b/llvm/lib/DWARFLinker/DWARFLinker.cpp
index 2b1c6e5ada26d..18b81468fc906 100644
--- a/llvm/lib/DWARFLinker/DWARFLinker.cpp
+++ b/llvm/lib/DWARFLinker/DWARFLinker.cpp
@@ -1169,6 +1169,7 @@ void DWARFLinker::DIECloner::cloneExpression(
       assert(ULEBsize <= 16);
 
       // Copy over the operation.
+      assert(!Op.getSubCode() && "SubOps not yet supported");
       OutputBuffer.push_back(Op.getCode());
       uint64_t RefOffset;
       if (Desc.Op.size() == 1) {
diff --git a/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp b/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp
index 7801170bc6e9e..187d530a00bcf 100644
--- a/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp
+++ b/llvm/lib/DebugInfo/DWARF/DWARFExpression.cpp
@@ -101,6 +101,10 @@ static std::vector<Desc> getOpDescriptions() {
   Descriptions[DW_OP_entry_value] = Desc(Op::Dwarf5, Op::SizeLEB);
   Descriptions[DW_OP_regval_type] =
       Desc(Op::Dwarf5, Op::SizeLEB, Op::BaseTypeRef);
+  // This Description acts as a marker that getSubOpDesc must be called
+  // to fetch the final Description for the operation. Each such final
+  // Description must share the same first SizeSubOpLEB operand.
+  Descriptions[DW_OP_LLVM_user] = Desc(Op::Dwarf5, Op::SizeSubOpLEB);
   return Descriptions;
 }
 
@@ -116,6 +120,23 @@ static Desc getOpDesc(unsigned Opcode) {
   return getDescImpl(Descriptions, Opcode);
 }
 
+static std::vector<Desc> getSubOpDescriptions() {
+  static constexpr unsigned LlvmUserDescriptionsSize = 1
+#define HANDLE_DW_OP_LLVM_USEROP(ID, NAME) +1
+#include "llvm/BinaryFormat/Dwarf.def"
+      ;
+  std::vector<Desc> Descriptions;
+  Descriptions.resize(LlvmUserDescriptionsSize);
+  Descriptions[DW_OP_LLVM_nop] = Desc(Op::Dwarf5, Op::SizeSubOpLEB);
+  return Descriptions;
+}
+
+static Desc getSubOpDesc(unsigned Opcode, unsigned SubOpcode) {
+  assert(Opcode == DW_OP_LLVM_user);
+  static std::vector<Desc> Descriptions = getSubOpDescriptions();
+  return getDescImpl(Descriptions, SubOpcode);
+}
+
 bool DWARFExpression::Operation::extract(DataExtractor Data,
                                          uint8_t AddressSize, uint64_t Offset,
                                          std::optional<DwarfFormat> Format) {
@@ -133,6 +154,15 @@ bool DWARFExpression::Operation::extract(DataExtractor Data,
     unsigned Signed = Size & Operation::SignBit;
 
     switch (Size & ~Operation::SignBit) {
+    case Operation::SizeSubOpLEB:
+      assert(Operand == 0 && "SubOp operand must be the first operand");
+      Operands[Operand] = Data.getULEB128(&Offset);
+      Desc = getSubOpDesc(Opcode, Operands[Operand]);
+      if (Desc.Version == Operation::DwarfNA)
+        return false;
+      assert(Desc.Op[Operand] == Operation::SizeSubOpLEB &&
+             "SizeSubOpLEB Description must begin with SizeSubOpLEB operand");
+      break;
     case Operation::Size1:
       Operands[Operand] = Data.getU8(&Offset);
       if (Signed)
@@ -257,6 +287,12 @@ bool DWARFExpression::prettyPrintRegisterOp(DWARFUnit *U, raw_ostream &OS,
   return false;
 }
 
+std::optional<unsigned> DWARFExpression::Operation::getSubCode() const {
+  if (!Desc.Op.size() || Desc.Op[0] != Operation::SizeSubOpLEB)
+    return std::nullopt;
+  return Operands[0];
+}
+
 bool DWARFExpression::Operation::print(raw_ostream &OS, DIDumpOptions DumpOpts,
                                        const DWARFExpression *Expr,
                                        DWARFUnit *U) const {
@@ -280,7 +316,11 @@ bool DWARFExpression::Operation::print(raw_ostream &OS, DIDumpOptions DumpOpts,
     unsigned Size = Desc.Op[Operand];
     unsigned Signed = Size & Operation::SignBit;
 
-    if (Size == Operation::BaseTypeRef && U) {
+    if (Size == Operation::SizeSubOpLEB) {
+      StringRef SubName = SubOperationEncodingString(Opcode, Operands[Operand]);
+      assert(!SubName.empty() && "DW_OP SubOp has no name!");
+      OS << " " << SubName;
+    } else if (Size == Operation::BaseTypeRef && U) {
       // For DW_OP_convert the operand may be 0 to indicate that conversion to
       // the generic type should be done. The same holds for DW_OP_reinterpret,
       // which is currently not supported.
@@ -450,6 +490,10 @@ static bool printCompactDWARFExpr(
     case dwarf::DW_OP_nop: {
       break;
     }
+    case dwarf::DW_OP_LLVM_user: {
+      assert(Op.getSubCode() && *Op.getSubCode() == dwarf::DW_OP_LLVM_nop);
+      break;
+    }
     default:
       if (Opcode >= dwarf::DW_OP_reg0 && Opcode <= dwarf::DW_OP_reg31) {
         // DW_OP_reg<N>: A register, with the register num implied by the
diff --git a/llvm/test/tools/llvm-dwarfdump/X86/DW_OP_LLVM_user.s b/llvm/test/tools/llvm-dwarfdump/X86/DW_OP_LLVM_user.s
new file mode 100644
index 0000000000000..bcb0dee0ce778
--- /dev/null
+++ b/llvm/test/tools/llvm-dwarfdump/X86/DW_OP_LLVM_user.s
@@ -0,0 +1,16 @@
+# RUN: llvm-mc %s -filetype=obj -triple=i686-pc-linux -o %t
+# RUN: llvm-dwarfdump -v %t | FileCheck %s
+
+# FIXME: Is there a better approach than using `DW_CFA_expression EAX <op>`?
+
+# CHECK:      .eh_frame contents:
+# CHECK:      FDE
+# CHECK-NEXT: Format: DWARF32
+
+foo:
+ .cfi_startproc
+ # CHECK-NEXT: DW_CFA_expression: EAX <decoding error> e9 00
+ .cfi_escape 0x10, 0x00, 0x02, 0xe9, 0x00
+ # CHECK-NEXT: DW_CFA_expression: EAX DW_OP_LLVM_user DW_OP_LLVM_nop
+ .cfi_escape 0x10, 0x00, 0x02, 0xe9, 0x01
+ .cfi_endproc
diff --git a/llvm/unittests/DebugInfo/DWARF/DWARFExpressionCompactPrinterTest.cpp b/llvm/unittests/DebugInfo/DWARF/DWARFExpressionCompactPrinterTest.cpp
index 819f7d56a4318..69edbe53c9e10 100644
--- a/llvm/unittests/DebugInfo/DWARF/DWARFExpressionCompactPrinterTest.cpp
+++ b/llvm/unittests/DebugInfo/DWARF/DWARFExpressionCompactPrinterTest.cpp
@@ -129,6 +129,15 @@ TEST_F(DWARFExpressionCompactPrinterTest, Test_OP_nop) {
   TestExprPrinter({DW_OP_nop}, "<stack of size 0, expected 1>");
 }
 
+TEST_F(DWARFExpressionCompactPrinterTest, Test_OP_LLVM_nop) {
+  TestExprPrinter({DW_OP_LLVM_user, DW_OP_LLVM_nop},
+                  "<stack of size 0, expected 1>");
+}
+
 TEST_F(DWARFExpressionCompactPrinterTest, Test_OP_nop_OP_reg) {
   TestExprPrinter({DW_OP_nop, DW_OP_reg0}, "R0");
 }
+
+TEST_F(DWARFExpressionCompactPrinterTest, Test_OP_LLVM_nop_OP_reg) {
+  TestExprPrinter({DW_OP_LLVM_user, DW_OP_LLVM_nop, DW_OP_reg0}, "R0");
+}

From b7a86d03cb49229634419be4ae44df00fadd990d Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 19 Jun 2023 14:53:22 -0700
Subject: [PATCH 075/130] [xray][test] Test __xray_typedevent after D43668

---
 .../TestCases/Posix/typed-event-logging.cpp   | 36 +++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 compiler-rt/test/xray/TestCases/Posix/typed-event-logging.cpp

diff --git a/compiler-rt/test/xray/TestCases/Posix/typed-event-logging.cpp b/compiler-rt/test/xray/TestCases/Posix/typed-event-logging.cpp
new file mode 100644
index 0000000000000..20c99557acc48
--- /dev/null
+++ b/compiler-rt/test/xray/TestCases/Posix/typed-event-logging.cpp
@@ -0,0 +1,36 @@
+// RUN: %clangxx_xray %s -o %t
+// RUN: XRAY_OPTIONS=patch_premain=false:verbosity=1 %run %t 2>&1 | FileCheck %s
+
+// REQUIRES: target={{x86_64-.*linux.*}}
+
+#include <assert.h>
+#include <stdio.h>
+#include "xray/xray_interface.h"
+
+[[clang::xray_always_instrument]] void foo() {
+  static constexpr char CustomLogged[] = "hello custom logging!";
+  printf("before calling the custom logging...\n");
+  __xray_typedevent(42, CustomLogged, sizeof(CustomLogged));
+  printf("after calling the custom logging...\n");
+}
+
+static void myprinter(uint16_t type, const void *ptr, size_t size) {
+  assert(type == 42);
+  printf("%.*s\n", static_cast<int>(size), static_cast<const char*>(ptr));
+}
+
+int main() {
+  // CHECK: before calling the custom logging...
+  // CHECK-NEXT: after calling the custom logging...
+  foo();
+  __xray_set_typedevent_handler(myprinter);
+  __xray_patch();
+  // CHECK-NEXT: before calling the custom logging...
+  // CHECK-NEXT: hello custom logging!
+  // CHECK-NEXT: after calling the custom logging...
+  foo();
+  // CHECK-NEXT: before calling the custom logging...
+  // CHECK-NEXT: after calling the custom logging...
+  __xray_remove_typedevent_handler();
+  foo();
+}

From 199f7dd421e84fb91d2528d30035b3f07af47264 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 19 Jun 2023 15:11:26 -0700
Subject: [PATCH 076/130] [XRay][X86] Remove sled version 0 support from
 patchCustomEvent

This is remnant after D140739.
---
 compiler-rt/lib/xray/xray_x86_64.cpp | 33 +++++-----------------------
 1 file changed, 6 insertions(+), 27 deletions(-)

diff --git a/compiler-rt/lib/xray/xray_x86_64.cpp b/compiler-rt/lib/xray/xray_x86_64.cpp
index 1bf241c1223fc..b9666a40861d4 100644
--- a/compiler-rt/lib/xray/xray_x86_64.cpp
+++ b/compiler-rt/lib/xray/xray_x86_64.cpp
@@ -250,10 +250,8 @@ bool patchCustomEvent(const bool Enable, const uint32_t FuncId,
                       const XRaySledEntry &Sled) XRAY_NEVER_INSTRUMENT {
   // Here we do the dance of replacing the following sled:
   //
-  // In Version 0:
-  //
   // xray_sled_n:
-  //   jmp +20          // 2 bytes
+  //   jmp +15          // 2 bytes
   //   ...
   //
   // With the following:
@@ -262,36 +260,17 @@ bool patchCustomEvent(const bool Enable, const uint32_t FuncId,
   //   ...
   //
   //
-  // The "unpatch" should just turn the 'nopw' back to a 'jmp +20'.
-  //
-  // ---
-  //
-  // In Version 1 or 2:
-  //
-  //   The jump offset is now 15 bytes (0x0f), so when restoring the nopw back
-  //   to a jmp, use 15 bytes instead.
-  //
+  // The "unpatch" should just turn the 'nopw' back to a 'jmp +15'.
   const uint64_t Address = Sled.address();
   if (Enable) {
     std::atomic_store_explicit(
         reinterpret_cast<std::atomic<uint16_t> *>(Address), NopwSeq,
         std::memory_order_release);
   } else {
-    switch (Sled.Version) {
-    case 1:
-    case 2:
-      std::atomic_store_explicit(
-          reinterpret_cast<std::atomic<uint16_t> *>(Address), Jmp15Seq,
-          std::memory_order_release);
-      break;
-    case 0:
-    default:
-      std::atomic_store_explicit(
-          reinterpret_cast<std::atomic<uint16_t> *>(Address), Jmp20Seq,
-          std::memory_order_release);
-      break;
-    }
-    }
+    std::atomic_store_explicit(
+        reinterpret_cast<std::atomic<uint16_t> *>(Address), Jmp15Seq,
+        std::memory_order_release);
+  }
   return false;
 }
 

From fec7c6457ced7e87d4e0837402b0b8a644cf6242 Mon Sep 17 00:00:00 2001
From: Vladislav Dzhidzhoev <vdzhidzhoev@accesssoftek.com>
Date: Tue, 20 Jun 2023 01:54:48 +0200
Subject: [PATCH 077/130] Revert "Reland "[DebugMetadata][DwarfDebug] Support
 function-local types in lexical block scopes (4/7)""

This reverts commit 2da45172c4bcd42f704c57c656926f56f32fc5ce.
Test local-type-as-template-parameter.ll fails on ppc64-aix.
---
 .../CodeGen/debug-info-codeview-unnamed.c     |  16 +-
 clang/test/CodeGen/debug-info-unused-types.c  |  14 +-
 .../test/CodeGen/debug-info-unused-types.cpp  |  12 +-
 clang/test/CodeGenCXX/debug-info-access.cpp   |   2 +-
 .../CodeGenCXX/debug-info-anon-union-vars.cpp |  12 +-
 .../debug-info-codeview-unnamed.cpp           | 110 ++---
 .../debug-info-gline-tables-only-codeview.cpp |   4 +-
 clang/test/CodeGenCXX/debug-lambda-this.cpp   |   4 +-
 llvm/include/llvm/IR/DIBuilder.h              |   6 +-
 llvm/lib/Bitcode/Reader/MetadataLoader.cpp    |  87 ++--
 .../CodeGen/AsmPrinter/DwarfCompileUnit.cpp   |  60 +--
 .../lib/CodeGen/AsmPrinter/DwarfCompileUnit.h |  16 +-
 llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp    |  13 +-
 llvm/lib/IR/DIBuilder.cpp                     |  34 +-
 llvm/lib/IR/Verifier.cpp                      |   6 +-
 llvm/test/Bitcode/upgrade-cu-locals.ll        |  68 ++-
 llvm/test/Bitcode/upgrade-cu-locals.ll.bc     | Bin 2780 -> 2688 bytes
 .../DebugInfo/Generic/inlined-local-type.ll   | 128 ------
 .../Generic/lexical-block-retained-types.ll   |  55 ---
 .../DebugInfo/Generic/lexical-block-types.ll  | 425 ------------------
 .../local-type-as-template-parameter.ll       | 161 -------
 .../split-dwarf-local-import.ll               |   1 +
 .../split-dwarf-local-import2.ll              |   1 +
 .../split-dwarf-local-import3.ll              |   0
 .../Generic/verifier-invalid-disubprogram.ll  |   2 +-
 llvm/test/DebugInfo/X86/set.ll                |   4 +-
 26 files changed, 189 insertions(+), 1052 deletions(-)
 delete mode 100644 llvm/test/DebugInfo/Generic/inlined-local-type.ll
 delete mode 100644 llvm/test/DebugInfo/Generic/lexical-block-retained-types.ll
 delete mode 100644 llvm/test/DebugInfo/Generic/lexical-block-types.ll
 delete mode 100644 llvm/test/DebugInfo/Generic/local-type-as-template-parameter.ll
 rename llvm/test/DebugInfo/{X86 => Generic}/split-dwarf-local-import.ll (98%)
 rename llvm/test/DebugInfo/{X86 => Generic}/split-dwarf-local-import2.ll (98%)
 rename llvm/test/DebugInfo/{X86 => Generic}/split-dwarf-local-import3.ll (100%)

diff --git a/clang/test/CodeGen/debug-info-codeview-unnamed.c b/clang/test/CodeGen/debug-info-codeview-unnamed.c
index 16ffb3682236f..bd2a7543e56b2 100644
--- a/clang/test/CodeGen/debug-info-codeview-unnamed.c
+++ b/clang/test/CodeGen/debug-info-codeview-unnamed.c
@@ -8,23 +8,23 @@ int main(int argc, char* argv[], char* arge[]) {
   //
   struct { int bar; } one = {42};
   //
-  // LINUX:      [[TYPE_OF_ONE:![0-9]+]] = distinct !DICompositeType(
+  // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "one"
+  // LINUX-SAME:     type: [[TYPE_OF_ONE:![0-9]+]]
+  // LINUX-SAME:     )
+  // LINUX:      [[TYPE_OF_ONE]] = distinct !DICompositeType(
   // LINUX-SAME:     tag: DW_TAG_structure_type
   // LINUX-NOT:      name:
   // LINUX-NOT:      identifier:
   // LINUX-SAME:     )
-  // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "one"
-  // LINUX-SAME:     type: [[TYPE_OF_ONE]]
-  // LINUX-SAME:     )
   //
-  // MSVC:       [[TYPE_OF_ONE:![0-9]+]] = distinct !DICompositeType
+  // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "one"
+  // MSVC-SAME:      type: [[TYPE_OF_ONE:![0-9]+]]
+  // MSVC-SAME:      )
+  // MSVC:       [[TYPE_OF_ONE]] = distinct !DICompositeType
   // MSVC-SAME:      tag: DW_TAG_structure_type
   // MSVC-NOT:       name:
   // MSVC-NOT:       identifier:
   // MSVC-SAME:      )
-  // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "one"
-  // MSVC-SAME:      type: [[TYPE_OF_ONE]]
-  // MSVC-SAME:      )
 
   return 0;
 }
diff --git a/clang/test/CodeGen/debug-info-unused-types.c b/clang/test/CodeGen/debug-info-unused-types.c
index 5bcc71f710a5c..3e9f7b07658e3 100644
--- a/clang/test/CodeGen/debug-info-unused-types.c
+++ b/clang/test/CodeGen/debug-info-unused-types.c
@@ -18,13 +18,13 @@ void quux(void) {
 // CHECK: !DICompileUnit{{.+}}retainedTypes: [[RETTYPES:![0-9]+]]
 // CHECK: [[TYPE0:![0-9]+]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "bar"
 // CHECK: [[TYPE1:![0-9]+]] = !DIEnumerator(name: "BAR"
-// CHECK: [[RETTYPES]] = !{[[TYPE2:![0-9]+]], [[TYPE3:![0-9]+]], [[TYPE0]], [[TYPE4:![0-9]+]], {{![0-9]+}}, [[TYPE5:![0-9]+]], [[TYPE6:![0-9]+]], [[TYPE8:![0-9]+]]}
-// CHECK: [[TYPE2]] = !DIDerivedType(tag: DW_TAG_typedef, name: "my_int"
-// CHECK: [[TYPE3]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "foo"
-// CHECK: [[TYPE4]] = distinct !DICompositeType(tag: DW_TAG_union_type, name: "baz"
-// CHECK: [[TYPE5]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "y"
-// CHECK: [[TYPE6]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "z"
-// CHECK: [[TYPE7:![0-9]+]] = !DIEnumerator(name: "Z"
+// CHECK: [[TYPE2:![0-9]+]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "z"
+// CHECK: [[TYPE3:![0-9]+]] = !DIEnumerator(name: "Z"
+// CHECK: [[RETTYPES]] = !{[[TYPE4:![0-9]+]], [[TYPE5:![0-9]+]], [[TYPE0]], [[TYPE6:![0-9]+]], {{![0-9]+}}, [[TYPE7:![0-9]+]], [[TYPE2]], [[TYPE8:![0-9]+]]}
+// CHECK: [[TYPE4]] = !DIDerivedType(tag: DW_TAG_typedef, name: "my_int"
+// CHECK: [[TYPE5]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "foo"
+// CHECK: [[TYPE6]] = distinct !DICompositeType(tag: DW_TAG_union_type, name: "baz"
+// CHECK: [[TYPE7]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "y"
 // CHECK: [[TYPE8]] = distinct !DICompositeType(tag: DW_TAG_union_type, name: "w"
 
 // Check that debug info is not emitted for the typedef, struct, enum, and
diff --git a/clang/test/CodeGen/debug-info-unused-types.cpp b/clang/test/CodeGen/debug-info-unused-types.cpp
index 715fada47eab8..023cac159faa4 100644
--- a/clang/test/CodeGen/debug-info-unused-types.cpp
+++ b/clang/test/CodeGen/debug-info-unused-types.cpp
@@ -13,12 +13,12 @@ void quux() {
 // CHECK: !DICompileUnit{{.+}}retainedTypes: [[RETTYPES:![0-9]+]]
 // CHECK: [[TYPE0:![0-9]+]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "baz"
 // CHECK: [[TYPE1:![0-9]+]] = !DIEnumerator(name: "BAZ"
-// CHECK: [[RETTYPES]] = !{[[TYPE2:![0-9]+]], [[TYPE3:![0-9]+]], [[TYPE0]], {{![0-9]+}}, [[TYPE4:![0-9]+]], [[TYPE5:![0-9]+]]}
-// CHECK: [[TYPE2]] = !DIDerivedType(tag: DW_TAG_typedef, name: "foo"
-// CHECK: [[TYPE3]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "bar"
-// CHECK: [[TYPE4]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "y"
-// CHECK: [[TYPE5]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "z"
-// CHECK: [[TYPE6:![0-9]+]] = !DIEnumerator(name: "Z"
+// CHECK: [[TYPE2:![0-9]+]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "z"
+// CHECK: [[TYPE3:![0-9]+]] = !DIEnumerator(name: "Z"
+// CHECK: [[RETTYPES]] = !{[[TYPE4:![0-9]+]], [[TYPE5:![0-9]+]], [[TYPE0]], {{![0-9]+}}, [[TYPE6:![0-9]+]], [[TYPE2]]}
+// CHECK: [[TYPE4]] = !DIDerivedType(tag: DW_TAG_typedef, name: "foo"
+// CHECK: [[TYPE5]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "bar"
+// CHECK: [[TYPE6]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "y"
 
 // NODBG-NOT: !DI{{CompositeType|Enumerator|DerivedType}}
 
diff --git a/clang/test/CodeGenCXX/debug-info-access.cpp b/clang/test/CodeGenCXX/debug-info-access.cpp
index 7c0bf8ccb0384..9f2c044843d0f 100644
--- a/clang/test/CodeGenCXX/debug-info-access.cpp
+++ b/clang/test/CodeGenCXX/debug-info-access.cpp
@@ -18,9 +18,9 @@ class B : public A {
   static int public_static;
 
 protected:
-  // CHECK-DAG: !DIDerivedType(tag: DW_TAG_typedef, name: "prot_using",{{.*}} line: [[@LINE+3]],{{.*}} flags: DIFlagProtected)
   // CHECK-DAG: !DIDerivedType(tag: DW_TAG_typedef, name: "prot_typedef",{{.*}} line: [[@LINE+1]],{{.*}} flags: DIFlagProtected)
   typedef int prot_typedef;
+  // CHECK-DAG: !DIDerivedType(tag: DW_TAG_typedef, name: "prot_using",{{.*}} line: [[@LINE+1]],{{.*}} flags: DIFlagProtected)
   using prot_using = prot_typedef;
   prot_using prot_member;
 
diff --git a/clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp b/clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp
index c91cf83c0405f..61b3c7c0526c8 100644
--- a/clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp
+++ b/clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp
@@ -51,13 +51,13 @@ void instantiate(int x) {
 // CHECK: !DIGlobalVariable(name: "b",{{.*}} file: [[FILE]], line: 6,{{.*}} isLocal: true, isDefinition: true
 // CHECK: !DIGlobalVariable(name: "result", {{.*}} isLocal: false, isDefinition: true
 // CHECK: !DIGlobalVariable(name: "value", {{.*}} isLocal: false, isDefinition: true
-// CHECK: ![[UNION:[0-9]+]] = distinct !DICompositeType(tag: DW_TAG_union_type,
-// CHECK-NOT: name:
-// CHECK: elements
-// CHECK: !DIDerivedType(tag: DW_TAG_member, name: "i", scope: ![[UNION]],
-// CHECK: !DIDerivedType(tag: DW_TAG_member, name: "c", scope: ![[UNION]],
 // CHECK: !DILocalVariable(name: "i", {{.*}}, flags: DIFlagArtificial
 // CHECK: !DILocalVariable(name: "c", {{.*}}, flags: DIFlagArtificial
 // CHECK: !DILocalVariable(
 // CHECK-NOT: name:
-// CHECK: type: ![[UNION]]
+// CHECK: type: ![[UNION:[0-9]+]]
+// CHECK: ![[UNION]] = distinct !DICompositeType(tag: DW_TAG_union_type,
+// CHECK-NOT: name:
+// CHECK: elements
+// CHECK: !DIDerivedType(tag: DW_TAG_member, name: "i", scope: ![[UNION]],
+// CHECK: !DIDerivedType(tag: DW_TAG_member, name: "c", scope: ![[UNION]],
diff --git a/clang/test/CodeGenCXX/debug-info-codeview-unnamed.cpp b/clang/test/CodeGenCXX/debug-info-codeview-unnamed.cpp
index 9602ac1b02497..b4c79936ab33e 100644
--- a/clang/test/CodeGenCXX/debug-info-codeview-unnamed.cpp
+++ b/clang/test/CodeGenCXX/debug-info-codeview-unnamed.cpp
@@ -3,60 +3,6 @@
 
 int main(int argc, char* argv[], char* arge[]) {
   //
-  // LINUX:      [[TYPE_OF_ONE:![0-9]+]] = distinct !DICompositeType(
-  // LINUX-SAME:     tag: DW_TAG_structure_type
-  // LINUX-NOT:      name:
-  // LINUX-NOT:      identifier:
-  // LINUX-SAME:     )
-  //
-  // MSVC:       [[TYPE_OF_ONE:![0-9]+]] = distinct !DICompositeType
-  // MSVC-SAME:      tag: DW_TAG_structure_type
-  // MSVC-SAME:      name: "<unnamed-type-one>"
-  // MSVC-SAME:      identifier: ".?AU<unnamed-type-one>@?1??main@@9@"
-  // MSVC-SAME:      )
-
-
-  //
-  // LINUX:      [[TYPE_OF_TWO:![0-9]+]] = distinct !DICompositeType(
-  // LINUX-SAME:     tag: DW_TAG_structure_type
-  // LINUX-NOT:      name:
-  // LINUX-NOT:      identifier:
-  // LINUX-SAME:     )
-  //
-  // MSVC:       [[TYPE_OF_TWO:![0-9]+]] = distinct !DICompositeType
-  // MSVC-SAME:      tag: DW_TAG_structure_type
-  // MSVC-SAME:      name: "<unnamed-type-two>"
-  // MSVC-SAME:      identifier: ".?AU<unnamed-type-two>@?2??main@@9@"
-  // MSVC-SAME:      )
-
-
-  //
-  // LINUX:      [[TYPE_OF_THREE:![0-9]+]] = distinct !DICompositeType(
-  // LINUX-SAME:     tag: DW_TAG_structure_type
-  // LINUX-SAME:     name: "named"
-  // LINUX-NOT:      identifier:
-  // LINUX-SAME:     )
-  //
-  // MSVC:       [[TYPE_OF_THREE:![0-9]+]] = distinct !DICompositeType
-  // MSVC-SAME:      tag: DW_TAG_structure_type
-  // MSVC-SAME:      name: "named"
-  // MSVC-SAME:      identifier: ".?AUnamed@?1??main@@9@"
-  // MSVC-SAME:      )
-
-  //
-  // LINUX:      [[TYPE_OF_FOUR:![0-9]+]] = distinct !DICompositeType(
-  // LINUX-SAME:     tag: DW_TAG_class_type
-  // LINUX-NOT:      name:
-  // LINUX-NOT:      identifier:
-  // LINUX-SAME:     )
-  //
-  // MSVC:       [[TYPE_OF_FOUR:![0-9]+]] = distinct !DICompositeType
-  // MSVC-SAME:      tag: DW_TAG_class_type
-  // MSVC-SAME:      name: "<lambda_0>"
-  // MSVC-SAME:      identifier: ".?AV<lambda_0>@?0??main@@9@"
-  // MSVC-SAME:      )
-
-
   // In CodeView, the LF_MFUNCTION entry for "bar()" refers to the forward
   // reference of the unnamed struct. Visual Studio requires a unique
   // identifier to match the LF_STRUCTURE forward reference to the definition.
@@ -64,11 +10,21 @@ int main(int argc, char* argv[], char* arge[]) {
   struct { void bar() {} } one;
   //
   // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "one"
-  // LINUX-SAME:     type: [[TYPE_OF_ONE]]
+  // LINUX-SAME:     type: [[TYPE_OF_ONE:![0-9]+]]
+  // LINUX-SAME:     )
+  // LINUX:      [[TYPE_OF_ONE]] = distinct !DICompositeType(
+  // LINUX-SAME:     tag: DW_TAG_structure_type
+  // LINUX-NOT:      name:
+  // LINUX-NOT:      identifier:
   // LINUX-SAME:     )
   //
   // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "one"
-  // MSVC-SAME:      type: [[TYPE_OF_ONE]]
+  // MSVC-SAME:      type: [[TYPE_OF_ONE:![0-9]+]]
+  // MSVC-SAME:      )
+  // MSVC:       [[TYPE_OF_ONE]] = distinct !DICompositeType
+  // MSVC-SAME:      tag: DW_TAG_structure_type
+  // MSVC-SAME:      name: "<unnamed-type-one>"
+  // MSVC-SAME:      identifier: ".?AU<unnamed-type-one>@?1??main@@9@"
   // MSVC-SAME:      )
 
 
@@ -80,11 +36,21 @@ int main(int argc, char* argv[], char* arge[]) {
   int decltype(two)::*ptr2unnamed = &decltype(two)::bar;
   //
   // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "two"
-  // LINUX-SAME:     type: [[TYPE_OF_TWO]]
+  // LINUX-SAME:     type: [[TYPE_OF_TWO:![0-9]+]]
+  // LINUX-SAME:     )
+  // LINUX:      [[TYPE_OF_TWO]] = distinct !DICompositeType(
+  // LINUX-SAME:     tag: DW_TAG_structure_type
+  // LINUX-NOT:      name:
+  // LINUX-NOT:      identifier:
   // LINUX-SAME:     )
   //
   // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "two"
-  // MSVC-SAME:      type: [[TYPE_OF_TWO]]
+  // MSVC-SAME:      type: [[TYPE_OF_TWO:![0-9]+]]
+  // MSVC-SAME:      )
+  // MSVC:       [[TYPE_OF_TWO]] = distinct !DICompositeType
+  // MSVC-SAME:      tag: DW_TAG_structure_type
+  // MSVC-SAME:      name: "<unnamed-type-two>"
+  // MSVC-SAME:      identifier: ".?AU<unnamed-type-two>@?2??main@@9@"
   // MSVC-SAME:      )
 
 
@@ -95,11 +61,21 @@ int main(int argc, char* argv[], char* arge[]) {
   struct named { int bar; int named::* p2mem; } three = { 42, &named::bar };
   //
   // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "three"
-  // LINUX-SAME:     type: [[TYPE_OF_THREE]]
+  // LINUX-SAME:     type: [[TYPE_OF_THREE:![0-9]+]]
+  // LINUX-SAME:     )
+  // LINUX:      [[TYPE_OF_THREE]] = distinct !DICompositeType(
+  // LINUX-SAME:     tag: DW_TAG_structure_type
+  // LINUX-SAME:     name: "named"
+  // LINUX-NOT:      identifier:
   // LINUX-SAME:     )
   //
   // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "three"
-  // MSVC-SAME:      type: [[TYPE_OF_THREE]]
+  // MSVC-SAME:      type: [[TYPE_OF_THREE:![0-9]+]]
+  // MSVC-SAME:      )
+  // MSVC:       [[TYPE_OF_THREE]] = distinct !DICompositeType
+  // MSVC-SAME:      tag: DW_TAG_structure_type
+  // MSVC-SAME:      name: "named"
+  // MSVC-SAME:      identifier: ".?AUnamed@?1??main@@9@"
   // MSVC-SAME:      )
 
 
@@ -111,11 +87,21 @@ int main(int argc, char* argv[], char* arge[]) {
   auto four = [argc](int i) -> int { return argc == i ? 1 : 0; };
   //
   // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "four"
-  // LINUX-SAME:     type: [[TYPE_OF_FOUR]]
+  // LINUX-SAME:     type: [[TYPE_OF_FOUR:![0-9]+]]
+  // LINUX-SAME:     )
+  // LINUX:      [[TYPE_OF_FOUR]] = distinct !DICompositeType(
+  // LINUX-SAME:     tag: DW_TAG_class_type
+  // LINUX-NOT:      name:
+  // LINUX-NOT:      identifier:
   // LINUX-SAME:     )
   //
   // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "four"
-  // MSVC-SAME:      type: [[TYPE_OF_FOUR]]
+  // MSVC-SAME:      type: [[TYPE_OF_FOUR:![0-9]+]]
+  // MSVC-SAME:      )
+  // MSVC:       [[TYPE_OF_FOUR]] = distinct !DICompositeType
+  // MSVC-SAME:      tag: DW_TAG_class_type
+  // MSVC-SAME:      name: "<lambda_0>"
+  // MSVC-SAME:      identifier: ".?AV<lambda_0>@?0??main@@9@"
   // MSVC-SAME:      )
 
   return 0;
diff --git a/clang/test/CodeGenCXX/debug-info-gline-tables-only-codeview.cpp b/clang/test/CodeGenCXX/debug-info-gline-tables-only-codeview.cpp
index 122e4aa62ea7d..6b9c9a243decd 100644
--- a/clang/test/CodeGenCXX/debug-info-gline-tables-only-codeview.cpp
+++ b/clang/test/CodeGenCXX/debug-info-gline-tables-only-codeview.cpp
@@ -51,9 +51,9 @@ void test() {
   // CHECK-SAME:                              name: "<lambda_2_1>",
   c.lambda_params();
 
-  // CHECK: ![[LAMBDA1:[0-9]+]] = !DICompositeType(tag: DW_TAG_class_type,
+  // CHECK: !DISubprogram(name: "operator()", scope: ![[LAMBDA1:[0-9]+]],
+  // CHECK: ![[LAMBDA1]] = !DICompositeType(tag: DW_TAG_class_type,
   // CHECK-SAME:                            name: "<lambda_1>",
   // CHECK-SAME:                            flags: DIFlagFwdDecl
-  // CHECK: !DISubprogram(name: "operator()", scope: ![[LAMBDA1]],
   c.lambda2();
 }
diff --git a/clang/test/CodeGenCXX/debug-lambda-this.cpp b/clang/test/CodeGenCXX/debug-lambda-this.cpp
index 3d659e7bfd004..eecbac6520ac9 100644
--- a/clang/test/CodeGenCXX/debug-lambda-this.cpp
+++ b/clang/test/CodeGenCXX/debug-lambda-this.cpp
@@ -13,10 +13,10 @@ int D::d(int x) {
 }
 
 // CHECK: ![[D:[0-9]+]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "D",
+// CHECK: ![[POINTER:.*]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: ![[D]], size: 64)
 // CHECK: !DIDerivedType(tag: DW_TAG_member, name: "this",
 // CHECK-SAME:           line: 11
-// CHECK-SAME:           baseType: ![[POINTER:[0-9]+]]
+// CHECK-SAME:           baseType: ![[POINTER]]
 // CHECK-SAME:           size: 64
 // CHECK-NOT:            offset: 0
 // CHECK-SAME:           ){{$}}
-// CHECK: ![[POINTER]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: ![[D]], size: 64)
diff --git a/llvm/include/llvm/IR/DIBuilder.h b/llvm/include/llvm/IR/DIBuilder.h
index 0aa180aec6d8a..ecd6dd7b0a4f8 100644
--- a/llvm/include/llvm/IR/DIBuilder.h
+++ b/llvm/include/llvm/IR/DIBuilder.h
@@ -49,7 +49,7 @@ namespace llvm {
     Function *LabelFn;       ///< llvm.dbg.label
     Function *AssignFn;      ///< llvm.dbg.assign
 
-    SmallVector<TrackingMDNodeRef, 4> EnumTypes;
+    SmallVector<TrackingMDNodeRef, 4> AllEnumTypes;
     /// Track the RetainTypes, since they can be updated later on.
     SmallVector<TrackingMDNodeRef, 4> AllRetainTypes;
     SmallVector<DISubprogram *, 4> AllSubprograms;
@@ -64,8 +64,8 @@ namespace llvm {
     SmallVector<TrackingMDNodeRef, 4> UnresolvedNodes;
     bool AllowUnresolvedNodes;
 
-    /// Each subprogram's preserved local variables, labels, imported entities,
-    /// and types.
+    /// Each subprogram's preserved local variables, labels and imported
+    /// entities.
     ///
     /// Do not use a std::vector.  Some versions of libc++ apparently copy
     /// instead of move on grow operations, and TrackingMDRef is expensive to
diff --git a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
index 321eae6fcb3b5..0a9a80688a419 100644
--- a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
+++ b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
@@ -548,8 +548,6 @@ class MetadataLoader::MetadataLoaderImpl {
 
   /// Move local imports from DICompileUnit's 'imports' field to
   /// DISubprogram's retainedNodes.
-  /// Move fucntion-local enums from DICompileUnit's enums
-  /// to DISubprogram's retainedNodes.
   void upgradeCULocals() {
     if (NamedMDNode *CUNodes = TheModule.getNamedMetadata("llvm.dbg.cu")) {
       for (unsigned I = 0, E = CUNodes->getNumOperands(); I != E; ++I) {
@@ -557,71 +555,48 @@ class MetadataLoader::MetadataLoaderImpl {
         if (!CU)
           continue;
 
-        SetVector<Metadata *> MetadataToRemove;
-        // Collect imported entities to be moved.
-        if (CU->getRawImportedEntities()) {
+        if (auto *RawImported = CU->getRawImportedEntities()) {
+          // Collect a set of imported entities to be moved.
+          SetVector<Metadata *> EntitiesToRemove;
           for (Metadata *Op : CU->getImportedEntities()->operands()) {
             auto *IE = cast<DIImportedEntity>(Op);
-            if (dyn_cast_or_null<DILocalScope>(IE->getScope())) {
-              MetadataToRemove.insert(IE);
+            if (auto *S = dyn_cast_or_null<DILocalScope>(IE->getScope())) {
+              EntitiesToRemove.insert(IE);
             }
           }
-        }
-        // Collect enums to be moved.
-        if (CU->getRawEnumTypes()) {
-          for (Metadata *Op : CU->getEnumTypes()->operands()) {
-            auto *Enum = cast<DICompositeType>(Op);
-            if (dyn_cast_or_null<DILocalScope>(Enum->getScope())) {
-              MetadataToRemove.insert(Enum);
-            }
-          }
-        }
 
-        if (!MetadataToRemove.empty()) {
-          // Make a new list of CU's 'imports'.
-          SmallVector<Metadata *> NewImports;
-          if (CU->getRawImportedEntities())
-            for (Metadata *Op : CU->getImportedEntities()->operands())
-              if (!MetadataToRemove.contains(Op))
+          if (!EntitiesToRemove.empty()) {
+            // Make a new list of CU's 'imports'.
+            SmallVector<Metadata *> NewImports;
+            for (Metadata *Op : CU->getImportedEntities()->operands()) {
+              if (!EntitiesToRemove.contains(cast<DIImportedEntity>(Op))) {
                 NewImports.push_back(Op);
+              }
+            }
 
-          // Make a new list of CU's 'enums'.
-          SmallVector<Metadata *> NewEnums;
-          if (CU->getRawEnumTypes())
-            for (Metadata *Op : CU->getEnumTypes()->operands())
-              if (!MetadataToRemove.contains(Op))
-                NewEnums.push_back(Op);
-
-          // Find DISubprogram corresponding to each entity.
-          std::map<DISubprogram *, SmallVector<Metadata *>> SPToEntities;
-          for (auto *I : MetadataToRemove) {
-            DILocalScope *Scope = nullptr;
-            if (auto *Entity = dyn_cast<DIImportedEntity>(I))
-              Scope = cast<DILocalScope>(Entity->getScope());
-            else if (auto *Enum = dyn_cast<DICompositeType>(I))
-              Scope = cast<DILocalScope>(Enum->getScope());
-
-            if (auto *SP = findEnclosingSubprogram(Scope)) {
-              SPToEntities[SP].push_back(I);
+            // Find DISubprogram corresponding to each entity.
+            std::map<DISubprogram *, SmallVector<Metadata *>> SPToEntities;
+            for (auto *I : EntitiesToRemove) {
+              auto *Entity = cast<DIImportedEntity>(I);
+              if (auto *SP = findEnclosingSubprogram(
+                      cast<DILocalScope>(Entity->getScope()))) {
+                SPToEntities[SP].push_back(Entity);
+              }
             }
-          }
 
-          // Update DISubprograms' retainedNodes.
-          for (auto I = SPToEntities.begin(); I != SPToEntities.end(); ++I) {
-            auto *SP = I->first;
-            auto RetainedNodes = SP->getRetainedNodes();
-            SmallVector<Metadata *> MDs(RetainedNodes.begin(),
-                                        RetainedNodes.end());
-            MDs.append(I->second);
-            SP->replaceRetainedNodes(MDNode::get(Context, MDs));
-          }
+            // Update DISubprograms' retainedNodes.
+            for (auto I = SPToEntities.begin(); I != SPToEntities.end(); ++I) {
+              auto *SP = I->first;
+              auto RetainedNodes = SP->getRetainedNodes();
+              SmallVector<Metadata *> MDs(RetainedNodes.begin(),
+                                          RetainedNodes.end());
+              MDs.append(I->second);
+              SP->replaceRetainedNodes(MDNode::get(Context, MDs));
+            }
 
-          // Remove entities with local scope from CU.
-          if (CU->getRawImportedEntities())
+            // Remove entities with local scope from CU.
             CU->replaceImportedEntities(MDTuple::get(Context, NewImports));
-          // Remove enums with local scope from CU.
-          if (CU->getRawEnumTypes())
-            CU->replaceEnumTypes(MDTuple::get(Context, NewEnums));
+          }
         }
       }
     }
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
index bf8bdbb7da49f..58ed21379d29b 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
@@ -598,9 +598,10 @@ void DwarfCompileUnit::constructScopeDIE(LexicalScope *Scope,
     return;
 
   // Emit lexical blocks.
-  DIE *ScopeDIE = getOrCreateLexicalBlockDIE(Scope, ParentScopeDIE);
+  DIE *ScopeDIE = constructLexicalScopeDIE(Scope);
   assert(ScopeDIE && "Scope DIE should not be null.");
 
+  ParentScopeDIE.addChild(ScopeDIE);
   createAndAddScopeChildren(Scope, *ScopeDIE);
 }
 
@@ -720,39 +721,24 @@ DIE *DwarfCompileUnit::constructInlinedScopeDIE(LexicalScope *Scope,
   return ScopeDIE;
 }
 
-DIE *DwarfCompileUnit::getOrCreateLexicalBlockDIE(LexicalScope *Scope,
-                                                  DIE &ParentScopeDIE) {
+// Construct new DW_TAG_lexical_block for this scope and attach
+// DW_AT_low_pc/DW_AT_high_pc labels.
+DIE *DwarfCompileUnit::constructLexicalScopeDIE(LexicalScope *Scope) {
   if (DD->isLexicalScopeDIENull(Scope))
     return nullptr;
   const auto *DS = Scope->getScopeNode();
-  DIE *ScopeDIE = nullptr;
-
-  // FIXME: We may have a concrete DIE for this scope already created.
-  // This may happen when we emit local variables for an abstract tree of
-  // an inlined function: if a local variable has a templated type with
-  // a function-local type as a template parameter. See PR55680 for details
-  // (see also llvm/test/DebugInfo/Generic/local-type-as-template-parameter.ll).
-  if (!Scope->isAbstractScope() && !Scope->getInlinedAt()) {
-    if (auto It = LexicalBlockDIEs.find(DS); It != LexicalBlockDIEs.end()) {
-      ScopeDIE = It->second;
-      assert(!ScopeDIE->findAttribute(dwarf::DW_AT_low_pc) &&
-             !ScopeDIE->findAttribute(dwarf::DW_AT_ranges));
-      assert(ScopeDIE->getParent() == &ParentScopeDIE);
-    }
-  }
-  if (!ScopeDIE) {
-    ScopeDIE = DIE::get(DIEValueAllocator, dwarf::DW_TAG_lexical_block);
-    ParentScopeDIE.addChild(ScopeDIE);
-
-    if (Scope->isAbstractScope()) {
-      assert(!getAbstractScopeDIEs().count(DS) &&
-             "Abstract DIE for this scope exists!");
-      getAbstractScopeDIEs()[DS] = ScopeDIE;
-      return ScopeDIE;
-    }
 
-    if (!Scope->getInlinedAt())
-      LexicalBlockDIEs[DS] = ScopeDIE;
+  auto ScopeDIE = DIE::get(DIEValueAllocator, dwarf::DW_TAG_lexical_block);
+  if (Scope->isAbstractScope()) {
+    assert(!getAbstractScopeDIEs().count(DS) &&
+           "Abstract DIE for this scope exists!");
+    getAbstractScopeDIEs()[DS] = ScopeDIE;
+    return ScopeDIE;
+  }
+  if (!Scope->getInlinedAt()) {
+    assert(!LexicalBlockDIEs.count(DS) &&
+           "Concrete out-of-line DIE for this scope exists!");
+    LexicalBlockDIEs[DS] = ScopeDIE;
   }
 
   attachRangesOrLowHighPC(*ScopeDIE, Scope->getRanges());
@@ -1695,21 +1681,15 @@ void DwarfCompileUnit::createBaseTypeDIEs() {
   }
 }
 
-DIE *DwarfCompileUnit::getLocalContextDIE(const DILexicalBlock *LB) {
+DIE *DwarfCompileUnit::getLexicalBlockDIE(const DILexicalBlock *LB) {
   // Assume if there is an abstract tree all the DIEs are already emitted.
   bool isAbstract = getAbstractScopeDIEs().count(LB->getSubprogram());
   if (isAbstract && getAbstractScopeDIEs().count(LB))
     return getAbstractScopeDIEs()[LB];
   assert(!isAbstract && "Missed lexical block DIE in abstract tree!");
 
-  // Check if we have a concrete DIE.
-  if (auto It = LexicalBlockDIEs.find(LB); It != LexicalBlockDIEs.end())
-    return It->second;
-
-  // If nothing available found, we cannot just create a new lexical block,
-  // because it isn't known where to put it into the DIE tree.
-  // So, we may only try to find the most close avaiable parent DIE.
-  return getOrCreateContextDIE(LB->getScope()->getNonLexicalBlockFileScope());
+  // Return a concrete DIE if it exists or nullptr otherwise.
+  return LexicalBlockDIEs.lookup(LB);
 }
 
 DIE *DwarfCompileUnit::getOrCreateContextDIE(const DIScope *Context) {
@@ -1717,7 +1697,7 @@ DIE *DwarfCompileUnit::getOrCreateContextDIE(const DIScope *Context) {
     if (auto *LFScope = dyn_cast<DILexicalBlockFile>(Context))
       Context = LFScope->getNonLexicalBlockFileScope();
     if (auto *LScope = dyn_cast<DILexicalBlock>(Context))
-      return getLocalContextDIE(LScope);
+      return getLexicalBlockDIE(LScope);
 
     // Otherwise the context must be a DISubprogram.
     auto *SPScope = cast<DISubprogram>(Context);
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
index 5662087bec322..6ef73ebd4f7f2 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
@@ -210,9 +210,14 @@ class DwarfCompileUnit final : public DwarfUnit {
   /// DIE to represent this concrete inlined copy of the function.
   DIE *constructInlinedScopeDIE(LexicalScope *Scope, DIE &ParentScopeDIE);
 
-  /// Get if available or create a new DW_TAG_lexical_block for the given
-  /// LexicalScope and attach DW_AT_low_pc/DW_AT_high_pc labels.
-  DIE *getOrCreateLexicalBlockDIE(LexicalScope *Scope, DIE &ParentDIE);
+  /// Construct new DW_TAG_lexical_block for this scope and
+  /// attach DW_AT_low_pc/DW_AT_high_pc labels.
+  DIE *constructLexicalScopeDIE(LexicalScope *Scope);
+
+  /// Get a DIE for the given DILexicalBlock.
+  /// Note that this function assumes that the DIE has been already created
+  /// and it's an error, if it hasn't.
+  DIE *getLexicalBlockDIE(const DILexicalBlock *LB);
 
   /// constructVariableDIE - Construct a DIE for the given DbgVariable.
   DIE *constructVariableDIE(DbgVariable &DV, bool Abstract = false);
@@ -229,11 +234,6 @@ class DwarfCompileUnit final : public DwarfUnit {
   /// This instance of 'getOrCreateContextDIE()' can handle DILocalScope.
   DIE *getOrCreateContextDIE(const DIScope *Ty) override;
 
-  /// Get DW_TAG_lexical_block for the given DILexicalBlock if available,
-  /// or the most close parent DIE, if no correspoding DW_TAG_lexical_block
-  /// exists.
-  DIE *getLocalContextDIE(const DILexicalBlock *LB);
-
   /// Construct a DIE for this subprogram scope.
   DIE &constructSubprogramScopeDIE(const DISubprogram *Sub,
                                    LexicalScope *Scope);
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index f0c13edaf4bd3..df2161410ad08 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -1229,13 +1229,12 @@ void DwarfDebug::beginModule(Module *M) {
         CU.getOrCreateGlobalVariableDIE(GV, sortGlobalExprs(GVMap[GV]));
     }
 
-    for (auto *Ty : CUNode->getEnumTypes()) {
-      assert(!isa_and_nonnull<DILocalScope>(Ty->getScope()) &&
-             "Unexpected function-local entity in 'enums' CU field.");
+    for (auto *Ty : CUNode->getEnumTypes())
       CU.getOrCreateTypeDIE(cast<DIType>(Ty));
-    }
 
     for (auto *Ty : CUNode->getRetainedTypes()) {
+      // The retained types array by design contains pointers to
+      // MDNodes rather than DIRefs. Unique them here.
       if (DIType *RT = dyn_cast<DIType>(Ty))
         // There is no point in force-emitting a forward declaration.
         CU.getOrCreateTypeDIE(RT);
@@ -1429,13 +1428,9 @@ void DwarfDebug::endModule() {
              "Unexpected function-local entity in 'imports' CU field.");
       CU->getOrCreateImportedEntityDIE(IE);
     }
-
-    // Emit function-local entities.
     for (const auto *D : CU->getDeferredLocalDecls()) {
       if (auto *IE = dyn_cast<DIImportedEntity>(D))
         CU->getOrCreateImportedEntityDIE(IE);
-      else if (auto *Ty = dyn_cast<DIType>(D))
-        CU->getOrCreateTypeDIE(Ty);
       else
         llvm_unreachable("Unexpected local retained node!");
     }
@@ -1533,8 +1528,6 @@ static const DILocalScope *getRetainedNodeScope(const MDNode *N) {
     S = L->getScope();
   else if (const auto *IE = dyn_cast<DIImportedEntity>(N))
     S = IE->getScope();
-  else if (const auto *T = dyn_cast<DIType>(N))
-    S = T->getScope();
   else
     llvm_unreachable("Unexpected retained node!");
 
diff --git a/llvm/lib/IR/DIBuilder.cpp b/llvm/lib/IR/DIBuilder.cpp
index 242ca44794262..ae1a1a37d7c74 100644
--- a/llvm/lib/IR/DIBuilder.cpp
+++ b/llvm/lib/IR/DIBuilder.cpp
@@ -29,7 +29,7 @@ DIBuilder::DIBuilder(Module &m, bool AllowUnresolvedNodes, DICompileUnit *CU)
       AllowUnresolvedNodes(AllowUnresolvedNodes) {
   if (CUNode) {
     if (const auto &ETs = CUNode->getEnumTypes())
-      EnumTypes.assign(ETs.begin(), ETs.end());
+      AllEnumTypes.assign(ETs.begin(), ETs.end());
     if (const auto &RTs = CUNode->getRetainedTypes())
       AllRetainTypes.assign(RTs.begin(), RTs.end());
     if (const auto &GVs = CUNode->getGlobalVariables())
@@ -66,10 +66,10 @@ void DIBuilder::finalize() {
     return;
   }
 
-  if (!EnumTypes.empty())
+  if (!AllEnumTypes.empty())
     CUNode->replaceEnumTypes(MDTuple::get(
-        VMContext, SmallVector<Metadata *, 16>(EnumTypes.begin(),
-                                               EnumTypes.end())));
+        VMContext, SmallVector<Metadata *, 16>(AllEnumTypes.begin(),
+                                               AllEnumTypes.end())));
 
   SmallVector<Metadata *, 16> RetainValues;
   // Declarations and definitions of the same type may be retained. Some
@@ -334,13 +334,10 @@ DIDerivedType *DIBuilder::createTypedef(DIType *Ty, StringRef Name,
                                         DIScope *Context, uint32_t AlignInBits,
                                         DINode::DIFlags Flags,
                                         DINodeArray Annotations) {
-  auto *T =
-      DIDerivedType::get(VMContext, dwarf::DW_TAG_typedef, Name, File, LineNo,
-                         getNonCompileUnitScope(Context), Ty, 0, AlignInBits, 0,
-                         std::nullopt, Flags, nullptr, Annotations);
-  if (isa_and_nonnull<DILocalScope>(Context))
-    getSubprogramNodesTrackingVector(Context).emplace_back(T);
-  return T;
+  return DIDerivedType::get(VMContext, dwarf::DW_TAG_typedef, Name, File,
+                            LineNo, getNonCompileUnitScope(Context), Ty, 0,
+                            AlignInBits, 0, std::nullopt, Flags, nullptr,
+                            Annotations);
 }
 
 DIDerivedType *DIBuilder::createFriend(DIType *Ty, DIType *FriendTy) {
@@ -488,8 +485,6 @@ DICompositeType *DIBuilder::createClassType(
       OffsetInBits, Flags, Elements, 0, VTableHolder,
       cast_or_null<MDTuple>(TemplateParams), UniqueIdentifier);
   trackIfUnresolved(R);
-  if (isa_and_nonnull<DILocalScope>(Context))
-    getSubprogramNodesTrackingVector(Context).emplace_back(R);
   return R;
 }
 
@@ -503,8 +498,6 @@ DICompositeType *DIBuilder::createStructType(
       getNonCompileUnitScope(Context), DerivedFrom, SizeInBits, AlignInBits, 0,
       Flags, Elements, RunTimeLang, VTableHolder, nullptr, UniqueIdentifier);
   trackIfUnresolved(R);
-  if (isa_and_nonnull<DILocalScope>(Context))
-    getSubprogramNodesTrackingVector(Context).emplace_back(R);
   return R;
 }
 
@@ -517,8 +510,6 @@ DICompositeType *DIBuilder::createUnionType(
       getNonCompileUnitScope(Scope), nullptr, SizeInBits, AlignInBits, 0, Flags,
       Elements, RunTimeLang, nullptr, nullptr, UniqueIdentifier);
   trackIfUnresolved(R);
-  if (isa_and_nonnull<DILocalScope>(Scope))
-    getSubprogramNodesTrackingVector(Scope).emplace_back(R);
   return R;
 }
 
@@ -551,10 +542,7 @@ DICompositeType *DIBuilder::createEnumerationType(
       getNonCompileUnitScope(Scope), UnderlyingType, SizeInBits, AlignInBits, 0,
       IsScoped ? DINode::FlagEnumClass : DINode::FlagZero, Elements, 0, nullptr,
       nullptr, UniqueIdentifier);
-  if (isa_and_nonnull<DILocalScope>(Scope))
-    getSubprogramNodesTrackingVector(Scope).emplace_back(CTy);
-  else
-    EnumTypes.emplace_back(CTy);
+  AllEnumTypes.emplace_back(CTy);
   trackIfUnresolved(CTy);
   return CTy;
 }
@@ -652,8 +640,6 @@ DIBuilder::createForwardDecl(unsigned Tag, StringRef Name, DIScope *Scope,
       SizeInBits, AlignInBits, 0, DINode::FlagFwdDecl, nullptr, RuntimeLang,
       nullptr, nullptr, UniqueIdentifier);
   trackIfUnresolved(RetTy);
-  if (isa_and_nonnull<DILocalScope>(Scope))
-    getSubprogramNodesTrackingVector(Scope).emplace_back(RetTy);
   return RetTy;
 }
 
@@ -670,8 +656,6 @@ DICompositeType *DIBuilder::createReplaceableCompositeType(
           nullptr, Annotations)
           .release();
   trackIfUnresolved(RetTy);
-  if (isa_and_nonnull<DILocalScope>(Scope))
-    getSubprogramNodesTrackingVector(Scope).emplace_back(RetTy);
   return RetTy;
 }
 
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 8f929dc7f48ab..293d443d754a4 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -1384,9 +1384,9 @@ void Verifier::visitDISubprogram(const DISubprogram &N) {
     CheckDI(Node, "invalid retained nodes list", &N, RawNode);
     for (Metadata *Op : Node->operands()) {
       CheckDI(Op && (isa<DILocalVariable>(Op) || isa<DILabel>(Op) ||
-                     isa<DIImportedEntity>(Op) || isa<DIType>(Op)),
-              "invalid retained nodes, expected DILocalVariable, DILabel, "
-              "DIImportedEntity or DIType",
+                     isa<DIImportedEntity>(Op)),
+              "invalid retained nodes, expected DILocalVariable, DILabel or "
+              "DIImportedEntity",
               &N, Node, Op);
     }
   }
diff --git a/llvm/test/Bitcode/upgrade-cu-locals.ll b/llvm/test/Bitcode/upgrade-cu-locals.ll
index 7052ab379bea4..9a590f0fc0774 100644
--- a/llvm/test/Bitcode/upgrade-cu-locals.ll
+++ b/llvm/test/Bitcode/upgrade-cu-locals.ll
@@ -1,4 +1,4 @@
-; Test moving of local imports/enums from DICompileUnit to DISubprogram's 'retainedNodes'
+; Test moving of local imports from DICompileUnit's 'imports' to DISubprogram's 'retainedNodes'
 ;
 ; RUN: llvm-dis -o - %s.bc | FileCheck %s
 
@@ -31,36 +31,31 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo
 ; CHECK: !4 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t4"
 
 ; CHECK: !5 = !{}
-; CHECK: !6 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !7, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !8, imports: !5, nameTableKind: GNU)
-; CHECK: !7 = !DIFile(filename: "b.cpp"
-; CHECK: !8 = !{!9}
-; CHECK: !9 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum2", scope: !6, file: !7, line: 4, size: 8, align: 8, elements: !5)
-
-; CHECK: !16 = distinct !DISubprogram(name: "main", scope: !7, file: !7, line: 2, type: !17, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !20)
-; CHECK: !20 = !{!21}
-; CHECK: !21 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !22, entity: !25,
-; CHECK: !22 = !DILexicalBlock(scope: !23, file: !7, line: 7, column: 35)
-; CHECK: !23 = !DILexicalBlock(scope: !24, file: !7, line: 7, column: 35)
-; CHECK: !24 = !DILexicalBlock(scope: !16, file: !7, line: 7, column: 35)
-; CHECK: !25 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t5", scope: !22,
-
-; CHECK: !27 = distinct !DISubprogram(name: "f1", linkageName: "_Z2f1v", scope: !1, file: !1, line: 3, type: !28, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !30)
-; CHECK: !30 = !{!31, !34, !36}
-; CHECK: !31 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !27, entity: !32,
-; CHECK: !32 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t1",
-; CHECK: !34 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !27, entity: !35,
-; CHECK: !35 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t2",
-; CHECK: !36 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !27, entity: !37,
-; CHECK: !37 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t3",
-
-; CHECK: !42 = distinct !DISubprogram(name: "main2", scope: !7, file: !7, line: 10, type: !17, scopeLine: 10, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !43)
-; CHECK: !43 = !{!44, !46, !48, !49}
-; CHECK: !44 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !42, entity: !45,
-; CHECK: !45 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t6"
-; CHECK: !46 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !42, entity: !47,
-; CHECK: !47 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t7",
-; CHECK: !48 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum", scope: !42, file: !7, line: 3, size: 8, align: 8, elements: !5)
-; CHECK: !49 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum3", scope: !42, file: !7, line: 5, size: 8, align: 8, elements: !5)
+; CHECK: !6 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !7, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, imports: !5, nameTableKind: GNU)
+
+; CHECK: !14 = distinct !DISubprogram(name: "main", scope: !7, file: !7, line: 2, type: !15, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !18)
+; CHECK: !18 = !{!19}
+; CHECK: !19 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !20, entity: !23,
+; CHECK: !20 = !DILexicalBlock(scope: !21, file: !7, line: 7, column: 35)
+; CHECK: !21 = !DILexicalBlock(scope: !22, file: !7, line: 7, column: 35)
+; CHECK: !22 = !DILexicalBlock(scope: !14, file: !7, line: 7, column: 35)
+; CHECK: !23 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t5", scope: !20,
+
+; CHECK: !25 = distinct !DISubprogram(name: "f1", linkageName: "_Z2f1v", scope: !1, file: !1, line: 3, type: !26, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !28)
+; CHECK: !28 = !{!29, !32, !34}
+; CHECK: !29 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !25, entity: !30,
+; CHECK: !30 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t1",
+; CHECK: !32 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !25, entity: !33,
+; CHECK: !33 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t2",
+; CHECK: !34 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !25, entity: !35,
+; CHECK: !35 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t3",
+
+; CHECK: !40 = distinct !DISubprogram(name: "main2", scope: !7, file: !7, line: 10, type: !15, scopeLine: 10, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !41)
+; CHECK: !41 = !{!42, !44}
+; CHECK: !42 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !40, entity: !43,
+; CHECK: !43 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t6"
+; CHECK: !44 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !40, entity: !45,
+; CHECK: !45 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t7",
 
 
 !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, imports: !2, nameTableKind: GNU)
@@ -87,7 +82,7 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo
 ; Leave t4 in CU
 !14 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !0, entity: !15, file: !1, line: 3)
 !15 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t4", scope: !0, file: !1, line: 1, size: 8, flags: DIFlagTypePassByValue, elements: !7, identifier: "_ZTSN2ns2t4E")
-!16 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !17, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, imports: !18, enums: !50, nameTableKind: GNU)
+!16 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !17, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, imports: !18, nameTableKind: GNU)
 !17 = !DIFile(filename: "b.cpp", directory: "/")
 !18 = !{!19, !28, !31}
 
@@ -121,12 +116,3 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo
 !41 = distinct !DILocation(line: 3, column: 3, scope: !23)
 !42 = !DILocation(line: 3, column: 41, scope: !4, inlinedAt: !41)
 !43 = !DILocation(line: 4, column: 1, scope: !23)
-
-!50 = !{!51, !52, !53}
-; Move to main2
-!51 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum", scope: !29, file: !17, line: 3, size: 8, align: 8, elements: !7)
-; Leave in b.cpp's CU
-!52 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum2", scope: !16, file: !17, line: 4, size: 8, align: 8, elements: !7)
-; Move to main2
-!53 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum3", scope: !29, file: !17, line: 5, size: 8, align: 8, elements: !7)
-
diff --git a/llvm/test/Bitcode/upgrade-cu-locals.ll.bc b/llvm/test/Bitcode/upgrade-cu-locals.ll.bc
index f73d05668af5b16f9ee03e400e2877faab339a0d..9d0ea8fd9a3704ea1fb3a92250500addc75f0587 100644
GIT binary patch
delta 1011
zcmYjPT}&fY6h1R;?+nAR8K7ukpbT3~NOmQ+|Ckt^#_fv6bXy??4aQDuk;YOAbd5%0
zrv<}iQ#P<|%$oR6v-sc!%?9*sAEv1*sKCpvhU}79AB@J-1png8D&A>B@aE3>`OdxH
z`DW%Ebf_IG{nX^@Bi~TU>Tgesr^SEYyYtha=j~yWY!HBg<7MPOE-e79rviCYP}RO~
z{gy^qhXoMu+C9!tM`Yd7K=*rCbF3X;n<h+~OH`|YIw-tvv~}+*HOenH<t0qpa4y8=
zHS>ZdEU$I902B_4N`Kn>%1fgo0Y`Gqk@ou2o|DYupHkvUM?9G>Oq(ubP_5n(a4sm=
zSie0NwAk3&#7VXnE9Y+cR78YGZ0=kSKu|xb4S$DwriFb<1z>Yhhyjpv9A9-m_myaT
zL(g*3@Li&q)A9VPEq8uL*&bj)mDsk61ggK}NN7uR?PWy%O-xFsCjs;lE_N)IMV>_D
zKJ<VL#yUlNktuhbP{;%YAPW0`^xgcQLag7?M(?gN<`w4afdQb^OCrcbyRM_~7ovJk
zNHh~b24E!gv^6YS+3S+7BYC33XFb+2(9pq;LKfhUKA1%l7^Y!o3AQ_nWw=G*URruh
zMutPA&9iE$Y~eK#oI*-qW-;T0g%$|!2;gj`Q;a5?4q3o2P@nm%-O&sQe%1rxYjsbU
z=Cxy{^=ua}TlJP3D30MR*uj5hdmoe^XJpi20TX^o;0UY4X?z)1s<H2RgJn{|6$#z<
zR;n`*f?W)!5|HZNntA7khsjPT59ziZB&B9$Lu%dtZpAW|tfCTdC6Y?b=`i<!I2Ng*
zNfkNTu9e33_iOjYx4+mM|MHjft$U@G=auN@>vPZ8&dgmG{g!t_+n1JiKi&Q4*X8vq
zpX}VuKUzy9i#hYNWpjYqoMKH>Ktt-x<rKPch!#4@&scBPKP}i01Gz9h`{B{i(UTAU
zA-4KAF_pZ}UMzRsq;W(hvxgQP(I6jBp>;^$Xi)Hp1+A%zQVaK+&J4<A;u~v5YD|{n
zlyG{H{@geu1nPyCNvpFYYBLlg1s69s5SfVLsK@JZx`(jG?sT~nm&;-I+LfWlT#gBk
cSBb^kZo4~HxXhgw;!~;V+4S6CBGKsbANlGKqW}N^

delta 1080
zcmYk5e`p(J7{{NxT<<RFwOop6V|rQMH7e;k%bOqVBKYItmQkU&t8>^v2T79^Yj3eX
zCa4ST^$4!hY-CFq`$I=`D*iELuKy_e=R!ISm`cGRL56hTA0ibxhWMA%_q{6H%X`oB
z-1ohE-p}*!zJ}+nr}PA4Tzv8e=ETMO&yU}{_3PDJt*yNt=um-!|A&k}H9J|6zN*x)
zz`pltPnw1RR8~Q?N^cBEwg8N+ilr9PDjEQ4=T<*p+z#fi@<&_{iTssC`dw0W77=EH
z*c_9SV-gA2gRx8VvGQFm-JQ(6b&5U_YNDso)49lW?sWONHl>_yJF*18yXh_f*wo~p
zm<*emHz?jF4h4_b4>b}uctVLpliLG87yzu1pRs>l{*5sRvWb=j0!DRwX=wDE!o5S^
zOf<4d(z>W#%y#_LCnWZe`V1s?y-g$s)sK#yRhD8G6zL2mOMNo{bO~2Hg=K}-PyV<3
zq#>kxqe*`rGx=)KWD%^aF+h`3j$?WG#v>f#G%w}o32^!&9A2suj-yWr%l!{I44%|a
zL4<$~0r4859CHsjn1Ia@0;sw`ra%slionW?#uo{nWGL`pFMPtkj$s~}XJN(nVGSA#
z4zOq@#XFE-No&eR!r!V%R>}e1C})ip;CBXV%(E=PadK-*f}<Bi)~=kQK=3&0zR$P?
zFab!z?O}~vR!8z0DHMT0x@`|!-w+nJg$D{euBG5HOG<>AheRU;84oDiwJ=Y7|E^bU
zPf#aaZ~liW_I^`?Z3Zt9R(*qaScJ9SUwv_3Q<kDq+B#R0>fX~?T-W%k-X<>8UgL=j
zd}ekQW|i;f7Vq_tsi(X3-JsT;L+v-o1XsyBp4;T7(%n-D;dAR-CS<UohPl7N`l(K4
zr0Ys`qSe~^Wumbb`*GsCKMuR^K<6vz%<9a-OTmGKYrRRW?O0>w{Q6hxpZ$6M`h_o>
zw~J42>AA|JXZ&N&#J$xHv5k3on>Xg(ukv3b;h>Ln<a)b(XM#LwT?+4icV}nkh0TA6
zo&B4bdNC2MEc==)o>a+U<_nRGCQVoQRnT!plNqI)a{8Ff@)_sAV5(!c!vJ$=_ozlX
z<-P*D&c#bd?4=WAs{s;qIV!309p_*u{7g7Dd~zy26wOR#W081lI2)ci5Q#<(#PJXw
bjz`1MY<74kUf$<AES)45abZx`X{vt#JnkA(

diff --git a/llvm/test/DebugInfo/Generic/inlined-local-type.ll b/llvm/test/DebugInfo/Generic/inlined-local-type.ll
deleted file mode 100644
index a6f7d294de928..0000000000000
--- a/llvm/test/DebugInfo/Generic/inlined-local-type.ll
+++ /dev/null
@@ -1,128 +0,0 @@
-; RUN: %llc_dwarf -O0 -filetype=obj < %s | llvm-dwarfdump -debug-info - | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s
-; REQUIRES: object-emission
-
-; inline __attribute__((always_inline))
-; int removed() { struct A {int i;}; struct A a; return a.i++; }
-;
-; __attribute__((always_inline))
-; int not_removed() { struct B {int i;}; struct B b; return b.i++; }
-;
-; int foo() { return removed() + not_removed(); }}
-
-; Ensure that function-local types have the correct subprogram parent even if
-; those subprograms are inlined.
-
-; CHECK: DW_TAG_compile_unit
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_AT_abstract_origin	({{0x.*}} "not_removed")
-; CHECK:     DW_TAG_variable
-; CHECK:     NULL
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_AT_name	("removed")
-; CHECK: [[A:0x.*]]: DW_TAG_structure_type
-; CHECK:       DW_AT_name	("A")
-; CHECK:       DW_TAG_member
-; CHECK:       NULL
-; CHECK:     DW_TAG_variable
-; CHECK:       DW_AT_type	([[A]] "A")
-; CHECK:     NULL
-; CHECK:   DW_TAG_base_type
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_AT_name	("not_removed")
-; CHECK: [[B:0x.*]]: DW_TAG_structure_type
-; CHECK:       DW_AT_name	("B")
-; CHECK:       DW_TAG_member
-; CHECK:       NULL
-; CHECK:     DW_TAG_variable
-; CHECK:       DW_AT_type	([[B]] "B")
-; CHECK:     NULL
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_TAG_inlined_subroutine
-; CHECK:       DW_TAG_variable
-; CHECK:       NULL
-; CHECK:     DW_TAG_inlined_subroutine
-; CHECK:       DW_TAG_variable
-; CHECK:       NULL
-; CHECK:     NULL
-; CHECK:   NULL
-
-%struct.B = type { i32 }
-%struct.A = type { i32 }
-
-define dso_local i32 @not_removed() !dbg !12 {
-  %1 = alloca %struct.B, align 4
-  call void @llvm.dbg.declare(metadata %struct.B* %1, metadata !18, metadata !DIExpression()), !dbg !22
-  %2 = getelementptr inbounds %struct.B, %struct.B* %1, i32 0, i32 0, !dbg !23
-  %3 = load i32, i32* %2, align 4, !dbg !24
-  %4 = add nsw i32 %3, 1, !dbg !24
-  store i32 %4, i32* %2, align 4, !dbg !24
-  ret i32 %3, !dbg !25
-}
-
-declare void @llvm.dbg.declare(metadata, metadata, metadata)
-
-define dso_local i32 @foo() !dbg !26 {
-  %1 = alloca %struct.A, align 4
-  %2 = alloca %struct.B, align 4
-  call void @llvm.dbg.declare(metadata %struct.A* %1, metadata !27, metadata !DIExpression()), !dbg !32
-  %3 = getelementptr inbounds %struct.A, %struct.A* %1, i32 0, i32 0, !dbg !34
-  %4 = load i32, i32* %3, align 4, !dbg !35
-  %5 = add nsw i32 %4, 1, !dbg !35
-  store i32 %5, i32* %3, align 4, !dbg !35
-  call void @llvm.dbg.declare(metadata %struct.B* %2, metadata !18, metadata !DIExpression()), !dbg !36
-  %6 = getelementptr inbounds %struct.B, %struct.B* %2, i32 0, i32 0, !dbg !38
-  %7 = load i32, i32* %6, align 4, !dbg !39
-  %8 = add nsw i32 %7, 1, !dbg !39
-  store i32 %8, i32* %6, align 4, !dbg !39
-  %9 = add nsw i32 %4, %7, !dbg !40
-  ret i32 %9, !dbg !41
-}
-
-!llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!2, !3, !4, !5, !6, !7, !8, !9, !10}
-!llvm.ident = !{!11}
-
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
-!1 = !DIFile(filename: "inlined-local-type.cpp", directory: "")
-!2 = !{i32 7, !"Dwarf Version", i32 4}
-!3 = !{i32 2, !"Debug Info Version", i32 3}
-!4 = !{i32 1, !"wchar_size", i32 4}
-!5 = !{i32 1, !"branch-target-enforcement", i32 0}
-!6 = !{i32 1, !"sign-return-address", i32 0}
-!7 = !{i32 1, !"sign-return-address-all", i32 0}
-!8 = !{i32 1, !"sign-return-address-with-bkey", i32 0}
-!9 = !{i32 7, !"uwtable", i32 1}
-!10 = !{i32 7, !"frame-pointer", i32 1}
-!11 = !{!"clang version 14.0.0"}
-!12 = distinct !DISubprogram(name: "not_removed", scope: !13, file: !13, line: 5, type: !14, scopeLine: 5, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !43)
-!13 = !DIFile(filename: "inlined-local-type.cpp", directory: "")
-!14 = !DISubroutineType(types: !15)
-!15 = !{!16}
-!16 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-!17 = !{}
-!18 = !DILocalVariable(name: "b", scope: !12, file: !13, line: 5, type: !19)
-!19 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "B", scope: !12, file: !13, line: 5, size: 32, elements: !20)
-!20 = !{!21}
-!21 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !19, file: !13, line: 5, baseType: !16, size: 32)
-!22 = !DILocation(line: 5, column: 49, scope: !12)
-!23 = !DILocation(line: 5, column: 61, scope: !12)
-!24 = !DILocation(line: 5, column: 62, scope: !12)
-!25 = !DILocation(line: 5, column: 52, scope: !12)
-!26 = distinct !DISubprogram(name: "foo", scope: !13, file: !13, line: 7, type: !14, scopeLine: 7, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !17)
-!27 = !DILocalVariable(name: "a", scope: !28, file: !13, line: 2, type: !29)
-!28 = distinct !DISubprogram(name: "removed", scope: !13, file: !13, line: 2, type: !14, scopeLine: 2, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !42)
-!29 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A", scope: !28, file: !13, line: 2, size: 32, elements: !30)
-!30 = !{!31}
-!31 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !29, file: !13, line: 2, baseType: !16, size: 32)
-!32 = !DILocation(line: 2, column: 45, scope: !28, inlinedAt: !33)
-!33 = distinct !DILocation(line: 7, column: 20, scope: !26)
-!34 = !DILocation(line: 2, column: 57, scope: !28, inlinedAt: !33)
-!35 = !DILocation(line: 2, column: 58, scope: !28, inlinedAt: !33)
-!36 = !DILocation(line: 5, column: 49, scope: !12, inlinedAt: !37)
-!37 = distinct !DILocation(line: 7, column: 32, scope: !26)
-!38 = !DILocation(line: 5, column: 61, scope: !12, inlinedAt: !37)
-!39 = !DILocation(line: 5, column: 62, scope: !12, inlinedAt: !37)
-!40 = !DILocation(line: 7, column: 30, scope: !26)
-!41 = !DILocation(line: 7, column: 13, scope: !26)
-!42 = !{!29}
-!43 = !{!19}
diff --git a/llvm/test/DebugInfo/Generic/lexical-block-retained-types.ll b/llvm/test/DebugInfo/Generic/lexical-block-retained-types.ll
deleted file mode 100644
index 64de1323c98c6..0000000000000
--- a/llvm/test/DebugInfo/Generic/lexical-block-retained-types.ll
+++ /dev/null
@@ -1,55 +0,0 @@
-; RUN: %llc_dwarf -O0 -filetype=obj < %s | llvm-dwarfdump - | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s
-
-; Test that retained unused (unreferenced) types emission.
-
-; Compiled from
-; $ clang -cc1 -debug-info-kind=unused-types test.cpp -emit-llvm
-
-; void test_unused() {
-;   struct Y {};
-;   {
-;     struct X {};
-;   }
-; }
-
-; CHECK: DW_TAG_compile_unit
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_AT_name	("test_unused")
-; CHECK:     DW_TAG_structure_type
-; CHECK:       DW_AT_name	("Y")
-
-; FIXME: here should be DW_TAG_lexical_block as a parent of structure 'X'.
-; But it's not possible to reliably emit a lexical block for which a LexicalScope
-; wasn't created, so we just fallback to the most close parent DIE
-; (see DwarfCompileUnit::getOrCreateLexicalBlockDIE() for details).
-
-; CHECK:     DW_TAG_structure_type
-; CHECK:       DW_AT_name	("X")
-; CHECK:     NULL
-; CHECK:   NULL
-
-define dso_local void @_Z11test_unusedv() !dbg !5 {
-entry:
-  ret void, !dbg !16
-}
-
-!llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!13, !14}
-!llvm.ident = !{!15}
-
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 15.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, retainedTypes: !2, splitDebugInlining: false, nameTableKind: None)
-!1 = !DIFile(filename: "<stdin>", directory: "/")
-!2 = !{!3, !10}
-!3 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Y", scope: !5, file: !4, line: 2, size: 8, flags: DIFlagTypePassByValue, elements: !8)
-!4 = !DIFile(filename: "test.cpp", directory: "/")
-!5 = distinct !DISubprogram(name: "test_unused", linkageName: "_Z11test_unusedv", scope: !4, file: !4, line: 1, type: !6, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !9)
-!6 = !DISubroutineType(types: !7)
-!7 = !{null}
-!8 = !{}
-!9 = !{!3, !10}
-!10 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "X", scope: !11, file: !4, line: 4, size: 8, flags: DIFlagTypePassByValue, elements: !8)
-!11 = distinct !DILexicalBlock(scope: !5, file: !4, line: 3, column: 3)
-!13 = !{i32 2, !"Debug Info Version", i32 3}
-!14 = !{i32 1, !"wchar_size", i32 4}
-!15 = !{!"clang version 15.0.0"}
-!16 = !DILocation(line: 6, column: 1, scope: !5)
diff --git a/llvm/test/DebugInfo/Generic/lexical-block-types.ll b/llvm/test/DebugInfo/Generic/lexical-block-types.ll
deleted file mode 100644
index aef59984351a7..0000000000000
--- a/llvm/test/DebugInfo/Generic/lexical-block-types.ll
+++ /dev/null
@@ -1,425 +0,0 @@
-; RUN: %llc_dwarf -O0 -filetype=obj < %s | llvm-dwarfdump -debug-info - | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s
-; REQUIRES: object-emission
-
-; inline __attribute__((always_inline))
-; void removed() {
-;   struct A1 { int i; };
-;   typedef int Int1;
-;   {
-;     struct I1 { Int1 j; };
-;     struct C1 { typedef char Char1; Char1 c; };
-;     A1 a1; a1.i++;
-;     {
-;       I1 i1; i1.j++;
-;       C1 c1; c1.c++;
-;     }
-;   }
-; }
-;
-; __attribute__((always_inline))
-; void not_removed() {
-;   struct A2 { int i; };
-;   typedef int Int2;
-;   {
-;     struct I2 { Int2 j; };
-;     struct C2 { typedef char Char2; Char2 c; };
-;     A2 a2; a2.i++;
-;     {
-;       I2 i2; i2.j++;
-;       C2 c2; c2.c++;
-;     }
-;   }
-; }
-;
-; void foo() {
-;   struct A3 { int i; };
-;   typedef int Int3;
-;   {
-;     struct I3 { Int3 j; };
-;     {
-;       struct C3 { typedef char Char3; Char3 c; };
-;       A3 a3; a3.i++;
-;       {
-;         I3 i3; i3.j++;
-;         C3 c3; c3.c++;
-;       }
-;     }
-;   }
-;   removed();
-;   not_removed();
-; }
-;
-; CHECK: DW_TAG_compile_unit
-
-; Out-of-line definition of `not_removed()` shouldn't contain any debug info for types.
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_AT_abstract_origin	{{.*}} "_Z11not_removedv"
-; CHECK:     DW_TAG_lexical_block
-; CHECK:       DW_TAG_variable
-; CHECK:         DW_AT_abstract_origin	{{.*}} "a2"
-; CHECK:       DW_TAG_lexical_block
-; CHECK:         DW_TAG_variable
-; CHECK:           DW_AT_abstract_origin	{{.*}} "i2"
-; CHECK:         DW_TAG_variable
-; CHECK:           DW_AT_abstract_origin	{{.*}} "c2"
-; CHECK:         NULL
-; CHECK:       NULL
-; CHECK:     NULL
-
-; Abstract definition of `removed()`.
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_AT_name	("removed")
-; CHECK:     DW_AT_inline	(DW_INL_inlined)
-
-; I1 and C1 defined in the first lexical block, typedef Char1 is a child of C1.
-; CHECK:     DW_TAG_lexical_block
-; CHECK:       DW_TAG_variable
-; CHECK:         DW_AT_name	("a1")
-; CHECK:         DW_AT_type	{{.*}} "A1"
-; CHECK:       DW_TAG_lexical_block
-; CHECK:         DW_TAG_variable
-; CHECK:           DW_AT_type	{{.*}} "I1"
-; CHECK:         DW_TAG_variable
-; CHECK:           DW_AT_type	{{.*}} "C1"
-; CHECK:         NULL
-; CHECK:       DW_TAG_structure_type
-; CHECK:         DW_AT_name	("I1")
-; CHECK:         DW_TAG_member
-; CHECK:           DW_AT_type	{{.*}} "Int1"
-; CHECK:         NULL
-; CHECK:       DW_TAG_structure_type
-; CHECK:         DW_AT_name	("C1")
-; CHECK:         DW_TAG_member
-; CHECK:           DW_AT_type	{{.*}} "Char1"
-; CHECK:         DW_TAG_typedef
-; CHECK:           DW_AT_name	("Char1")
-; CHECK:         NULL
-; CHECK:       NULL
-
-; A1 and typedef Int1 defined in the subprogram scope.
-; CHECK:     DW_TAG_structure_type
-; CHECK:       DW_AT_name	("A1")
-; CHECK:       DW_TAG_member
-; CHECK:       NULL
-; CHECK:     DW_TAG_typedef
-; CHECK:       DW_AT_name	("Int1")
-; CHECK:     NULL
-
-; CHECK:   DW_TAG_base_type
-; CHECK:   DW_TAG_base_type
-
-; Abstract definition of `not_removed()`.
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_AT_name	("not_removed")
-; CHECK:     DW_AT_inline	(DW_INL_inlined)
-
-; I2 and C2 defined in the first lexical block, typedef Char2 is a child of C2.
-; CHECK:     DW_TAG_lexical_block
-; CHECK:       DW_TAG_variable
-; CHECK:         DW_AT_name	("a2")
-; CHECK:         DW_AT_type	{{.*}} "A2"
-; CHECK:       DW_TAG_lexical_block
-; CHECK:         DW_TAG_variable
-; CHECK:           DW_AT_name	("i2")
-; CHECK:           DW_AT_type	{{.*}} "I2"
-; CHECK:         DW_TAG_variable
-; CHECK:           DW_AT_name	("c2")
-; CHECK:           DW_AT_type	{{.*}} "C2"
-; CHECK:         NULL
-; CHECK:       DW_TAG_structure_type
-; CHECK:         DW_AT_name	("I2")
-; CHECK:         DW_TAG_member
-; CHECK:           DW_AT_type	{{.*}} "Int2"
-; CHECK:         NULL
-; CHECK:       DW_TAG_structure_type
-; CHECK:         DW_AT_name	("C2")
-; CHECK:         DW_TAG_member
-; CHECK:           DW_AT_type	{{.*}} "Char2"
-; CHECK:         DW_TAG_typedef
-; CHECK:           DW_AT_name	("Char2")
-; CHECK:         NULL
-; CHECK:       NULL
-
-; A2 and typedef Int2 defined in subprogram scope.
-; CHECK:     DW_TAG_structure_type
-; CHECK:       DW_AT_name	("A2")
-; CHECK:       DW_TAG_member
-; CHECK:       NULL
-; CHECK:     DW_TAG_typedef
-; CHECK:       DW_AT_name	("Int2")
-; CHECK:     NULL
-
-; Definition of `foo()`.
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_AT_name	("foo")
-
-; CHECK:     DW_TAG_lexical_block
-; CHECK:       DW_TAG_lexical_block
-; CHECK:         DW_TAG_variable
-; CHECK:           DW_AT_name	("a3")
-; CHECK:           DW_AT_type	{{.*}} "A3"
-; CHECK:         DW_TAG_lexical_block
-; CHECK:           DW_TAG_variable
-; CHECK:             DW_AT_name	("i3")
-; CHECK:             DW_AT_type	{{.*}} "I3"
-; CHECK:           DW_TAG_variable
-; CHECK:             DW_AT_name	("c3")
-; CHECK:             DW_AT_type	{{.*}} "C3"
-; CHECK:           NULL
-
-; C3 has the inner lexical block scope, typedef Char3 is a child of C3.
-; CHECK:         DW_TAG_structure_type
-; CHECK:           DW_AT_name	("C3")
-; CHECK:           DW_TAG_member
-; CHECK:             DW_AT_type	{{.*}} "Char3"
-; CHECK:           DW_TAG_typedef
-; CHECK:             DW_AT_name	("Char3")
-; CHECK:           NULL
-; CHECK:         NULL
-
-; I3 has the outer lexical block scope.
-; CHECK:       DW_TAG_structure_type
-; CHECK:         DW_AT_name	("I3")
-; CHECK:         DW_TAG_member
-; CHECK:           DW_AT_type	{{.*}} "Int3"
-; CHECK:         NULL
-; CHECK:       NULL
-
-; CHECK:     DW_TAG_inlined_subroutine
-; CHECK:       DW_AT_abstract_origin	{{.*}} "_Z7removedv"
-; CHECK:       DW_TAG_lexical_block
-; CHECK:         DW_TAG_variable
-; CHECK:         DW_TAG_lexical_block
-; CHECK:           DW_TAG_variable
-; CHECK:           DW_TAG_variable
-; CHECK:           NULL
-; CHECK:         NULL
-; CHECK:       NULL
-
-; CHECK:     DW_TAG_inlined_subroutine
-; CHECK:       DW_AT_abstract_origin	{{.*}} "_Z11not_removedv"
-; CHECK:       DW_TAG_lexical_block
-; CHECK:         DW_TAG_variable
-; CHECK:         DW_TAG_lexical_block
-; CHECK:           DW_TAG_variable
-; CHECK:           DW_TAG_variable
-; CHECK:           NULL
-; CHECK:         NULL
-; CHECK:       NULL
-
-; A3 and Int3 defined within the subprogam scope.
-; CHECK:     DW_TAG_structure_type
-; CHECK:       DW_AT_name	("A3")
-; CHECK:       DW_TAG_member
-; CHECK:       NULL
-; CHECK:     DW_TAG_typedef
-; CHECK:       DW_AT_name	("Int3")
-; CHECK:     NULL
-; CHECK:   NULL
-
-%struct.A2 = type { i32 }
-%struct.I2 = type { i32 }
-%struct.C2 = type { i8 }
-%struct.A1 = type { i32 }
-%struct.I1 = type { i32 }
-%struct.C1 = type { i8 }
-%struct.A3 = type { i32 }
-%struct.I3 = type { i32 }
-%struct.C3 = type { i8 }
-
-define dso_local void @_Z11not_removedv() !dbg !8 {
-entry:
-  %a2 = alloca %struct.A2, align 4
-  %i2 = alloca %struct.I2, align 4
-  %c2 = alloca %struct.C2, align 1
-  call void @llvm.dbg.declare(metadata %struct.A2* %a2, metadata !12, metadata !DIExpression()), !dbg !18
-  %i = getelementptr inbounds %struct.A2, %struct.A2* %a2, i32 0, i32 0, !dbg !19
-  %0 = load i32, i32* %i, align 4, !dbg !20
-  %inc = add nsw i32 %0, 1, !dbg !20
-  store i32 %inc, i32* %i, align 4, !dbg !20
-  call void @llvm.dbg.declare(metadata %struct.I2* %i2, metadata !21, metadata !DIExpression()), !dbg !27
-  %j = getelementptr inbounds %struct.I2, %struct.I2* %i2, i32 0, i32 0, !dbg !28
-  %1 = load i32, i32* %j, align 4, !dbg !29
-  %inc1 = add nsw i32 %1, 1, !dbg !29
-  store i32 %inc1, i32* %j, align 4, !dbg !29
-  call void @llvm.dbg.declare(metadata %struct.C2* %c2, metadata !30, metadata !DIExpression()), !dbg !36
-  %c = getelementptr inbounds %struct.C2, %struct.C2* %c2, i32 0, i32 0, !dbg !37
-  %2 = load i8, i8* %c, align 1, !dbg !38
-  %inc2 = add i8 %2, 1, !dbg !38
-  store i8 %inc2, i8* %c, align 1, !dbg !38
-  ret void, !dbg !39
-}
-
-declare void @llvm.dbg.declare(metadata, metadata, metadata)
-
-define dso_local void @_Z3foov() !dbg !40 {
-entry:
-  %a1.i = alloca %struct.A1, align 4
-  %i1.i = alloca %struct.I1, align 4
-  %c1.i = alloca %struct.C1, align 1
-  %a2.i = alloca %struct.A2, align 4
-  %i2.i = alloca %struct.I2, align 4
-  %c2.i = alloca %struct.C2, align 1
-  %a3 = alloca %struct.A3, align 4
-  %i3 = alloca %struct.I3, align 4
-  %c3 = alloca %struct.C3, align 1
-  call void @llvm.dbg.declare(metadata %struct.A3* %a3, metadata !41, metadata !DIExpression()), !dbg !47
-  %i = getelementptr inbounds %struct.A3, %struct.A3* %a3, i32 0, i32 0, !dbg !48
-  %0 = load i32, i32* %i, align 4, !dbg !49
-  %inc = add nsw i32 %0, 1, !dbg !49
-  store i32 %inc, i32* %i, align 4, !dbg !49
-  call void @llvm.dbg.declare(metadata %struct.I3* %i3, metadata !50, metadata !DIExpression()), !dbg !56
-  %j = getelementptr inbounds %struct.I3, %struct.I3* %i3, i32 0, i32 0, !dbg !57
-  %1 = load i32, i32* %j, align 4, !dbg !58
-  %inc1 = add nsw i32 %1, 1, !dbg !58
-  store i32 %inc1, i32* %j, align 4, !dbg !58
-  call void @llvm.dbg.declare(metadata %struct.C3* %c3, metadata !59, metadata !DIExpression()), !dbg !64
-  %c = getelementptr inbounds %struct.C3, %struct.C3* %c3, i32 0, i32 0, !dbg !65
-  %2 = load i8, i8* %c, align 1, !dbg !66
-  %inc2 = add i8 %2, 1, !dbg !66
-  store i8 %inc2, i8* %c, align 1, !dbg !66
-  call void @llvm.dbg.declare(metadata %struct.A1* %a1.i, metadata !67, metadata !DIExpression()), !dbg !73
-  %i.i3 = getelementptr inbounds %struct.A1, %struct.A1* %a1.i, i32 0, i32 0, !dbg !75
-  %3 = load i32, i32* %i.i3, align 4, !dbg !76
-  %inc.i4 = add nsw i32 %3, 1, !dbg !76
-  store i32 %inc.i4, i32* %i.i3, align 4, !dbg !76
-  call void @llvm.dbg.declare(metadata %struct.I1* %i1.i, metadata !77, metadata !DIExpression()), !dbg !83
-  %j.i5 = getelementptr inbounds %struct.I1, %struct.I1* %i1.i, i32 0, i32 0, !dbg !84
-  %4 = load i32, i32* %j.i5, align 4, !dbg !85
-  %inc1.i6 = add nsw i32 %4, 1, !dbg !85
-  store i32 %inc1.i6, i32* %j.i5, align 4, !dbg !85
-  call void @llvm.dbg.declare(metadata %struct.C1* %c1.i, metadata !86, metadata !DIExpression()), !dbg !91
-  %c.i7 = getelementptr inbounds %struct.C1, %struct.C1* %c1.i, i32 0, i32 0, !dbg !92
-  %5 = load i8, i8* %c.i7, align 1, !dbg !93
-  %inc2.i8 = add i8 %5, 1, !dbg !93
-  store i8 %inc2.i8, i8* %c.i7, align 1, !dbg !93
-  call void @llvm.dbg.declare(metadata %struct.A2* %a2.i, metadata !12, metadata !DIExpression()), !dbg !94
-  %i.i = getelementptr inbounds %struct.A2, %struct.A2* %a2.i, i32 0, i32 0, !dbg !96
-  %6 = load i32, i32* %i.i, align 4, !dbg !97
-  %inc.i = add nsw i32 %6, 1, !dbg !97
-  store i32 %inc.i, i32* %i.i, align 4, !dbg !97
-  call void @llvm.dbg.declare(metadata %struct.I2* %i2.i, metadata !21, metadata !DIExpression()), !dbg !98
-  %j.i = getelementptr inbounds %struct.I2, %struct.I2* %i2.i, i32 0, i32 0, !dbg !99
-  %7 = load i32, i32* %j.i, align 4, !dbg !100
-  %inc1.i = add nsw i32 %7, 1, !dbg !100
-  store i32 %inc1.i, i32* %j.i, align 4, !dbg !100
-  call void @llvm.dbg.declare(metadata %struct.C2* %c2.i, metadata !30, metadata !DIExpression()), !dbg !101
-  %c.i = getelementptr inbounds %struct.C2, %struct.C2* %c2.i, i32 0, i32 0, !dbg !102
-  %8 = load i8, i8* %c.i, align 1, !dbg !103
-  %inc2.i = add i8 %8, 1, !dbg !103
-  store i8 %inc2.i, i8* %c.i, align 1, !dbg !103
-  ret void, !dbg !104
-}
-
-!llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!2, !3, !4, !5, !6}
-!llvm.ident = !{!7}
-
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
-!1 = !DIFile(filename: "test.cpp", directory: "/")
-!2 = !{i32 7, !"Dwarf Version", i32 4}
-!3 = !{i32 2, !"Debug Info Version", i32 3}
-!4 = !{i32 1, !"wchar_size", i32 4}
-!5 = !{i32 7, !"uwtable", i32 1}
-!6 = !{i32 7, !"frame-pointer", i32 2}
-!7 = !{!"clang version 14.0.0"}
-!8 = distinct !DISubprogram(name: "not_removed", linkageName: "_Z11not_removedv", scope: !1, file: !1, line: 17, type: !9, scopeLine: 17, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !105)
-!9 = !DISubroutineType(types: !10)
-!10 = !{null}
-!11 = !{}
-!12 = !DILocalVariable(name: "a2", scope: !13, file: !1, line: 23, type: !14)
-!13 = distinct !DILexicalBlock(scope: !8, file: !1, line: 20, column: 3)
-!14 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A2", scope: !8, file: !1, line: 18, size: 32, flags: DIFlagTypePassByValue, elements: !15)
-!15 = !{!16}
-!16 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !14, file: !1, line: 18, baseType: !17, size: 32)
-!17 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-!18 = !DILocation(line: 23, column: 8, scope: !13)
-!19 = !DILocation(line: 23, column: 15, scope: !13)
-!20 = !DILocation(line: 23, column: 16, scope: !13)
-!21 = !DILocalVariable(name: "i2", scope: !22, file: !1, line: 25, type: !23)
-!22 = distinct !DILexicalBlock(scope: !13, file: !1, line: 24, column: 5)
-!23 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "I2", scope: !13, file: !1, line: 21, size: 32, flags: DIFlagTypePassByValue, elements: !24)
-!24 = !{!25}
-!25 = !DIDerivedType(tag: DW_TAG_member, name: "j", scope: !23, file: !1, line: 21, baseType: !26, size: 32)
-!26 = !DIDerivedType(tag: DW_TAG_typedef, name: "Int2", scope: !8, file: !1, line: 19, baseType: !17)
-!27 = !DILocation(line: 25, column: 10, scope: !22)
-!28 = !DILocation(line: 25, column: 17, scope: !22)
-!29 = !DILocation(line: 25, column: 18, scope: !22)
-!30 = !DILocalVariable(name: "c2", scope: !22, file: !1, line: 26, type: !31)
-!31 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "C2", scope: !13, file: !1, line: 22, size: 8, flags: DIFlagTypePassByValue, elements: !32)
-!32 = !{!33}
-!33 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !31, file: !1, line: 22, baseType: !34, size: 8)
-!34 = !DIDerivedType(tag: DW_TAG_typedef, name: "Char2", scope: !31, file: !1, line: 22, baseType: !35)
-!35 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
-!36 = !DILocation(line: 26, column: 10, scope: !22)
-!37 = !DILocation(line: 26, column: 17, scope: !22)
-!38 = !DILocation(line: 26, column: 18, scope: !22)
-!39 = !DILocation(line: 29, column: 1, scope: !8)
-!40 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !1, file: !1, line: 31, type: !9, scopeLine: 31, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !107)
-!41 = !DILocalVariable(name: "a3", scope: !42, file: !1, line: 38, type: !44)
-!42 = distinct !DILexicalBlock(scope: !43, file: !1, line: 36, column: 5)
-!43 = distinct !DILexicalBlock(scope: !40, file: !1, line: 34, column: 3)
-!44 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A3", scope: !40, file: !1, line: 32, size: 32, flags: DIFlagTypePassByValue, elements: !45)
-!45 = !{!46}
-!46 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !44, file: !1, line: 32, baseType: !17, size: 32)
-!47 = !DILocation(line: 38, column: 10, scope: !42)
-!48 = !DILocation(line: 38, column: 17, scope: !42)
-!49 = !DILocation(line: 38, column: 18, scope: !42)
-!50 = !DILocalVariable(name: "i3", scope: !51, file: !1, line: 40, type: !52)
-!51 = distinct !DILexicalBlock(scope: !42, file: !1, line: 39, column: 7)
-!52 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "I3", scope: !43, file: !1, line: 35, size: 32, flags: DIFlagTypePassByValue, elements: !53)
-!53 = !{!54}
-!54 = !DIDerivedType(tag: DW_TAG_member, name: "j", scope: !52, file: !1, line: 35, baseType: !55, size: 32)
-!55 = !DIDerivedType(tag: DW_TAG_typedef, name: "Int3", scope: !40, file: !1, line: 33, baseType: !17)
-!56 = !DILocation(line: 40, column: 12, scope: !51)
-!57 = !DILocation(line: 40, column: 19, scope: !51)
-!58 = !DILocation(line: 40, column: 20, scope: !51)
-!59 = !DILocalVariable(name: "c3", scope: !51, file: !1, line: 41, type: !60)
-!60 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "C3", scope: !42, file: !1, line: 37, size: 8, flags: DIFlagTypePassByValue, elements: !61)
-!61 = !{!62}
-!62 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !60, file: !1, line: 37, baseType: !63, size: 8)
-!63 = !DIDerivedType(tag: DW_TAG_typedef, name: "Char3", scope: !60, file: !1, line: 37, baseType: !35)
-!64 = !DILocation(line: 41, column: 12, scope: !51)
-!65 = !DILocation(line: 41, column: 19, scope: !51)
-!66 = !DILocation(line: 41, column: 20, scope: !51)
-!67 = !DILocalVariable(name: "a1", scope: !68, file: !1, line: 8, type: !70)
-!68 = distinct !DILexicalBlock(scope: !69, file: !1, line: 5, column: 3)
-!69 = distinct !DISubprogram(name: "removed", linkageName: "_Z7removedv", scope: !1, file: !1, line: 2, type: !9, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !110)
-!70 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A1", scope: !69, file: !1, line: 3, size: 32, flags: DIFlagTypePassByValue, elements: !71, identifier: "_ZTSZ7removedvE2A1")
-!71 = !{!72}
-!72 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !70, file: !1, line: 3, baseType: !17, size: 32)
-!73 = !DILocation(line: 8, column: 8, scope: !68, inlinedAt: !74)
-!74 = distinct !DILocation(line: 45, column: 3, scope: !40)
-!75 = !DILocation(line: 8, column: 15, scope: !68, inlinedAt: !74)
-!76 = !DILocation(line: 8, column: 16, scope: !68, inlinedAt: !74)
-!77 = !DILocalVariable(name: "i1", scope: !78, file: !1, line: 10, type: !79)
-!78 = distinct !DILexicalBlock(scope: !68, file: !1, line: 9, column: 5)
-!79 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "I1", scope: !68, file: !1, line: 6, size: 32, flags: DIFlagTypePassByValue, elements: !80, identifier: "_ZTSZ7removedvE2I1")
-!80 = !{!81}
-!81 = !DIDerivedType(tag: DW_TAG_member, name: "j", scope: !79, file: !1, line: 6, baseType: !82, size: 32)
-!82 = !DIDerivedType(tag: DW_TAG_typedef, name: "Int1", scope: !69, file: !1, line: 4, baseType: !17)
-!83 = !DILocation(line: 10, column: 10, scope: !78, inlinedAt: !74)
-!84 = !DILocation(line: 10, column: 17, scope: !78, inlinedAt: !74)
-!85 = !DILocation(line: 10, column: 18, scope: !78, inlinedAt: !74)
-!86 = !DILocalVariable(name: "c1", scope: !78, file: !1, line: 11, type: !87)
-!87 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "C1", scope: !68, file: !1, line: 7, size: 8, flags: DIFlagTypePassByValue, elements: !88, identifier: "_ZTSZ7removedvE2C1")
-!88 = !{!89}
-!89 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !87, file: !1, line: 7, baseType: !90, size: 8)
-!90 = !DIDerivedType(tag: DW_TAG_typedef, name: "Char1", scope: !87, file: !1, line: 7, baseType: !35)
-!91 = !DILocation(line: 11, column: 10, scope: !78, inlinedAt: !74)
-!92 = !DILocation(line: 11, column: 17, scope: !78, inlinedAt: !74)
-!93 = !DILocation(line: 11, column: 18, scope: !78, inlinedAt: !74)
-!94 = !DILocation(line: 23, column: 8, scope: !13, inlinedAt: !95)
-!95 = distinct !DILocation(line: 46, column: 3, scope: !40)
-!96 = !DILocation(line: 23, column: 15, scope: !13, inlinedAt: !95)
-!97 = !DILocation(line: 23, column: 16, scope: !13, inlinedAt: !95)
-!98 = !DILocation(line: 25, column: 10, scope: !22, inlinedAt: !95)
-!99 = !DILocation(line: 25, column: 17, scope: !22, inlinedAt: !95)
-!100 = !DILocation(line: 25, column: 18, scope: !22, inlinedAt: !95)
-!101 = !DILocation(line: 26, column: 10, scope: !22, inlinedAt: !95)
-!102 = !DILocation(line: 26, column: 17, scope: !22, inlinedAt: !95)
-!103 = !DILocation(line: 26, column: 18, scope: !22, inlinedAt: !95)
-!104 = !DILocation(line: 47, column: 1, scope: !40)
-!105 = !{!14, !23, !26, !31}
-!107 = !{!44, !52, !55, !60}
-!110 = !{!70, !79, !82, !87}
diff --git a/llvm/test/DebugInfo/Generic/local-type-as-template-parameter.ll b/llvm/test/DebugInfo/Generic/local-type-as-template-parameter.ll
deleted file mode 100644
index 3fc9af7ccc0f7..0000000000000
--- a/llvm/test/DebugInfo/Generic/local-type-as-template-parameter.ll
+++ /dev/null
@@ -1,161 +0,0 @@
-; REQUIERES: system-linux
-; RUN: %llc_dwarf -O0 -filetype=obj < %s              \
-; RUN:  | llvm-dwarfdump --show-children --name=foo - \
-; RUN:  | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s
-
-; The test ensures that AsmPrinter doesn't crashed compiling this.
-; It also demostrates misplacement for a local type (see PR55680 for details).
-
-; The test compiled from:
-
-; template<typename T>
-; struct A {
-;   A(T &in) : a(in) {}
-;   T a;
-; };
-;
-; __attribute__((always_inline))
-; void foo() {
-;   struct B { int i; };
-;   B objB;
-;   A<B> objA(objB);
-; }
-;
-; int main() {
-;   foo();
-; }
-
-; Concrete out-of-line tree of foo().
-; CHECK: DW_TAG_subprogram
-; CHECK:   DW_AT_abstract_origin {{.*}} "_Z3foov"
-
-; FIXME: 'struct B' should be in the abstract tree below, not here.
-; CHECK:   DW_TAG_structure_type
-; CHECK:     DW_AT_name	("B")
-; CHECK:     DW_TAG_member
-; CHECK:     NULL
-;
-; CHECK:   DW_TAG_variable
-; CHECK:     DW_AT_abstract_origin {{.*}} "objB"
-; CHECK:   DW_TAG_variable
-; CHECK:     DW_AT_abstract_origin {{.*}} "objA"
-
-; CHECK:   NULL
-
-; Abstract tree of foo().
-; CHECK: DW_TAG_subprogram
-; CHECK:   DW_AT_name	("foo")
-; CHECK:   DW_AT_inline	(DW_INL_inlined)
-
-; CHECK:   DW_TAG_variable
-; CHECK:     DW_AT_name	("objB")
-; CHECK:   DW_TAG_variable
-; CHECK:     DW_AT_name	("objA")
-
-; CHECK:   NULL
-
-; CHECK: DW_TAG_inlined_subroutine
-; CHECK:   DW_AT_abstract_origin {{.*}} "_Z3foov"
-; CHECK:   DW_TAG_variable
-; CHECK:     DW_AT_abstract_origin {{.*}} "objB"
-; CHECK:   DW_TAG_variable
-; CHECK:     DW_AT_abstract_origin {{.*}} "objA"
-; CHECK:   NULL
-
-%struct.B = type { i32 }
-%struct.A = type { %struct.B }
-
-define dso_local void @_Z3foov() !dbg !7 {
-entry:
-  %objB = alloca %struct.B, align 4
-  %objA = alloca %struct.A, align 4
-  call void @llvm.dbg.declare(metadata ptr %objB, metadata !30, metadata !DIExpression()), !dbg !31
-  call void @llvm.dbg.declare(metadata ptr %objA, metadata !32, metadata !DIExpression()), !dbg !33
-  call void @_ZN1AIZ3foovE1BEC2ERS0_(ptr noundef nonnull align 4 dereferenceable(4) %objA, ptr noundef nonnull align 4 dereferenceable(4) %objB), !dbg !33
-  ret void, !dbg !34
-}
-
-declare void @llvm.dbg.declare(metadata, metadata, metadata)
-
-define internal void @_ZN1AIZ3foovE1BEC2ERS0_(ptr noundef nonnull align 4 dereferenceable(4) %this, ptr noundef nonnull align 4 dereferenceable(4) %in) unnamed_addr align 2 !dbg !35 {
-entry:
-  %this.addr = alloca ptr, align 8
-  %in.addr = alloca ptr, align 8
-  store ptr %this, ptr %this.addr, align 8
-  call void @llvm.dbg.declare(metadata ptr %this.addr, metadata !36, metadata !DIExpression()), !dbg !38
-  store ptr %in, ptr %in.addr, align 8
-  call void @llvm.dbg.declare(metadata ptr %in.addr, metadata !39, metadata !DIExpression()), !dbg !40
-  %this1 = load ptr, ptr %this.addr, align 8
-  %a = getelementptr inbounds %struct.A, ptr %this1, i32 0, i32 0, !dbg !41
-  %0 = load ptr, ptr %in.addr, align 8, !dbg !42
-  call void @llvm.memcpy.p0.p0.i64(ptr align 4 %a, ptr align 4 %0, i64 4, i1 false), !dbg !41
-  ret void, !dbg !43
-}
-
-define dso_local noundef i32 @main() !dbg !44 {
-entry:
-  %objB.i = alloca %struct.B, align 4
-  %objA.i = alloca %struct.A, align 4
-  call void @llvm.dbg.declare(metadata ptr %objB.i, metadata !30, metadata !DIExpression()), !dbg !47
-  call void @llvm.dbg.declare(metadata ptr %objA.i, metadata !32, metadata !DIExpression()), !dbg !49
-  call void @_ZN1AIZ3foovE1BEC2ERS0_(ptr noundef nonnull align 4 dereferenceable(4) %objA.i, ptr noundef nonnull align 4 dereferenceable(4) %objB.i), !dbg !49
-  ret i32 0, !dbg !50
-}
-
-declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg)
-
-!llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!22, !23, !24, !25, !26, !27, !28}
-!llvm.ident = !{!29}
-
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 15.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, retainedTypes: !2, splitDebugInlining: false, nameTableKind: None)
-!1 = !DIFile(filename: "test.cpp", directory: "/", checksumkind: CSK_MD5, checksum: "aec7fd397e86f8655ef7f4bb4233b849")
-!2 = !{!3}
-!3 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A<B>", file: !1, line: 2, size: 32, flags: DIFlagTypePassByValue | DIFlagNonTrivial, elements: !4, templateParams: !20)
-!4 = !{!5, !15}
-!5 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !3, file: !1, line: 4, baseType: !6, size: 32)
-!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "B", scope: !7, file: !1, line: 9, size: 32, flags: DIFlagTypePassByValue, elements: !12)
-!7 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !1, file: !1, line: 8, type: !8, scopeLine: 8, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !11)
-!8 = !DISubroutineType(types: !9)
-!9 = !{null}
-!10 = !{}
-!11 = !{!6}
-!12 = !{!13}
-!13 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !6, file: !1, line: 9, baseType: !14, size: 32)
-!14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-!15 = !DISubprogram(name: "A", scope: !3, file: !1, line: 3, type: !16, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagLocalToUnit)
-!16 = !DISubroutineType(types: !17)
-!17 = !{null, !18, !19}
-!18 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !3, size: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
-!19 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !6, size: 64)
-!20 = !{!21}
-!21 = !DITemplateTypeParameter(name: "T", type: !6)
-!22 = !{i32 7, !"Dwarf Version", i32 5}
-!23 = !{i32 2, !"Debug Info Version", i32 3}
-!24 = !{i32 1, !"wchar_size", i32 4}
-!25 = !{i32 7, !"PIC Level", i32 2}
-!26 = !{i32 7, !"PIE Level", i32 2}
-!27 = !{i32 7, !"uwtable", i32 2}
-!28 = !{i32 7, !"frame-pointer", i32 2}
-!29 = !{!"clang version 15.0.0"}
-!30 = !DILocalVariable(name: "objB", scope: !7, file: !1, line: 10, type: !6)
-!31 = !DILocation(line: 10, column: 5, scope: !7)
-!32 = !DILocalVariable(name: "objA", scope: !7, file: !1, line: 11, type: !3)
-!33 = !DILocation(line: 11, column: 8, scope: !7)
-!34 = !DILocation(line: 12, column: 1, scope: !7)
-!35 = distinct !DISubprogram(name: "A", linkageName: "_ZN1AIZ3foovE1BEC2ERS0_", scope: !3, file: !1, line: 3, type: !16, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !0, declaration: !15, retainedNodes: !10)
-!36 = !DILocalVariable(name: "this", arg: 1, scope: !35, type: !37, flags: DIFlagArtificial | DIFlagObjectPointer)
-!37 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !3, size: 64)
-!38 = !DILocation(line: 0, scope: !35)
-!39 = !DILocalVariable(name: "in", arg: 2, scope: !35, file: !1, line: 3, type: !19)
-!40 = !DILocation(line: 3, column: 8, scope: !35)
-!41 = !DILocation(line: 3, column: 14, scope: !35)
-!42 = !DILocation(line: 3, column: 16, scope: !35)
-!43 = !DILocation(line: 3, column: 21, scope: !35)
-!44 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 14, type: !45, scopeLine: 14, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !10)
-!45 = !DISubroutineType(types: !46)
-!46 = !{!14}
-!47 = !DILocation(line: 10, column: 5, scope: !7, inlinedAt: !48)
-!48 = distinct !DILocation(line: 15, column: 3, scope: !44)
-!49 = !DILocation(line: 11, column: 8, scope: !7, inlinedAt: !48)
-!50 = !DILocation(line: 16, column: 1, scope: !44)
diff --git a/llvm/test/DebugInfo/X86/split-dwarf-local-import.ll b/llvm/test/DebugInfo/Generic/split-dwarf-local-import.ll
similarity index 98%
rename from llvm/test/DebugInfo/X86/split-dwarf-local-import.ll
rename to llvm/test/DebugInfo/Generic/split-dwarf-local-import.ll
index 676ef23b679e8..1a41f3f348ca0 100644
--- a/llvm/test/DebugInfo/X86/split-dwarf-local-import.ll
+++ b/llvm/test/DebugInfo/Generic/split-dwarf-local-import.ll
@@ -1,3 +1,4 @@
+; REQUIRES: x86-registered-target
 ; RUN: %llc_dwarf -O1 -filetype=obj -split-dwarf-file=%t.dwo < %s | llvm-dwarfdump -debug-info - | FileCheck %s --implicit-check-not "{{DW_TAG|NULL}}"
 
 ; CHECK-LABEL: debug_info contents
diff --git a/llvm/test/DebugInfo/X86/split-dwarf-local-import2.ll b/llvm/test/DebugInfo/Generic/split-dwarf-local-import2.ll
similarity index 98%
rename from llvm/test/DebugInfo/X86/split-dwarf-local-import2.ll
rename to llvm/test/DebugInfo/Generic/split-dwarf-local-import2.ll
index 71f80992d1b04..0057f675f9b31 100644
--- a/llvm/test/DebugInfo/X86/split-dwarf-local-import2.ll
+++ b/llvm/test/DebugInfo/Generic/split-dwarf-local-import2.ll
@@ -1,3 +1,4 @@
+; REQUIRES: x86-registered-target
 ; RUN: %llc_dwarf -split-dwarf-file=%t.dwo < %s | FileCheck %s
 
 ; Ensure function-local DW_TAG_imported_declaration get skipped if its parent subprogram was not emitted.
diff --git a/llvm/test/DebugInfo/X86/split-dwarf-local-import3.ll b/llvm/test/DebugInfo/Generic/split-dwarf-local-import3.ll
similarity index 100%
rename from llvm/test/DebugInfo/X86/split-dwarf-local-import3.ll
rename to llvm/test/DebugInfo/Generic/split-dwarf-local-import3.ll
diff --git a/llvm/test/DebugInfo/Generic/verifier-invalid-disubprogram.ll b/llvm/test/DebugInfo/Generic/verifier-invalid-disubprogram.ll
index 54ce1c56c6b30..6d4d0e93d38f9 100644
--- a/llvm/test/DebugInfo/Generic/verifier-invalid-disubprogram.ll
+++ b/llvm/test/DebugInfo/Generic/verifier-invalid-disubprogram.ll
@@ -38,7 +38,7 @@ define void @invalid_subprogram_declaration() !dbg !9 { ret void }
 define void @invalid_retained_nodes_list() !dbg !10 { ret void }
 !10 = distinct !DISubprogram(retainedNodes: !0)
 
-; CHECK: invalid retained nodes, expected DILocalVariable, DILabel, DIImportedEntity or DIType
+; CHECK: invalid retained nodes, expected DILocalVariable, DILabel or DIImportedEntity
 define void @invalid_retained_nodes_expected() !dbg !11 { ret void }
 !11 = distinct !DISubprogram(retainedNodes: !{!0})
 
diff --git a/llvm/test/DebugInfo/X86/set.ll b/llvm/test/DebugInfo/X86/set.ll
index 7dbec4520ed6c..292c7c6e4a577 100644
--- a/llvm/test/DebugInfo/X86/set.ll
+++ b/llvm/test/DebugInfo/X86/set.ll
@@ -68,11 +68,11 @@ attributes #1 = { nofree nosync nounwind readnone speculatable willreturn }
 !llvm.module.flags = !{!18, !19, !20}
 
 !0 = !{!"versions- cm3: d5.10.0 llvm: 9.0"}
-!1 = distinct !DICompileUnit(language: DW_LANG_Modula3, file: !2, producer: "cm3", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = distinct !DICompileUnit(language: DW_LANG_Modula3, file: !2, producer: "cm3", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !3)
 !2 = !DIFile(filename: "Main.m3", directory: "/home/cm3/settest/src")
 !3 = !{!4}
 !4 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum", scope: !5, file: !2, line: 11, size: 8, align: 8, elements: !9)
-!5 = distinct !DISubprogram(name: "Test", linkageName: "Main__Test", scope: !2, file: !2, line: 11, type: !6, scopeLine: 11, spFlags: DISPFlagDefinition, unit: !1, retainedNodes: !3)
+!5 = distinct !DISubprogram(name: "Test", linkageName: "Main__Test", scope: !2, file: !2, line: 11, type: !6, scopeLine: 11, spFlags: DISPFlagDefinition, unit: !1, retainedNodes: !8)
 !6 = !DISubroutineType(types: !7)
 !7 = !{null}
 !8 = !{}

From cb9ac7051589ea0d05507f9370d0716bef86b4ae Mon Sep 17 00:00:00 2001
From: Vladislav Dzhidzhoev <vdzhidzhoev@accesssoftek.com>
Date: Mon, 19 Jun 2023 16:42:05 +0200
Subject: [PATCH 078/130] Reland "[DebugMetadata][DwarfDebug] Support
 function-local types in lexical block scopes (4/7)" (2)

Test "local-type-as-template-parameter.ll" is now enabled only for
x86_64.

Authored-by: Kristina Bessonova <kbessonova@accesssoftek.com>

Differential Revision: https://reviews.llvm.org/D144006

Depends on D144005
---
 .../CodeGen/debug-info-codeview-unnamed.c     |  16 +-
 clang/test/CodeGen/debug-info-unused-types.c  |  14 +-
 .../test/CodeGen/debug-info-unused-types.cpp  |  12 +-
 clang/test/CodeGenCXX/debug-info-access.cpp   |   2 +-
 .../CodeGenCXX/debug-info-anon-union-vars.cpp |  12 +-
 .../debug-info-codeview-unnamed.cpp           | 110 +++--
 .../debug-info-gline-tables-only-codeview.cpp |   4 +-
 clang/test/CodeGenCXX/debug-lambda-this.cpp   |   4 +-
 llvm/include/llvm/IR/DIBuilder.h              |   6 +-
 llvm/lib/Bitcode/Reader/MetadataLoader.cpp    |  87 ++--
 .../CodeGen/AsmPrinter/DwarfCompileUnit.cpp   |  60 ++-
 .../lib/CodeGen/AsmPrinter/DwarfCompileUnit.h |  16 +-
 llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp    |  13 +-
 llvm/lib/IR/DIBuilder.cpp                     |  34 +-
 llvm/lib/IR/Verifier.cpp                      |   6 +-
 llvm/test/Bitcode/upgrade-cu-locals.ll        |  68 +--
 llvm/test/Bitcode/upgrade-cu-locals.ll.bc     | Bin 2688 -> 2780 bytes
 .../DebugInfo/Generic/inlined-local-type.ll   | 128 ++++++
 .../Generic/lexical-block-retained-types.ll   |  55 +++
 .../DebugInfo/Generic/lexical-block-types.ll  | 425 ++++++++++++++++++
 .../Generic/verifier-invalid-disubprogram.ll  |   2 +-
 .../X86/local-type-as-template-parameter.ll   | 161 +++++++
 llvm/test/DebugInfo/X86/set.ll                |   4 +-
 .../split-dwarf-local-import.ll               |   1 -
 .../split-dwarf-local-import2.ll              |   1 -
 .../split-dwarf-local-import3.ll              |   0
 26 files changed, 1052 insertions(+), 189 deletions(-)
 create mode 100644 llvm/test/DebugInfo/Generic/inlined-local-type.ll
 create mode 100644 llvm/test/DebugInfo/Generic/lexical-block-retained-types.ll
 create mode 100644 llvm/test/DebugInfo/Generic/lexical-block-types.ll
 create mode 100644 llvm/test/DebugInfo/X86/local-type-as-template-parameter.ll
 rename llvm/test/DebugInfo/{Generic => X86}/split-dwarf-local-import.ll (98%)
 rename llvm/test/DebugInfo/{Generic => X86}/split-dwarf-local-import2.ll (98%)
 rename llvm/test/DebugInfo/{Generic => X86}/split-dwarf-local-import3.ll (100%)

diff --git a/clang/test/CodeGen/debug-info-codeview-unnamed.c b/clang/test/CodeGen/debug-info-codeview-unnamed.c
index bd2a7543e56b2..16ffb3682236f 100644
--- a/clang/test/CodeGen/debug-info-codeview-unnamed.c
+++ b/clang/test/CodeGen/debug-info-codeview-unnamed.c
@@ -8,23 +8,23 @@ int main(int argc, char* argv[], char* arge[]) {
   //
   struct { int bar; } one = {42};
   //
-  // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "one"
-  // LINUX-SAME:     type: [[TYPE_OF_ONE:![0-9]+]]
-  // LINUX-SAME:     )
-  // LINUX:      [[TYPE_OF_ONE]] = distinct !DICompositeType(
+  // LINUX:      [[TYPE_OF_ONE:![0-9]+]] = distinct !DICompositeType(
   // LINUX-SAME:     tag: DW_TAG_structure_type
   // LINUX-NOT:      name:
   // LINUX-NOT:      identifier:
   // LINUX-SAME:     )
+  // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "one"
+  // LINUX-SAME:     type: [[TYPE_OF_ONE]]
+  // LINUX-SAME:     )
   //
-  // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "one"
-  // MSVC-SAME:      type: [[TYPE_OF_ONE:![0-9]+]]
-  // MSVC-SAME:      )
-  // MSVC:       [[TYPE_OF_ONE]] = distinct !DICompositeType
+  // MSVC:       [[TYPE_OF_ONE:![0-9]+]] = distinct !DICompositeType
   // MSVC-SAME:      tag: DW_TAG_structure_type
   // MSVC-NOT:       name:
   // MSVC-NOT:       identifier:
   // MSVC-SAME:      )
+  // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "one"
+  // MSVC-SAME:      type: [[TYPE_OF_ONE]]
+  // MSVC-SAME:      )
 
   return 0;
 }
diff --git a/clang/test/CodeGen/debug-info-unused-types.c b/clang/test/CodeGen/debug-info-unused-types.c
index 3e9f7b07658e3..5bcc71f710a5c 100644
--- a/clang/test/CodeGen/debug-info-unused-types.c
+++ b/clang/test/CodeGen/debug-info-unused-types.c
@@ -18,13 +18,13 @@ void quux(void) {
 // CHECK: !DICompileUnit{{.+}}retainedTypes: [[RETTYPES:![0-9]+]]
 // CHECK: [[TYPE0:![0-9]+]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "bar"
 // CHECK: [[TYPE1:![0-9]+]] = !DIEnumerator(name: "BAR"
-// CHECK: [[TYPE2:![0-9]+]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "z"
-// CHECK: [[TYPE3:![0-9]+]] = !DIEnumerator(name: "Z"
-// CHECK: [[RETTYPES]] = !{[[TYPE4:![0-9]+]], [[TYPE5:![0-9]+]], [[TYPE0]], [[TYPE6:![0-9]+]], {{![0-9]+}}, [[TYPE7:![0-9]+]], [[TYPE2]], [[TYPE8:![0-9]+]]}
-// CHECK: [[TYPE4]] = !DIDerivedType(tag: DW_TAG_typedef, name: "my_int"
-// CHECK: [[TYPE5]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "foo"
-// CHECK: [[TYPE6]] = distinct !DICompositeType(tag: DW_TAG_union_type, name: "baz"
-// CHECK: [[TYPE7]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "y"
+// CHECK: [[RETTYPES]] = !{[[TYPE2:![0-9]+]], [[TYPE3:![0-9]+]], [[TYPE0]], [[TYPE4:![0-9]+]], {{![0-9]+}}, [[TYPE5:![0-9]+]], [[TYPE6:![0-9]+]], [[TYPE8:![0-9]+]]}
+// CHECK: [[TYPE2]] = !DIDerivedType(tag: DW_TAG_typedef, name: "my_int"
+// CHECK: [[TYPE3]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "foo"
+// CHECK: [[TYPE4]] = distinct !DICompositeType(tag: DW_TAG_union_type, name: "baz"
+// CHECK: [[TYPE5]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "y"
+// CHECK: [[TYPE6]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "z"
+// CHECK: [[TYPE7:![0-9]+]] = !DIEnumerator(name: "Z"
 // CHECK: [[TYPE8]] = distinct !DICompositeType(tag: DW_TAG_union_type, name: "w"
 
 // Check that debug info is not emitted for the typedef, struct, enum, and
diff --git a/clang/test/CodeGen/debug-info-unused-types.cpp b/clang/test/CodeGen/debug-info-unused-types.cpp
index 023cac159faa4..715fada47eab8 100644
--- a/clang/test/CodeGen/debug-info-unused-types.cpp
+++ b/clang/test/CodeGen/debug-info-unused-types.cpp
@@ -13,12 +13,12 @@ void quux() {
 // CHECK: !DICompileUnit{{.+}}retainedTypes: [[RETTYPES:![0-9]+]]
 // CHECK: [[TYPE0:![0-9]+]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "baz"
 // CHECK: [[TYPE1:![0-9]+]] = !DIEnumerator(name: "BAZ"
-// CHECK: [[TYPE2:![0-9]+]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "z"
-// CHECK: [[TYPE3:![0-9]+]] = !DIEnumerator(name: "Z"
-// CHECK: [[RETTYPES]] = !{[[TYPE4:![0-9]+]], [[TYPE5:![0-9]+]], [[TYPE0]], {{![0-9]+}}, [[TYPE6:![0-9]+]], [[TYPE2]]}
-// CHECK: [[TYPE4]] = !DIDerivedType(tag: DW_TAG_typedef, name: "foo"
-// CHECK: [[TYPE5]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "bar"
-// CHECK: [[TYPE6]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "y"
+// CHECK: [[RETTYPES]] = !{[[TYPE2:![0-9]+]], [[TYPE3:![0-9]+]], [[TYPE0]], {{![0-9]+}}, [[TYPE4:![0-9]+]], [[TYPE5:![0-9]+]]}
+// CHECK: [[TYPE2]] = !DIDerivedType(tag: DW_TAG_typedef, name: "foo"
+// CHECK: [[TYPE3]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "bar"
+// CHECK: [[TYPE4]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "y"
+// CHECK: [[TYPE5]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "z"
+// CHECK: [[TYPE6:![0-9]+]] = !DIEnumerator(name: "Z"
 
 // NODBG-NOT: !DI{{CompositeType|Enumerator|DerivedType}}
 
diff --git a/clang/test/CodeGenCXX/debug-info-access.cpp b/clang/test/CodeGenCXX/debug-info-access.cpp
index 9f2c044843d0f..7c0bf8ccb0384 100644
--- a/clang/test/CodeGenCXX/debug-info-access.cpp
+++ b/clang/test/CodeGenCXX/debug-info-access.cpp
@@ -18,9 +18,9 @@ class B : public A {
   static int public_static;
 
 protected:
+  // CHECK-DAG: !DIDerivedType(tag: DW_TAG_typedef, name: "prot_using",{{.*}} line: [[@LINE+3]],{{.*}} flags: DIFlagProtected)
   // CHECK-DAG: !DIDerivedType(tag: DW_TAG_typedef, name: "prot_typedef",{{.*}} line: [[@LINE+1]],{{.*}} flags: DIFlagProtected)
   typedef int prot_typedef;
-  // CHECK-DAG: !DIDerivedType(tag: DW_TAG_typedef, name: "prot_using",{{.*}} line: [[@LINE+1]],{{.*}} flags: DIFlagProtected)
   using prot_using = prot_typedef;
   prot_using prot_member;
 
diff --git a/clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp b/clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp
index 61b3c7c0526c8..c91cf83c0405f 100644
--- a/clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp
+++ b/clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp
@@ -51,13 +51,13 @@ void instantiate(int x) {
 // CHECK: !DIGlobalVariable(name: "b",{{.*}} file: [[FILE]], line: 6,{{.*}} isLocal: true, isDefinition: true
 // CHECK: !DIGlobalVariable(name: "result", {{.*}} isLocal: false, isDefinition: true
 // CHECK: !DIGlobalVariable(name: "value", {{.*}} isLocal: false, isDefinition: true
-// CHECK: !DILocalVariable(name: "i", {{.*}}, flags: DIFlagArtificial
-// CHECK: !DILocalVariable(name: "c", {{.*}}, flags: DIFlagArtificial
-// CHECK: !DILocalVariable(
-// CHECK-NOT: name:
-// CHECK: type: ![[UNION:[0-9]+]]
-// CHECK: ![[UNION]] = distinct !DICompositeType(tag: DW_TAG_union_type,
+// CHECK: ![[UNION:[0-9]+]] = distinct !DICompositeType(tag: DW_TAG_union_type,
 // CHECK-NOT: name:
 // CHECK: elements
 // CHECK: !DIDerivedType(tag: DW_TAG_member, name: "i", scope: ![[UNION]],
 // CHECK: !DIDerivedType(tag: DW_TAG_member, name: "c", scope: ![[UNION]],
+// CHECK: !DILocalVariable(name: "i", {{.*}}, flags: DIFlagArtificial
+// CHECK: !DILocalVariable(name: "c", {{.*}}, flags: DIFlagArtificial
+// CHECK: !DILocalVariable(
+// CHECK-NOT: name:
+// CHECK: type: ![[UNION]]
diff --git a/clang/test/CodeGenCXX/debug-info-codeview-unnamed.cpp b/clang/test/CodeGenCXX/debug-info-codeview-unnamed.cpp
index b4c79936ab33e..9602ac1b02497 100644
--- a/clang/test/CodeGenCXX/debug-info-codeview-unnamed.cpp
+++ b/clang/test/CodeGenCXX/debug-info-codeview-unnamed.cpp
@@ -3,6 +3,60 @@
 
 int main(int argc, char* argv[], char* arge[]) {
   //
+  // LINUX:      [[TYPE_OF_ONE:![0-9]+]] = distinct !DICompositeType(
+  // LINUX-SAME:     tag: DW_TAG_structure_type
+  // LINUX-NOT:      name:
+  // LINUX-NOT:      identifier:
+  // LINUX-SAME:     )
+  //
+  // MSVC:       [[TYPE_OF_ONE:![0-9]+]] = distinct !DICompositeType
+  // MSVC-SAME:      tag: DW_TAG_structure_type
+  // MSVC-SAME:      name: "<unnamed-type-one>"
+  // MSVC-SAME:      identifier: ".?AU<unnamed-type-one>@?1??main@@9@"
+  // MSVC-SAME:      )
+
+
+  //
+  // LINUX:      [[TYPE_OF_TWO:![0-9]+]] = distinct !DICompositeType(
+  // LINUX-SAME:     tag: DW_TAG_structure_type
+  // LINUX-NOT:      name:
+  // LINUX-NOT:      identifier:
+  // LINUX-SAME:     )
+  //
+  // MSVC:       [[TYPE_OF_TWO:![0-9]+]] = distinct !DICompositeType
+  // MSVC-SAME:      tag: DW_TAG_structure_type
+  // MSVC-SAME:      name: "<unnamed-type-two>"
+  // MSVC-SAME:      identifier: ".?AU<unnamed-type-two>@?2??main@@9@"
+  // MSVC-SAME:      )
+
+
+  //
+  // LINUX:      [[TYPE_OF_THREE:![0-9]+]] = distinct !DICompositeType(
+  // LINUX-SAME:     tag: DW_TAG_structure_type
+  // LINUX-SAME:     name: "named"
+  // LINUX-NOT:      identifier:
+  // LINUX-SAME:     )
+  //
+  // MSVC:       [[TYPE_OF_THREE:![0-9]+]] = distinct !DICompositeType
+  // MSVC-SAME:      tag: DW_TAG_structure_type
+  // MSVC-SAME:      name: "named"
+  // MSVC-SAME:      identifier: ".?AUnamed@?1??main@@9@"
+  // MSVC-SAME:      )
+
+  //
+  // LINUX:      [[TYPE_OF_FOUR:![0-9]+]] = distinct !DICompositeType(
+  // LINUX-SAME:     tag: DW_TAG_class_type
+  // LINUX-NOT:      name:
+  // LINUX-NOT:      identifier:
+  // LINUX-SAME:     )
+  //
+  // MSVC:       [[TYPE_OF_FOUR:![0-9]+]] = distinct !DICompositeType
+  // MSVC-SAME:      tag: DW_TAG_class_type
+  // MSVC-SAME:      name: "<lambda_0>"
+  // MSVC-SAME:      identifier: ".?AV<lambda_0>@?0??main@@9@"
+  // MSVC-SAME:      )
+
+
   // In CodeView, the LF_MFUNCTION entry for "bar()" refers to the forward
   // reference of the unnamed struct. Visual Studio requires a unique
   // identifier to match the LF_STRUCTURE forward reference to the definition.
@@ -10,21 +64,11 @@ int main(int argc, char* argv[], char* arge[]) {
   struct { void bar() {} } one;
   //
   // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "one"
-  // LINUX-SAME:     type: [[TYPE_OF_ONE:![0-9]+]]
-  // LINUX-SAME:     )
-  // LINUX:      [[TYPE_OF_ONE]] = distinct !DICompositeType(
-  // LINUX-SAME:     tag: DW_TAG_structure_type
-  // LINUX-NOT:      name:
-  // LINUX-NOT:      identifier:
+  // LINUX-SAME:     type: [[TYPE_OF_ONE]]
   // LINUX-SAME:     )
   //
   // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "one"
-  // MSVC-SAME:      type: [[TYPE_OF_ONE:![0-9]+]]
-  // MSVC-SAME:      )
-  // MSVC:       [[TYPE_OF_ONE]] = distinct !DICompositeType
-  // MSVC-SAME:      tag: DW_TAG_structure_type
-  // MSVC-SAME:      name: "<unnamed-type-one>"
-  // MSVC-SAME:      identifier: ".?AU<unnamed-type-one>@?1??main@@9@"
+  // MSVC-SAME:      type: [[TYPE_OF_ONE]]
   // MSVC-SAME:      )
 
 
@@ -36,21 +80,11 @@ int main(int argc, char* argv[], char* arge[]) {
   int decltype(two)::*ptr2unnamed = &decltype(two)::bar;
   //
   // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "two"
-  // LINUX-SAME:     type: [[TYPE_OF_TWO:![0-9]+]]
-  // LINUX-SAME:     )
-  // LINUX:      [[TYPE_OF_TWO]] = distinct !DICompositeType(
-  // LINUX-SAME:     tag: DW_TAG_structure_type
-  // LINUX-NOT:      name:
-  // LINUX-NOT:      identifier:
+  // LINUX-SAME:     type: [[TYPE_OF_TWO]]
   // LINUX-SAME:     )
   //
   // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "two"
-  // MSVC-SAME:      type: [[TYPE_OF_TWO:![0-9]+]]
-  // MSVC-SAME:      )
-  // MSVC:       [[TYPE_OF_TWO]] = distinct !DICompositeType
-  // MSVC-SAME:      tag: DW_TAG_structure_type
-  // MSVC-SAME:      name: "<unnamed-type-two>"
-  // MSVC-SAME:      identifier: ".?AU<unnamed-type-two>@?2??main@@9@"
+  // MSVC-SAME:      type: [[TYPE_OF_TWO]]
   // MSVC-SAME:      )
 
 
@@ -61,21 +95,11 @@ int main(int argc, char* argv[], char* arge[]) {
   struct named { int bar; int named::* p2mem; } three = { 42, &named::bar };
   //
   // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "three"
-  // LINUX-SAME:     type: [[TYPE_OF_THREE:![0-9]+]]
-  // LINUX-SAME:     )
-  // LINUX:      [[TYPE_OF_THREE]] = distinct !DICompositeType(
-  // LINUX-SAME:     tag: DW_TAG_structure_type
-  // LINUX-SAME:     name: "named"
-  // LINUX-NOT:      identifier:
+  // LINUX-SAME:     type: [[TYPE_OF_THREE]]
   // LINUX-SAME:     )
   //
   // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "three"
-  // MSVC-SAME:      type: [[TYPE_OF_THREE:![0-9]+]]
-  // MSVC-SAME:      )
-  // MSVC:       [[TYPE_OF_THREE]] = distinct !DICompositeType
-  // MSVC-SAME:      tag: DW_TAG_structure_type
-  // MSVC-SAME:      name: "named"
-  // MSVC-SAME:      identifier: ".?AUnamed@?1??main@@9@"
+  // MSVC-SAME:      type: [[TYPE_OF_THREE]]
   // MSVC-SAME:      )
 
 
@@ -87,21 +111,11 @@ int main(int argc, char* argv[], char* arge[]) {
   auto four = [argc](int i) -> int { return argc == i ? 1 : 0; };
   //
   // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "four"
-  // LINUX-SAME:     type: [[TYPE_OF_FOUR:![0-9]+]]
-  // LINUX-SAME:     )
-  // LINUX:      [[TYPE_OF_FOUR]] = distinct !DICompositeType(
-  // LINUX-SAME:     tag: DW_TAG_class_type
-  // LINUX-NOT:      name:
-  // LINUX-NOT:      identifier:
+  // LINUX-SAME:     type: [[TYPE_OF_FOUR]]
   // LINUX-SAME:     )
   //
   // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "four"
-  // MSVC-SAME:      type: [[TYPE_OF_FOUR:![0-9]+]]
-  // MSVC-SAME:      )
-  // MSVC:       [[TYPE_OF_FOUR]] = distinct !DICompositeType
-  // MSVC-SAME:      tag: DW_TAG_class_type
-  // MSVC-SAME:      name: "<lambda_0>"
-  // MSVC-SAME:      identifier: ".?AV<lambda_0>@?0??main@@9@"
+  // MSVC-SAME:      type: [[TYPE_OF_FOUR]]
   // MSVC-SAME:      )
 
   return 0;
diff --git a/clang/test/CodeGenCXX/debug-info-gline-tables-only-codeview.cpp b/clang/test/CodeGenCXX/debug-info-gline-tables-only-codeview.cpp
index 6b9c9a243decd..122e4aa62ea7d 100644
--- a/clang/test/CodeGenCXX/debug-info-gline-tables-only-codeview.cpp
+++ b/clang/test/CodeGenCXX/debug-info-gline-tables-only-codeview.cpp
@@ -51,9 +51,9 @@ void test() {
   // CHECK-SAME:                              name: "<lambda_2_1>",
   c.lambda_params();
 
-  // CHECK: !DISubprogram(name: "operator()", scope: ![[LAMBDA1:[0-9]+]],
-  // CHECK: ![[LAMBDA1]] = !DICompositeType(tag: DW_TAG_class_type,
+  // CHECK: ![[LAMBDA1:[0-9]+]] = !DICompositeType(tag: DW_TAG_class_type,
   // CHECK-SAME:                            name: "<lambda_1>",
   // CHECK-SAME:                            flags: DIFlagFwdDecl
+  // CHECK: !DISubprogram(name: "operator()", scope: ![[LAMBDA1]],
   c.lambda2();
 }
diff --git a/clang/test/CodeGenCXX/debug-lambda-this.cpp b/clang/test/CodeGenCXX/debug-lambda-this.cpp
index eecbac6520ac9..3d659e7bfd004 100644
--- a/clang/test/CodeGenCXX/debug-lambda-this.cpp
+++ b/clang/test/CodeGenCXX/debug-lambda-this.cpp
@@ -13,10 +13,10 @@ int D::d(int x) {
 }
 
 // CHECK: ![[D:[0-9]+]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "D",
-// CHECK: ![[POINTER:.*]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: ![[D]], size: 64)
 // CHECK: !DIDerivedType(tag: DW_TAG_member, name: "this",
 // CHECK-SAME:           line: 11
-// CHECK-SAME:           baseType: ![[POINTER]]
+// CHECK-SAME:           baseType: ![[POINTER:[0-9]+]]
 // CHECK-SAME:           size: 64
 // CHECK-NOT:            offset: 0
 // CHECK-SAME:           ){{$}}
+// CHECK: ![[POINTER]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: ![[D]], size: 64)
diff --git a/llvm/include/llvm/IR/DIBuilder.h b/llvm/include/llvm/IR/DIBuilder.h
index ecd6dd7b0a4f8..0aa180aec6d8a 100644
--- a/llvm/include/llvm/IR/DIBuilder.h
+++ b/llvm/include/llvm/IR/DIBuilder.h
@@ -49,7 +49,7 @@ namespace llvm {
     Function *LabelFn;       ///< llvm.dbg.label
     Function *AssignFn;      ///< llvm.dbg.assign
 
-    SmallVector<TrackingMDNodeRef, 4> AllEnumTypes;
+    SmallVector<TrackingMDNodeRef, 4> EnumTypes;
     /// Track the RetainTypes, since they can be updated later on.
     SmallVector<TrackingMDNodeRef, 4> AllRetainTypes;
     SmallVector<DISubprogram *, 4> AllSubprograms;
@@ -64,8 +64,8 @@ namespace llvm {
     SmallVector<TrackingMDNodeRef, 4> UnresolvedNodes;
     bool AllowUnresolvedNodes;
 
-    /// Each subprogram's preserved local variables, labels and imported
-    /// entities.
+    /// Each subprogram's preserved local variables, labels, imported entities,
+    /// and types.
     ///
     /// Do not use a std::vector.  Some versions of libc++ apparently copy
     /// instead of move on grow operations, and TrackingMDRef is expensive to
diff --git a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
index 0a9a80688a419..321eae6fcb3b5 100644
--- a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
+++ b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
@@ -548,6 +548,8 @@ class MetadataLoader::MetadataLoaderImpl {
 
   /// Move local imports from DICompileUnit's 'imports' field to
   /// DISubprogram's retainedNodes.
+  /// Move fucntion-local enums from DICompileUnit's enums
+  /// to DISubprogram's retainedNodes.
   void upgradeCULocals() {
     if (NamedMDNode *CUNodes = TheModule.getNamedMetadata("llvm.dbg.cu")) {
       for (unsigned I = 0, E = CUNodes->getNumOperands(); I != E; ++I) {
@@ -555,48 +557,71 @@ class MetadataLoader::MetadataLoaderImpl {
         if (!CU)
           continue;
 
-        if (auto *RawImported = CU->getRawImportedEntities()) {
-          // Collect a set of imported entities to be moved.
-          SetVector<Metadata *> EntitiesToRemove;
+        SetVector<Metadata *> MetadataToRemove;
+        // Collect imported entities to be moved.
+        if (CU->getRawImportedEntities()) {
           for (Metadata *Op : CU->getImportedEntities()->operands()) {
             auto *IE = cast<DIImportedEntity>(Op);
-            if (auto *S = dyn_cast_or_null<DILocalScope>(IE->getScope())) {
-              EntitiesToRemove.insert(IE);
+            if (dyn_cast_or_null<DILocalScope>(IE->getScope())) {
+              MetadataToRemove.insert(IE);
             }
           }
+        }
+        // Collect enums to be moved.
+        if (CU->getRawEnumTypes()) {
+          for (Metadata *Op : CU->getEnumTypes()->operands()) {
+            auto *Enum = cast<DICompositeType>(Op);
+            if (dyn_cast_or_null<DILocalScope>(Enum->getScope())) {
+              MetadataToRemove.insert(Enum);
+            }
+          }
+        }
 
-          if (!EntitiesToRemove.empty()) {
-            // Make a new list of CU's 'imports'.
-            SmallVector<Metadata *> NewImports;
-            for (Metadata *Op : CU->getImportedEntities()->operands()) {
-              if (!EntitiesToRemove.contains(cast<DIImportedEntity>(Op))) {
+        if (!MetadataToRemove.empty()) {
+          // Make a new list of CU's 'imports'.
+          SmallVector<Metadata *> NewImports;
+          if (CU->getRawImportedEntities())
+            for (Metadata *Op : CU->getImportedEntities()->operands())
+              if (!MetadataToRemove.contains(Op))
                 NewImports.push_back(Op);
-              }
-            }
 
-            // Find DISubprogram corresponding to each entity.
-            std::map<DISubprogram *, SmallVector<Metadata *>> SPToEntities;
-            for (auto *I : EntitiesToRemove) {
-              auto *Entity = cast<DIImportedEntity>(I);
-              if (auto *SP = findEnclosingSubprogram(
-                      cast<DILocalScope>(Entity->getScope()))) {
-                SPToEntities[SP].push_back(Entity);
-              }
+          // Make a new list of CU's 'enums'.
+          SmallVector<Metadata *> NewEnums;
+          if (CU->getRawEnumTypes())
+            for (Metadata *Op : CU->getEnumTypes()->operands())
+              if (!MetadataToRemove.contains(Op))
+                NewEnums.push_back(Op);
+
+          // Find DISubprogram corresponding to each entity.
+          std::map<DISubprogram *, SmallVector<Metadata *>> SPToEntities;
+          for (auto *I : MetadataToRemove) {
+            DILocalScope *Scope = nullptr;
+            if (auto *Entity = dyn_cast<DIImportedEntity>(I))
+              Scope = cast<DILocalScope>(Entity->getScope());
+            else if (auto *Enum = dyn_cast<DICompositeType>(I))
+              Scope = cast<DILocalScope>(Enum->getScope());
+
+            if (auto *SP = findEnclosingSubprogram(Scope)) {
+              SPToEntities[SP].push_back(I);
             }
+          }
 
-            // Update DISubprograms' retainedNodes.
-            for (auto I = SPToEntities.begin(); I != SPToEntities.end(); ++I) {
-              auto *SP = I->first;
-              auto RetainedNodes = SP->getRetainedNodes();
-              SmallVector<Metadata *> MDs(RetainedNodes.begin(),
-                                          RetainedNodes.end());
-              MDs.append(I->second);
-              SP->replaceRetainedNodes(MDNode::get(Context, MDs));
-            }
+          // Update DISubprograms' retainedNodes.
+          for (auto I = SPToEntities.begin(); I != SPToEntities.end(); ++I) {
+            auto *SP = I->first;
+            auto RetainedNodes = SP->getRetainedNodes();
+            SmallVector<Metadata *> MDs(RetainedNodes.begin(),
+                                        RetainedNodes.end());
+            MDs.append(I->second);
+            SP->replaceRetainedNodes(MDNode::get(Context, MDs));
+          }
 
-            // Remove entities with local scope from CU.
+          // Remove entities with local scope from CU.
+          if (CU->getRawImportedEntities())
             CU->replaceImportedEntities(MDTuple::get(Context, NewImports));
-          }
+          // Remove enums with local scope from CU.
+          if (CU->getRawEnumTypes())
+            CU->replaceEnumTypes(MDTuple::get(Context, NewEnums));
         }
       }
     }
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
index 58ed21379d29b..bf8bdbb7da49f 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
@@ -598,10 +598,9 @@ void DwarfCompileUnit::constructScopeDIE(LexicalScope *Scope,
     return;
 
   // Emit lexical blocks.
-  DIE *ScopeDIE = constructLexicalScopeDIE(Scope);
+  DIE *ScopeDIE = getOrCreateLexicalBlockDIE(Scope, ParentScopeDIE);
   assert(ScopeDIE && "Scope DIE should not be null.");
 
-  ParentScopeDIE.addChild(ScopeDIE);
   createAndAddScopeChildren(Scope, *ScopeDIE);
 }
 
@@ -721,24 +720,39 @@ DIE *DwarfCompileUnit::constructInlinedScopeDIE(LexicalScope *Scope,
   return ScopeDIE;
 }
 
-// Construct new DW_TAG_lexical_block for this scope and attach
-// DW_AT_low_pc/DW_AT_high_pc labels.
-DIE *DwarfCompileUnit::constructLexicalScopeDIE(LexicalScope *Scope) {
+DIE *DwarfCompileUnit::getOrCreateLexicalBlockDIE(LexicalScope *Scope,
+                                                  DIE &ParentScopeDIE) {
   if (DD->isLexicalScopeDIENull(Scope))
     return nullptr;
   const auto *DS = Scope->getScopeNode();
-
-  auto ScopeDIE = DIE::get(DIEValueAllocator, dwarf::DW_TAG_lexical_block);
-  if (Scope->isAbstractScope()) {
-    assert(!getAbstractScopeDIEs().count(DS) &&
-           "Abstract DIE for this scope exists!");
-    getAbstractScopeDIEs()[DS] = ScopeDIE;
-    return ScopeDIE;
+  DIE *ScopeDIE = nullptr;
+
+  // FIXME: We may have a concrete DIE for this scope already created.
+  // This may happen when we emit local variables for an abstract tree of
+  // an inlined function: if a local variable has a templated type with
+  // a function-local type as a template parameter. See PR55680 for details
+  // (see also llvm/test/DebugInfo/Generic/local-type-as-template-parameter.ll).
+  if (!Scope->isAbstractScope() && !Scope->getInlinedAt()) {
+    if (auto It = LexicalBlockDIEs.find(DS); It != LexicalBlockDIEs.end()) {
+      ScopeDIE = It->second;
+      assert(!ScopeDIE->findAttribute(dwarf::DW_AT_low_pc) &&
+             !ScopeDIE->findAttribute(dwarf::DW_AT_ranges));
+      assert(ScopeDIE->getParent() == &ParentScopeDIE);
+    }
   }
-  if (!Scope->getInlinedAt()) {
-    assert(!LexicalBlockDIEs.count(DS) &&
-           "Concrete out-of-line DIE for this scope exists!");
-    LexicalBlockDIEs[DS] = ScopeDIE;
+  if (!ScopeDIE) {
+    ScopeDIE = DIE::get(DIEValueAllocator, dwarf::DW_TAG_lexical_block);
+    ParentScopeDIE.addChild(ScopeDIE);
+
+    if (Scope->isAbstractScope()) {
+      assert(!getAbstractScopeDIEs().count(DS) &&
+             "Abstract DIE for this scope exists!");
+      getAbstractScopeDIEs()[DS] = ScopeDIE;
+      return ScopeDIE;
+    }
+
+    if (!Scope->getInlinedAt())
+      LexicalBlockDIEs[DS] = ScopeDIE;
   }
 
   attachRangesOrLowHighPC(*ScopeDIE, Scope->getRanges());
@@ -1681,15 +1695,21 @@ void DwarfCompileUnit::createBaseTypeDIEs() {
   }
 }
 
-DIE *DwarfCompileUnit::getLexicalBlockDIE(const DILexicalBlock *LB) {
+DIE *DwarfCompileUnit::getLocalContextDIE(const DILexicalBlock *LB) {
   // Assume if there is an abstract tree all the DIEs are already emitted.
   bool isAbstract = getAbstractScopeDIEs().count(LB->getSubprogram());
   if (isAbstract && getAbstractScopeDIEs().count(LB))
     return getAbstractScopeDIEs()[LB];
   assert(!isAbstract && "Missed lexical block DIE in abstract tree!");
 
-  // Return a concrete DIE if it exists or nullptr otherwise.
-  return LexicalBlockDIEs.lookup(LB);
+  // Check if we have a concrete DIE.
+  if (auto It = LexicalBlockDIEs.find(LB); It != LexicalBlockDIEs.end())
+    return It->second;
+
+  // If nothing available found, we cannot just create a new lexical block,
+  // because it isn't known where to put it into the DIE tree.
+  // So, we may only try to find the most close avaiable parent DIE.
+  return getOrCreateContextDIE(LB->getScope()->getNonLexicalBlockFileScope());
 }
 
 DIE *DwarfCompileUnit::getOrCreateContextDIE(const DIScope *Context) {
@@ -1697,7 +1717,7 @@ DIE *DwarfCompileUnit::getOrCreateContextDIE(const DIScope *Context) {
     if (auto *LFScope = dyn_cast<DILexicalBlockFile>(Context))
       Context = LFScope->getNonLexicalBlockFileScope();
     if (auto *LScope = dyn_cast<DILexicalBlock>(Context))
-      return getLexicalBlockDIE(LScope);
+      return getLocalContextDIE(LScope);
 
     // Otherwise the context must be a DISubprogram.
     auto *SPScope = cast<DISubprogram>(Context);
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
index 6ef73ebd4f7f2..5662087bec322 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
@@ -210,14 +210,9 @@ class DwarfCompileUnit final : public DwarfUnit {
   /// DIE to represent this concrete inlined copy of the function.
   DIE *constructInlinedScopeDIE(LexicalScope *Scope, DIE &ParentScopeDIE);
 
-  /// Construct new DW_TAG_lexical_block for this scope and
-  /// attach DW_AT_low_pc/DW_AT_high_pc labels.
-  DIE *constructLexicalScopeDIE(LexicalScope *Scope);
-
-  /// Get a DIE for the given DILexicalBlock.
-  /// Note that this function assumes that the DIE has been already created
-  /// and it's an error, if it hasn't.
-  DIE *getLexicalBlockDIE(const DILexicalBlock *LB);
+  /// Get if available or create a new DW_TAG_lexical_block for the given
+  /// LexicalScope and attach DW_AT_low_pc/DW_AT_high_pc labels.
+  DIE *getOrCreateLexicalBlockDIE(LexicalScope *Scope, DIE &ParentDIE);
 
   /// constructVariableDIE - Construct a DIE for the given DbgVariable.
   DIE *constructVariableDIE(DbgVariable &DV, bool Abstract = false);
@@ -234,6 +229,11 @@ class DwarfCompileUnit final : public DwarfUnit {
   /// This instance of 'getOrCreateContextDIE()' can handle DILocalScope.
   DIE *getOrCreateContextDIE(const DIScope *Ty) override;
 
+  /// Get DW_TAG_lexical_block for the given DILexicalBlock if available,
+  /// or the most close parent DIE, if no correspoding DW_TAG_lexical_block
+  /// exists.
+  DIE *getLocalContextDIE(const DILexicalBlock *LB);
+
   /// Construct a DIE for this subprogram scope.
   DIE &constructSubprogramScopeDIE(const DISubprogram *Sub,
                                    LexicalScope *Scope);
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index df2161410ad08..f0c13edaf4bd3 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -1229,12 +1229,13 @@ void DwarfDebug::beginModule(Module *M) {
         CU.getOrCreateGlobalVariableDIE(GV, sortGlobalExprs(GVMap[GV]));
     }
 
-    for (auto *Ty : CUNode->getEnumTypes())
+    for (auto *Ty : CUNode->getEnumTypes()) {
+      assert(!isa_and_nonnull<DILocalScope>(Ty->getScope()) &&
+             "Unexpected function-local entity in 'enums' CU field.");
       CU.getOrCreateTypeDIE(cast<DIType>(Ty));
+    }
 
     for (auto *Ty : CUNode->getRetainedTypes()) {
-      // The retained types array by design contains pointers to
-      // MDNodes rather than DIRefs. Unique them here.
       if (DIType *RT = dyn_cast<DIType>(Ty))
         // There is no point in force-emitting a forward declaration.
         CU.getOrCreateTypeDIE(RT);
@@ -1428,9 +1429,13 @@ void DwarfDebug::endModule() {
              "Unexpected function-local entity in 'imports' CU field.");
       CU->getOrCreateImportedEntityDIE(IE);
     }
+
+    // Emit function-local entities.
     for (const auto *D : CU->getDeferredLocalDecls()) {
       if (auto *IE = dyn_cast<DIImportedEntity>(D))
         CU->getOrCreateImportedEntityDIE(IE);
+      else if (auto *Ty = dyn_cast<DIType>(D))
+        CU->getOrCreateTypeDIE(Ty);
       else
         llvm_unreachable("Unexpected local retained node!");
     }
@@ -1528,6 +1533,8 @@ static const DILocalScope *getRetainedNodeScope(const MDNode *N) {
     S = L->getScope();
   else if (const auto *IE = dyn_cast<DIImportedEntity>(N))
     S = IE->getScope();
+  else if (const auto *T = dyn_cast<DIType>(N))
+    S = T->getScope();
   else
     llvm_unreachable("Unexpected retained node!");
 
diff --git a/llvm/lib/IR/DIBuilder.cpp b/llvm/lib/IR/DIBuilder.cpp
index ae1a1a37d7c74..242ca44794262 100644
--- a/llvm/lib/IR/DIBuilder.cpp
+++ b/llvm/lib/IR/DIBuilder.cpp
@@ -29,7 +29,7 @@ DIBuilder::DIBuilder(Module &m, bool AllowUnresolvedNodes, DICompileUnit *CU)
       AllowUnresolvedNodes(AllowUnresolvedNodes) {
   if (CUNode) {
     if (const auto &ETs = CUNode->getEnumTypes())
-      AllEnumTypes.assign(ETs.begin(), ETs.end());
+      EnumTypes.assign(ETs.begin(), ETs.end());
     if (const auto &RTs = CUNode->getRetainedTypes())
       AllRetainTypes.assign(RTs.begin(), RTs.end());
     if (const auto &GVs = CUNode->getGlobalVariables())
@@ -66,10 +66,10 @@ void DIBuilder::finalize() {
     return;
   }
 
-  if (!AllEnumTypes.empty())
+  if (!EnumTypes.empty())
     CUNode->replaceEnumTypes(MDTuple::get(
-        VMContext, SmallVector<Metadata *, 16>(AllEnumTypes.begin(),
-                                               AllEnumTypes.end())));
+        VMContext, SmallVector<Metadata *, 16>(EnumTypes.begin(),
+                                               EnumTypes.end())));
 
   SmallVector<Metadata *, 16> RetainValues;
   // Declarations and definitions of the same type may be retained. Some
@@ -334,10 +334,13 @@ DIDerivedType *DIBuilder::createTypedef(DIType *Ty, StringRef Name,
                                         DIScope *Context, uint32_t AlignInBits,
                                         DINode::DIFlags Flags,
                                         DINodeArray Annotations) {
-  return DIDerivedType::get(VMContext, dwarf::DW_TAG_typedef, Name, File,
-                            LineNo, getNonCompileUnitScope(Context), Ty, 0,
-                            AlignInBits, 0, std::nullopt, Flags, nullptr,
-                            Annotations);
+  auto *T =
+      DIDerivedType::get(VMContext, dwarf::DW_TAG_typedef, Name, File, LineNo,
+                         getNonCompileUnitScope(Context), Ty, 0, AlignInBits, 0,
+                         std::nullopt, Flags, nullptr, Annotations);
+  if (isa_and_nonnull<DILocalScope>(Context))
+    getSubprogramNodesTrackingVector(Context).emplace_back(T);
+  return T;
 }
 
 DIDerivedType *DIBuilder::createFriend(DIType *Ty, DIType *FriendTy) {
@@ -485,6 +488,8 @@ DICompositeType *DIBuilder::createClassType(
       OffsetInBits, Flags, Elements, 0, VTableHolder,
       cast_or_null<MDTuple>(TemplateParams), UniqueIdentifier);
   trackIfUnresolved(R);
+  if (isa_and_nonnull<DILocalScope>(Context))
+    getSubprogramNodesTrackingVector(Context).emplace_back(R);
   return R;
 }
 
@@ -498,6 +503,8 @@ DICompositeType *DIBuilder::createStructType(
       getNonCompileUnitScope(Context), DerivedFrom, SizeInBits, AlignInBits, 0,
       Flags, Elements, RunTimeLang, VTableHolder, nullptr, UniqueIdentifier);
   trackIfUnresolved(R);
+  if (isa_and_nonnull<DILocalScope>(Context))
+    getSubprogramNodesTrackingVector(Context).emplace_back(R);
   return R;
 }
 
@@ -510,6 +517,8 @@ DICompositeType *DIBuilder::createUnionType(
       getNonCompileUnitScope(Scope), nullptr, SizeInBits, AlignInBits, 0, Flags,
       Elements, RunTimeLang, nullptr, nullptr, UniqueIdentifier);
   trackIfUnresolved(R);
+  if (isa_and_nonnull<DILocalScope>(Scope))
+    getSubprogramNodesTrackingVector(Scope).emplace_back(R);
   return R;
 }
 
@@ -542,7 +551,10 @@ DICompositeType *DIBuilder::createEnumerationType(
       getNonCompileUnitScope(Scope), UnderlyingType, SizeInBits, AlignInBits, 0,
       IsScoped ? DINode::FlagEnumClass : DINode::FlagZero, Elements, 0, nullptr,
       nullptr, UniqueIdentifier);
-  AllEnumTypes.emplace_back(CTy);
+  if (isa_and_nonnull<DILocalScope>(Scope))
+    getSubprogramNodesTrackingVector(Scope).emplace_back(CTy);
+  else
+    EnumTypes.emplace_back(CTy);
   trackIfUnresolved(CTy);
   return CTy;
 }
@@ -640,6 +652,8 @@ DIBuilder::createForwardDecl(unsigned Tag, StringRef Name, DIScope *Scope,
       SizeInBits, AlignInBits, 0, DINode::FlagFwdDecl, nullptr, RuntimeLang,
       nullptr, nullptr, UniqueIdentifier);
   trackIfUnresolved(RetTy);
+  if (isa_and_nonnull<DILocalScope>(Scope))
+    getSubprogramNodesTrackingVector(Scope).emplace_back(RetTy);
   return RetTy;
 }
 
@@ -656,6 +670,8 @@ DICompositeType *DIBuilder::createReplaceableCompositeType(
           nullptr, Annotations)
           .release();
   trackIfUnresolved(RetTy);
+  if (isa_and_nonnull<DILocalScope>(Scope))
+    getSubprogramNodesTrackingVector(Scope).emplace_back(RetTy);
   return RetTy;
 }
 
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 293d443d754a4..8f929dc7f48ab 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -1384,9 +1384,9 @@ void Verifier::visitDISubprogram(const DISubprogram &N) {
     CheckDI(Node, "invalid retained nodes list", &N, RawNode);
     for (Metadata *Op : Node->operands()) {
       CheckDI(Op && (isa<DILocalVariable>(Op) || isa<DILabel>(Op) ||
-                     isa<DIImportedEntity>(Op)),
-              "invalid retained nodes, expected DILocalVariable, DILabel or "
-              "DIImportedEntity",
+                     isa<DIImportedEntity>(Op) || isa<DIType>(Op)),
+              "invalid retained nodes, expected DILocalVariable, DILabel, "
+              "DIImportedEntity or DIType",
               &N, Node, Op);
     }
   }
diff --git a/llvm/test/Bitcode/upgrade-cu-locals.ll b/llvm/test/Bitcode/upgrade-cu-locals.ll
index 9a590f0fc0774..7052ab379bea4 100644
--- a/llvm/test/Bitcode/upgrade-cu-locals.ll
+++ b/llvm/test/Bitcode/upgrade-cu-locals.ll
@@ -1,4 +1,4 @@
-; Test moving of local imports from DICompileUnit's 'imports' to DISubprogram's 'retainedNodes'
+; Test moving of local imports/enums from DICompileUnit to DISubprogram's 'retainedNodes'
 ;
 ; RUN: llvm-dis -o - %s.bc | FileCheck %s
 
@@ -31,31 +31,36 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo
 ; CHECK: !4 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t4"
 
 ; CHECK: !5 = !{}
-; CHECK: !6 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !7, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, imports: !5, nameTableKind: GNU)
-
-; CHECK: !14 = distinct !DISubprogram(name: "main", scope: !7, file: !7, line: 2, type: !15, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !18)
-; CHECK: !18 = !{!19}
-; CHECK: !19 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !20, entity: !23,
-; CHECK: !20 = !DILexicalBlock(scope: !21, file: !7, line: 7, column: 35)
-; CHECK: !21 = !DILexicalBlock(scope: !22, file: !7, line: 7, column: 35)
-; CHECK: !22 = !DILexicalBlock(scope: !14, file: !7, line: 7, column: 35)
-; CHECK: !23 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t5", scope: !20,
-
-; CHECK: !25 = distinct !DISubprogram(name: "f1", linkageName: "_Z2f1v", scope: !1, file: !1, line: 3, type: !26, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !28)
-; CHECK: !28 = !{!29, !32, !34}
-; CHECK: !29 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !25, entity: !30,
-; CHECK: !30 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t1",
-; CHECK: !32 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !25, entity: !33,
-; CHECK: !33 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t2",
-; CHECK: !34 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !25, entity: !35,
-; CHECK: !35 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t3",
-
-; CHECK: !40 = distinct !DISubprogram(name: "main2", scope: !7, file: !7, line: 10, type: !15, scopeLine: 10, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !41)
-; CHECK: !41 = !{!42, !44}
-; CHECK: !42 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !40, entity: !43,
-; CHECK: !43 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t6"
-; CHECK: !44 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !40, entity: !45,
-; CHECK: !45 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t7",
+; CHECK: !6 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !7, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !8, imports: !5, nameTableKind: GNU)
+; CHECK: !7 = !DIFile(filename: "b.cpp"
+; CHECK: !8 = !{!9}
+; CHECK: !9 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum2", scope: !6, file: !7, line: 4, size: 8, align: 8, elements: !5)
+
+; CHECK: !16 = distinct !DISubprogram(name: "main", scope: !7, file: !7, line: 2, type: !17, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !20)
+; CHECK: !20 = !{!21}
+; CHECK: !21 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !22, entity: !25,
+; CHECK: !22 = !DILexicalBlock(scope: !23, file: !7, line: 7, column: 35)
+; CHECK: !23 = !DILexicalBlock(scope: !24, file: !7, line: 7, column: 35)
+; CHECK: !24 = !DILexicalBlock(scope: !16, file: !7, line: 7, column: 35)
+; CHECK: !25 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t5", scope: !22,
+
+; CHECK: !27 = distinct !DISubprogram(name: "f1", linkageName: "_Z2f1v", scope: !1, file: !1, line: 3, type: !28, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !30)
+; CHECK: !30 = !{!31, !34, !36}
+; CHECK: !31 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !27, entity: !32,
+; CHECK: !32 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t1",
+; CHECK: !34 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !27, entity: !35,
+; CHECK: !35 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t2",
+; CHECK: !36 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !27, entity: !37,
+; CHECK: !37 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t3",
+
+; CHECK: !42 = distinct !DISubprogram(name: "main2", scope: !7, file: !7, line: 10, type: !17, scopeLine: 10, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !43)
+; CHECK: !43 = !{!44, !46, !48, !49}
+; CHECK: !44 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !42, entity: !45,
+; CHECK: !45 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t6"
+; CHECK: !46 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !42, entity: !47,
+; CHECK: !47 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t7",
+; CHECK: !48 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum", scope: !42, file: !7, line: 3, size: 8, align: 8, elements: !5)
+; CHECK: !49 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum3", scope: !42, file: !7, line: 5, size: 8, align: 8, elements: !5)
 
 
 !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, imports: !2, nameTableKind: GNU)
@@ -82,7 +87,7 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo
 ; Leave t4 in CU
 !14 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !0, entity: !15, file: !1, line: 3)
 !15 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t4", scope: !0, file: !1, line: 1, size: 8, flags: DIFlagTypePassByValue, elements: !7, identifier: "_ZTSN2ns2t4E")
-!16 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !17, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, imports: !18, nameTableKind: GNU)
+!16 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !17, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, imports: !18, enums: !50, nameTableKind: GNU)
 !17 = !DIFile(filename: "b.cpp", directory: "/")
 !18 = !{!19, !28, !31}
 
@@ -116,3 +121,12 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo
 !41 = distinct !DILocation(line: 3, column: 3, scope: !23)
 !42 = !DILocation(line: 3, column: 41, scope: !4, inlinedAt: !41)
 !43 = !DILocation(line: 4, column: 1, scope: !23)
+
+!50 = !{!51, !52, !53}
+; Move to main2
+!51 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum", scope: !29, file: !17, line: 3, size: 8, align: 8, elements: !7)
+; Leave in b.cpp's CU
+!52 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum2", scope: !16, file: !17, line: 4, size: 8, align: 8, elements: !7)
+; Move to main2
+!53 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum3", scope: !29, file: !17, line: 5, size: 8, align: 8, elements: !7)
+
diff --git a/llvm/test/Bitcode/upgrade-cu-locals.ll.bc b/llvm/test/Bitcode/upgrade-cu-locals.ll.bc
index 9d0ea8fd9a3704ea1fb3a92250500addc75f0587..f73d05668af5b16f9ee03e400e2877faab339a0d 100644
GIT binary patch
delta 1080
zcmYk5e`p(J7{{NxT<<RFwOop6V|rQMH7e;k%bOqVBKYItmQkU&t8>^v2T79^Yj3eX
zCa4ST^$4!hY-CFq`$I=`D*iELuKy_e=R!ISm`cGRL56hTA0ibxhWMA%_q{6H%X`oB
z-1ohE-p}*!zJ}+nr}PA4Tzv8e=ETMO&yU}{_3PDJt*yNt=um-!|A&k}H9J|6zN*x)
zz`pltPnw1RR8~Q?N^cBEwg8N+ilr9PDjEQ4=T<*p+z#fi@<&_{iTssC`dw0W77=EH
z*c_9SV-gA2gRx8VvGQFm-JQ(6b&5U_YNDso)49lW?sWONHl>_yJF*18yXh_f*wo~p
zm<*emHz?jF4h4_b4>b}uctVLpliLG87yzu1pRs>l{*5sRvWb=j0!DRwX=wDE!o5S^
zOf<4d(z>W#%y#_LCnWZe`V1s?y-g$s)sK#yRhD8G6zL2mOMNo{bO~2Hg=K}-PyV<3
zq#>kxqe*`rGx=)KWD%^aF+h`3j$?WG#v>f#G%w}o32^!&9A2suj-yWr%l!{I44%|a
zL4<$~0r4859CHsjn1Ia@0;sw`ra%slionW?#uo{nWGL`pFMPtkj$s~}XJN(nVGSA#
z4zOq@#XFE-No&eR!r!V%R>}e1C})ip;CBXV%(E=PadK-*f}<Bi)~=kQK=3&0zR$P?
zFab!z?O}~vR!8z0DHMT0x@`|!-w+nJg$D{euBG5HOG<>AheRU;84oDiwJ=Y7|E^bU
zPf#aaZ~liW_I^`?Z3Zt9R(*qaScJ9SUwv_3Q<kDq+B#R0>fX~?T-W%k-X<>8UgL=j
zd}ekQW|i;f7Vq_tsi(X3-JsT;L+v-o1XsyBp4;T7(%n-D;dAR-CS<UohPl7N`l(K4
zr0Ys`qSe~^Wumbb`*GsCKMuR^K<6vz%<9a-OTmGKYrRRW?O0>w{Q6hxpZ$6M`h_o>
zw~J42>AA|JXZ&N&#J$xHv5k3on>Xg(ukv3b;h>Ln<a)b(XM#LwT?+4icV}nkh0TA6
zo&B4bdNC2MEc==)o>a+U<_nRGCQVoQRnT!plNqI)a{8Ff@)_sAV5(!c!vJ$=_ozlX
z<-P*D&c#bd?4=WAs{s;qIV!309p_*u{7g7Dd~zy26wOR#W081lI2)ci5Q#<(#PJXw
bjz`1MY<74kUf$<AES)45abZx`X{vt#JnkA(

delta 1011
zcmYjPT}&fY6h1R;?+nAR8K7ukpbT3~NOmQ+|Ckt^#_fv6bXy??4aQDuk;YOAbd5%0
zrv<}iQ#P<|%$oR6v-sc!%?9*sAEv1*sKCpvhU}79AB@J-1png8D&A>B@aE3>`OdxH
z`DW%Ebf_IG{nX^@Bi~TU>Tgesr^SEYyYtha=j~yWY!HBg<7MPOE-e79rviCYP}RO~
z{gy^qhXoMu+C9!tM`Yd7K=*rCbF3X;n<h+~OH`|YIw-tvv~}+*HOenH<t0qpa4y8=
zHS>ZdEU$I902B_4N`Kn>%1fgo0Y`Gqk@ou2o|DYupHkvUM?9G>Oq(ubP_5n(a4sm=
zSie0NwAk3&#7VXnE9Y+cR78YGZ0=kSKu|xb4S$DwriFb<1z>Yhhyjpv9A9-m_myaT
zL(g*3@Li&q)A9VPEq8uL*&bj)mDsk61ggK}NN7uR?PWy%O-xFsCjs;lE_N)IMV>_D
zKJ<VL#yUlNktuhbP{;%YAPW0`^xgcQLag7?M(?gN<`w4afdQb^OCrcbyRM_~7ovJk
zNHh~b24E!gv^6YS+3S+7BYC33XFb+2(9pq;LKfhUKA1%l7^Y!o3AQ_nWw=G*URruh
zMutPA&9iE$Y~eK#oI*-qW-;T0g%$|!2;gj`Q;a5?4q3o2P@nm%-O&sQe%1rxYjsbU
z=Cxy{^=ua}TlJP3D30MR*uj5hdmoe^XJpi20TX^o;0UY4X?z)1s<H2RgJn{|6$#z<
zR;n`*f?W)!5|HZNntA7khsjPT59ziZB&B9$Lu%dtZpAW|tfCTdC6Y?b=`i<!I2Ng*
zNfkNTu9e33_iOjYx4+mM|MHjft$U@G=auN@>vPZ8&dgmG{g!t_+n1JiKi&Q4*X8vq
zpX}VuKUzy9i#hYNWpjYqoMKH>Ktt-x<rKPch!#4@&scBPKP}i01Gz9h`{B{i(UTAU
zA-4KAF_pZ}UMzRsq;W(hvxgQP(I6jBp>;^$Xi)Hp1+A%zQVaK+&J4<A;u~v5YD|{n
zlyG{H{@geu1nPyCNvpFYYBLlg1s69s5SfVLsK@JZx`(jG?sT~nm&;-I+LfWlT#gBk
cSBb^kZo4~HxXhgw;!~;V+4S6CBGKsbANlGKqW}N^

diff --git a/llvm/test/DebugInfo/Generic/inlined-local-type.ll b/llvm/test/DebugInfo/Generic/inlined-local-type.ll
new file mode 100644
index 0000000000000..a6f7d294de928
--- /dev/null
+++ b/llvm/test/DebugInfo/Generic/inlined-local-type.ll
@@ -0,0 +1,128 @@
+; RUN: %llc_dwarf -O0 -filetype=obj < %s | llvm-dwarfdump -debug-info - | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s
+; REQUIRES: object-emission
+
+; inline __attribute__((always_inline))
+; int removed() { struct A {int i;}; struct A a; return a.i++; }
+;
+; __attribute__((always_inline))
+; int not_removed() { struct B {int i;}; struct B b; return b.i++; }
+;
+; int foo() { return removed() + not_removed(); }}
+
+; Ensure that function-local types have the correct subprogram parent even if
+; those subprograms are inlined.
+
+; CHECK: DW_TAG_compile_unit
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_AT_abstract_origin	({{0x.*}} "not_removed")
+; CHECK:     DW_TAG_variable
+; CHECK:     NULL
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_AT_name	("removed")
+; CHECK: [[A:0x.*]]: DW_TAG_structure_type
+; CHECK:       DW_AT_name	("A")
+; CHECK:       DW_TAG_member
+; CHECK:       NULL
+; CHECK:     DW_TAG_variable
+; CHECK:       DW_AT_type	([[A]] "A")
+; CHECK:     NULL
+; CHECK:   DW_TAG_base_type
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_AT_name	("not_removed")
+; CHECK: [[B:0x.*]]: DW_TAG_structure_type
+; CHECK:       DW_AT_name	("B")
+; CHECK:       DW_TAG_member
+; CHECK:       NULL
+; CHECK:     DW_TAG_variable
+; CHECK:       DW_AT_type	([[B]] "B")
+; CHECK:     NULL
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_TAG_inlined_subroutine
+; CHECK:       DW_TAG_variable
+; CHECK:       NULL
+; CHECK:     DW_TAG_inlined_subroutine
+; CHECK:       DW_TAG_variable
+; CHECK:       NULL
+; CHECK:     NULL
+; CHECK:   NULL
+
+%struct.B = type { i32 }
+%struct.A = type { i32 }
+
+define dso_local i32 @not_removed() !dbg !12 {
+  %1 = alloca %struct.B, align 4
+  call void @llvm.dbg.declare(metadata %struct.B* %1, metadata !18, metadata !DIExpression()), !dbg !22
+  %2 = getelementptr inbounds %struct.B, %struct.B* %1, i32 0, i32 0, !dbg !23
+  %3 = load i32, i32* %2, align 4, !dbg !24
+  %4 = add nsw i32 %3, 1, !dbg !24
+  store i32 %4, i32* %2, align 4, !dbg !24
+  ret i32 %3, !dbg !25
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+define dso_local i32 @foo() !dbg !26 {
+  %1 = alloca %struct.A, align 4
+  %2 = alloca %struct.B, align 4
+  call void @llvm.dbg.declare(metadata %struct.A* %1, metadata !27, metadata !DIExpression()), !dbg !32
+  %3 = getelementptr inbounds %struct.A, %struct.A* %1, i32 0, i32 0, !dbg !34
+  %4 = load i32, i32* %3, align 4, !dbg !35
+  %5 = add nsw i32 %4, 1, !dbg !35
+  store i32 %5, i32* %3, align 4, !dbg !35
+  call void @llvm.dbg.declare(metadata %struct.B* %2, metadata !18, metadata !DIExpression()), !dbg !36
+  %6 = getelementptr inbounds %struct.B, %struct.B* %2, i32 0, i32 0, !dbg !38
+  %7 = load i32, i32* %6, align 4, !dbg !39
+  %8 = add nsw i32 %7, 1, !dbg !39
+  store i32 %8, i32* %6, align 4, !dbg !39
+  %9 = add nsw i32 %4, %7, !dbg !40
+  ret i32 %9, !dbg !41
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3, !4, !5, !6, !7, !8, !9, !10}
+!llvm.ident = !{!11}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "inlined-local-type.cpp", directory: "")
+!2 = !{i32 7, !"Dwarf Version", i32 4}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"wchar_size", i32 4}
+!5 = !{i32 1, !"branch-target-enforcement", i32 0}
+!6 = !{i32 1, !"sign-return-address", i32 0}
+!7 = !{i32 1, !"sign-return-address-all", i32 0}
+!8 = !{i32 1, !"sign-return-address-with-bkey", i32 0}
+!9 = !{i32 7, !"uwtable", i32 1}
+!10 = !{i32 7, !"frame-pointer", i32 1}
+!11 = !{!"clang version 14.0.0"}
+!12 = distinct !DISubprogram(name: "not_removed", scope: !13, file: !13, line: 5, type: !14, scopeLine: 5, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !43)
+!13 = !DIFile(filename: "inlined-local-type.cpp", directory: "")
+!14 = !DISubroutineType(types: !15)
+!15 = !{!16}
+!16 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!17 = !{}
+!18 = !DILocalVariable(name: "b", scope: !12, file: !13, line: 5, type: !19)
+!19 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "B", scope: !12, file: !13, line: 5, size: 32, elements: !20)
+!20 = !{!21}
+!21 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !19, file: !13, line: 5, baseType: !16, size: 32)
+!22 = !DILocation(line: 5, column: 49, scope: !12)
+!23 = !DILocation(line: 5, column: 61, scope: !12)
+!24 = !DILocation(line: 5, column: 62, scope: !12)
+!25 = !DILocation(line: 5, column: 52, scope: !12)
+!26 = distinct !DISubprogram(name: "foo", scope: !13, file: !13, line: 7, type: !14, scopeLine: 7, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !17)
+!27 = !DILocalVariable(name: "a", scope: !28, file: !13, line: 2, type: !29)
+!28 = distinct !DISubprogram(name: "removed", scope: !13, file: !13, line: 2, type: !14, scopeLine: 2, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !42)
+!29 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A", scope: !28, file: !13, line: 2, size: 32, elements: !30)
+!30 = !{!31}
+!31 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !29, file: !13, line: 2, baseType: !16, size: 32)
+!32 = !DILocation(line: 2, column: 45, scope: !28, inlinedAt: !33)
+!33 = distinct !DILocation(line: 7, column: 20, scope: !26)
+!34 = !DILocation(line: 2, column: 57, scope: !28, inlinedAt: !33)
+!35 = !DILocation(line: 2, column: 58, scope: !28, inlinedAt: !33)
+!36 = !DILocation(line: 5, column: 49, scope: !12, inlinedAt: !37)
+!37 = distinct !DILocation(line: 7, column: 32, scope: !26)
+!38 = !DILocation(line: 5, column: 61, scope: !12, inlinedAt: !37)
+!39 = !DILocation(line: 5, column: 62, scope: !12, inlinedAt: !37)
+!40 = !DILocation(line: 7, column: 30, scope: !26)
+!41 = !DILocation(line: 7, column: 13, scope: !26)
+!42 = !{!29}
+!43 = !{!19}
diff --git a/llvm/test/DebugInfo/Generic/lexical-block-retained-types.ll b/llvm/test/DebugInfo/Generic/lexical-block-retained-types.ll
new file mode 100644
index 0000000000000..64de1323c98c6
--- /dev/null
+++ b/llvm/test/DebugInfo/Generic/lexical-block-retained-types.ll
@@ -0,0 +1,55 @@
+; RUN: %llc_dwarf -O0 -filetype=obj < %s | llvm-dwarfdump - | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s
+
+; Test that retained unused (unreferenced) types emission.
+
+; Compiled from
+; $ clang -cc1 -debug-info-kind=unused-types test.cpp -emit-llvm
+
+; void test_unused() {
+;   struct Y {};
+;   {
+;     struct X {};
+;   }
+; }
+
+; CHECK: DW_TAG_compile_unit
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_AT_name	("test_unused")
+; CHECK:     DW_TAG_structure_type
+; CHECK:       DW_AT_name	("Y")
+
+; FIXME: here should be DW_TAG_lexical_block as a parent of structure 'X'.
+; But it's not possible to reliably emit a lexical block for which a LexicalScope
+; wasn't created, so we just fallback to the most close parent DIE
+; (see DwarfCompileUnit::getOrCreateLexicalBlockDIE() for details).
+
+; CHECK:     DW_TAG_structure_type
+; CHECK:       DW_AT_name	("X")
+; CHECK:     NULL
+; CHECK:   NULL
+
+define dso_local void @_Z11test_unusedv() !dbg !5 {
+entry:
+  ret void, !dbg !16
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!13, !14}
+!llvm.ident = !{!15}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 15.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, retainedTypes: !2, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "<stdin>", directory: "/")
+!2 = !{!3, !10}
+!3 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Y", scope: !5, file: !4, line: 2, size: 8, flags: DIFlagTypePassByValue, elements: !8)
+!4 = !DIFile(filename: "test.cpp", directory: "/")
+!5 = distinct !DISubprogram(name: "test_unused", linkageName: "_Z11test_unusedv", scope: !4, file: !4, line: 1, type: !6, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !9)
+!6 = !DISubroutineType(types: !7)
+!7 = !{null}
+!8 = !{}
+!9 = !{!3, !10}
+!10 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "X", scope: !11, file: !4, line: 4, size: 8, flags: DIFlagTypePassByValue, elements: !8)
+!11 = distinct !DILexicalBlock(scope: !5, file: !4, line: 3, column: 3)
+!13 = !{i32 2, !"Debug Info Version", i32 3}
+!14 = !{i32 1, !"wchar_size", i32 4}
+!15 = !{!"clang version 15.0.0"}
+!16 = !DILocation(line: 6, column: 1, scope: !5)
diff --git a/llvm/test/DebugInfo/Generic/lexical-block-types.ll b/llvm/test/DebugInfo/Generic/lexical-block-types.ll
new file mode 100644
index 0000000000000..aef59984351a7
--- /dev/null
+++ b/llvm/test/DebugInfo/Generic/lexical-block-types.ll
@@ -0,0 +1,425 @@
+; RUN: %llc_dwarf -O0 -filetype=obj < %s | llvm-dwarfdump -debug-info - | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s
+; REQUIRES: object-emission
+
+; inline __attribute__((always_inline))
+; void removed() {
+;   struct A1 { int i; };
+;   typedef int Int1;
+;   {
+;     struct I1 { Int1 j; };
+;     struct C1 { typedef char Char1; Char1 c; };
+;     A1 a1; a1.i++;
+;     {
+;       I1 i1; i1.j++;
+;       C1 c1; c1.c++;
+;     }
+;   }
+; }
+;
+; __attribute__((always_inline))
+; void not_removed() {
+;   struct A2 { int i; };
+;   typedef int Int2;
+;   {
+;     struct I2 { Int2 j; };
+;     struct C2 { typedef char Char2; Char2 c; };
+;     A2 a2; a2.i++;
+;     {
+;       I2 i2; i2.j++;
+;       C2 c2; c2.c++;
+;     }
+;   }
+; }
+;
+; void foo() {
+;   struct A3 { int i; };
+;   typedef int Int3;
+;   {
+;     struct I3 { Int3 j; };
+;     {
+;       struct C3 { typedef char Char3; Char3 c; };
+;       A3 a3; a3.i++;
+;       {
+;         I3 i3; i3.j++;
+;         C3 c3; c3.c++;
+;       }
+;     }
+;   }
+;   removed();
+;   not_removed();
+; }
+;
+; CHECK: DW_TAG_compile_unit
+
+; Out-of-line definition of `not_removed()` shouldn't contain any debug info for types.
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_AT_abstract_origin	{{.*}} "_Z11not_removedv"
+; CHECK:     DW_TAG_lexical_block
+; CHECK:       DW_TAG_variable
+; CHECK:         DW_AT_abstract_origin	{{.*}} "a2"
+; CHECK:       DW_TAG_lexical_block
+; CHECK:         DW_TAG_variable
+; CHECK:           DW_AT_abstract_origin	{{.*}} "i2"
+; CHECK:         DW_TAG_variable
+; CHECK:           DW_AT_abstract_origin	{{.*}} "c2"
+; CHECK:         NULL
+; CHECK:       NULL
+; CHECK:     NULL
+
+; Abstract definition of `removed()`.
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_AT_name	("removed")
+; CHECK:     DW_AT_inline	(DW_INL_inlined)
+
+; I1 and C1 defined in the first lexical block, typedef Char1 is a child of C1.
+; CHECK:     DW_TAG_lexical_block
+; CHECK:       DW_TAG_variable
+; CHECK:         DW_AT_name	("a1")
+; CHECK:         DW_AT_type	{{.*}} "A1"
+; CHECK:       DW_TAG_lexical_block
+; CHECK:         DW_TAG_variable
+; CHECK:           DW_AT_type	{{.*}} "I1"
+; CHECK:         DW_TAG_variable
+; CHECK:           DW_AT_type	{{.*}} "C1"
+; CHECK:         NULL
+; CHECK:       DW_TAG_structure_type
+; CHECK:         DW_AT_name	("I1")
+; CHECK:         DW_TAG_member
+; CHECK:           DW_AT_type	{{.*}} "Int1"
+; CHECK:         NULL
+; CHECK:       DW_TAG_structure_type
+; CHECK:         DW_AT_name	("C1")
+; CHECK:         DW_TAG_member
+; CHECK:           DW_AT_type	{{.*}} "Char1"
+; CHECK:         DW_TAG_typedef
+; CHECK:           DW_AT_name	("Char1")
+; CHECK:         NULL
+; CHECK:       NULL
+
+; A1 and typedef Int1 defined in the subprogram scope.
+; CHECK:     DW_TAG_structure_type
+; CHECK:       DW_AT_name	("A1")
+; CHECK:       DW_TAG_member
+; CHECK:       NULL
+; CHECK:     DW_TAG_typedef
+; CHECK:       DW_AT_name	("Int1")
+; CHECK:     NULL
+
+; CHECK:   DW_TAG_base_type
+; CHECK:   DW_TAG_base_type
+
+; Abstract definition of `not_removed()`.
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_AT_name	("not_removed")
+; CHECK:     DW_AT_inline	(DW_INL_inlined)
+
+; I2 and C2 defined in the first lexical block, typedef Char2 is a child of C2.
+; CHECK:     DW_TAG_lexical_block
+; CHECK:       DW_TAG_variable
+; CHECK:         DW_AT_name	("a2")
+; CHECK:         DW_AT_type	{{.*}} "A2"
+; CHECK:       DW_TAG_lexical_block
+; CHECK:         DW_TAG_variable
+; CHECK:           DW_AT_name	("i2")
+; CHECK:           DW_AT_type	{{.*}} "I2"
+; CHECK:         DW_TAG_variable
+; CHECK:           DW_AT_name	("c2")
+; CHECK:           DW_AT_type	{{.*}} "C2"
+; CHECK:         NULL
+; CHECK:       DW_TAG_structure_type
+; CHECK:         DW_AT_name	("I2")
+; CHECK:         DW_TAG_member
+; CHECK:           DW_AT_type	{{.*}} "Int2"
+; CHECK:         NULL
+; CHECK:       DW_TAG_structure_type
+; CHECK:         DW_AT_name	("C2")
+; CHECK:         DW_TAG_member
+; CHECK:           DW_AT_type	{{.*}} "Char2"
+; CHECK:         DW_TAG_typedef
+; CHECK:           DW_AT_name	("Char2")
+; CHECK:         NULL
+; CHECK:       NULL
+
+; A2 and typedef Int2 defined in subprogram scope.
+; CHECK:     DW_TAG_structure_type
+; CHECK:       DW_AT_name	("A2")
+; CHECK:       DW_TAG_member
+; CHECK:       NULL
+; CHECK:     DW_TAG_typedef
+; CHECK:       DW_AT_name	("Int2")
+; CHECK:     NULL
+
+; Definition of `foo()`.
+; CHECK:   DW_TAG_subprogram
+; CHECK:     DW_AT_name	("foo")
+
+; CHECK:     DW_TAG_lexical_block
+; CHECK:       DW_TAG_lexical_block
+; CHECK:         DW_TAG_variable
+; CHECK:           DW_AT_name	("a3")
+; CHECK:           DW_AT_type	{{.*}} "A3"
+; CHECK:         DW_TAG_lexical_block
+; CHECK:           DW_TAG_variable
+; CHECK:             DW_AT_name	("i3")
+; CHECK:             DW_AT_type	{{.*}} "I3"
+; CHECK:           DW_TAG_variable
+; CHECK:             DW_AT_name	("c3")
+; CHECK:             DW_AT_type	{{.*}} "C3"
+; CHECK:           NULL
+
+; C3 has the inner lexical block scope, typedef Char3 is a child of C3.
+; CHECK:         DW_TAG_structure_type
+; CHECK:           DW_AT_name	("C3")
+; CHECK:           DW_TAG_member
+; CHECK:             DW_AT_type	{{.*}} "Char3"
+; CHECK:           DW_TAG_typedef
+; CHECK:             DW_AT_name	("Char3")
+; CHECK:           NULL
+; CHECK:         NULL
+
+; I3 has the outer lexical block scope.
+; CHECK:       DW_TAG_structure_type
+; CHECK:         DW_AT_name	("I3")
+; CHECK:         DW_TAG_member
+; CHECK:           DW_AT_type	{{.*}} "Int3"
+; CHECK:         NULL
+; CHECK:       NULL
+
+; CHECK:     DW_TAG_inlined_subroutine
+; CHECK:       DW_AT_abstract_origin	{{.*}} "_Z7removedv"
+; CHECK:       DW_TAG_lexical_block
+; CHECK:         DW_TAG_variable
+; CHECK:         DW_TAG_lexical_block
+; CHECK:           DW_TAG_variable
+; CHECK:           DW_TAG_variable
+; CHECK:           NULL
+; CHECK:         NULL
+; CHECK:       NULL
+
+; CHECK:     DW_TAG_inlined_subroutine
+; CHECK:       DW_AT_abstract_origin	{{.*}} "_Z11not_removedv"
+; CHECK:       DW_TAG_lexical_block
+; CHECK:         DW_TAG_variable
+; CHECK:         DW_TAG_lexical_block
+; CHECK:           DW_TAG_variable
+; CHECK:           DW_TAG_variable
+; CHECK:           NULL
+; CHECK:         NULL
+; CHECK:       NULL
+
+; A3 and Int3 defined within the subprogam scope.
+; CHECK:     DW_TAG_structure_type
+; CHECK:       DW_AT_name	("A3")
+; CHECK:       DW_TAG_member
+; CHECK:       NULL
+; CHECK:     DW_TAG_typedef
+; CHECK:       DW_AT_name	("Int3")
+; CHECK:     NULL
+; CHECK:   NULL
+
+%struct.A2 = type { i32 }
+%struct.I2 = type { i32 }
+%struct.C2 = type { i8 }
+%struct.A1 = type { i32 }
+%struct.I1 = type { i32 }
+%struct.C1 = type { i8 }
+%struct.A3 = type { i32 }
+%struct.I3 = type { i32 }
+%struct.C3 = type { i8 }
+
+define dso_local void @_Z11not_removedv() !dbg !8 {
+entry:
+  %a2 = alloca %struct.A2, align 4
+  %i2 = alloca %struct.I2, align 4
+  %c2 = alloca %struct.C2, align 1
+  call void @llvm.dbg.declare(metadata %struct.A2* %a2, metadata !12, metadata !DIExpression()), !dbg !18
+  %i = getelementptr inbounds %struct.A2, %struct.A2* %a2, i32 0, i32 0, !dbg !19
+  %0 = load i32, i32* %i, align 4, !dbg !20
+  %inc = add nsw i32 %0, 1, !dbg !20
+  store i32 %inc, i32* %i, align 4, !dbg !20
+  call void @llvm.dbg.declare(metadata %struct.I2* %i2, metadata !21, metadata !DIExpression()), !dbg !27
+  %j = getelementptr inbounds %struct.I2, %struct.I2* %i2, i32 0, i32 0, !dbg !28
+  %1 = load i32, i32* %j, align 4, !dbg !29
+  %inc1 = add nsw i32 %1, 1, !dbg !29
+  store i32 %inc1, i32* %j, align 4, !dbg !29
+  call void @llvm.dbg.declare(metadata %struct.C2* %c2, metadata !30, metadata !DIExpression()), !dbg !36
+  %c = getelementptr inbounds %struct.C2, %struct.C2* %c2, i32 0, i32 0, !dbg !37
+  %2 = load i8, i8* %c, align 1, !dbg !38
+  %inc2 = add i8 %2, 1, !dbg !38
+  store i8 %inc2, i8* %c, align 1, !dbg !38
+  ret void, !dbg !39
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+define dso_local void @_Z3foov() !dbg !40 {
+entry:
+  %a1.i = alloca %struct.A1, align 4
+  %i1.i = alloca %struct.I1, align 4
+  %c1.i = alloca %struct.C1, align 1
+  %a2.i = alloca %struct.A2, align 4
+  %i2.i = alloca %struct.I2, align 4
+  %c2.i = alloca %struct.C2, align 1
+  %a3 = alloca %struct.A3, align 4
+  %i3 = alloca %struct.I3, align 4
+  %c3 = alloca %struct.C3, align 1
+  call void @llvm.dbg.declare(metadata %struct.A3* %a3, metadata !41, metadata !DIExpression()), !dbg !47
+  %i = getelementptr inbounds %struct.A3, %struct.A3* %a3, i32 0, i32 0, !dbg !48
+  %0 = load i32, i32* %i, align 4, !dbg !49
+  %inc = add nsw i32 %0, 1, !dbg !49
+  store i32 %inc, i32* %i, align 4, !dbg !49
+  call void @llvm.dbg.declare(metadata %struct.I3* %i3, metadata !50, metadata !DIExpression()), !dbg !56
+  %j = getelementptr inbounds %struct.I3, %struct.I3* %i3, i32 0, i32 0, !dbg !57
+  %1 = load i32, i32* %j, align 4, !dbg !58
+  %inc1 = add nsw i32 %1, 1, !dbg !58
+  store i32 %inc1, i32* %j, align 4, !dbg !58
+  call void @llvm.dbg.declare(metadata %struct.C3* %c3, metadata !59, metadata !DIExpression()), !dbg !64
+  %c = getelementptr inbounds %struct.C3, %struct.C3* %c3, i32 0, i32 0, !dbg !65
+  %2 = load i8, i8* %c, align 1, !dbg !66
+  %inc2 = add i8 %2, 1, !dbg !66
+  store i8 %inc2, i8* %c, align 1, !dbg !66
+  call void @llvm.dbg.declare(metadata %struct.A1* %a1.i, metadata !67, metadata !DIExpression()), !dbg !73
+  %i.i3 = getelementptr inbounds %struct.A1, %struct.A1* %a1.i, i32 0, i32 0, !dbg !75
+  %3 = load i32, i32* %i.i3, align 4, !dbg !76
+  %inc.i4 = add nsw i32 %3, 1, !dbg !76
+  store i32 %inc.i4, i32* %i.i3, align 4, !dbg !76
+  call void @llvm.dbg.declare(metadata %struct.I1* %i1.i, metadata !77, metadata !DIExpression()), !dbg !83
+  %j.i5 = getelementptr inbounds %struct.I1, %struct.I1* %i1.i, i32 0, i32 0, !dbg !84
+  %4 = load i32, i32* %j.i5, align 4, !dbg !85
+  %inc1.i6 = add nsw i32 %4, 1, !dbg !85
+  store i32 %inc1.i6, i32* %j.i5, align 4, !dbg !85
+  call void @llvm.dbg.declare(metadata %struct.C1* %c1.i, metadata !86, metadata !DIExpression()), !dbg !91
+  %c.i7 = getelementptr inbounds %struct.C1, %struct.C1* %c1.i, i32 0, i32 0, !dbg !92
+  %5 = load i8, i8* %c.i7, align 1, !dbg !93
+  %inc2.i8 = add i8 %5, 1, !dbg !93
+  store i8 %inc2.i8, i8* %c.i7, align 1, !dbg !93
+  call void @llvm.dbg.declare(metadata %struct.A2* %a2.i, metadata !12, metadata !DIExpression()), !dbg !94
+  %i.i = getelementptr inbounds %struct.A2, %struct.A2* %a2.i, i32 0, i32 0, !dbg !96
+  %6 = load i32, i32* %i.i, align 4, !dbg !97
+  %inc.i = add nsw i32 %6, 1, !dbg !97
+  store i32 %inc.i, i32* %i.i, align 4, !dbg !97
+  call void @llvm.dbg.declare(metadata %struct.I2* %i2.i, metadata !21, metadata !DIExpression()), !dbg !98
+  %j.i = getelementptr inbounds %struct.I2, %struct.I2* %i2.i, i32 0, i32 0, !dbg !99
+  %7 = load i32, i32* %j.i, align 4, !dbg !100
+  %inc1.i = add nsw i32 %7, 1, !dbg !100
+  store i32 %inc1.i, i32* %j.i, align 4, !dbg !100
+  call void @llvm.dbg.declare(metadata %struct.C2* %c2.i, metadata !30, metadata !DIExpression()), !dbg !101
+  %c.i = getelementptr inbounds %struct.C2, %struct.C2* %c2.i, i32 0, i32 0, !dbg !102
+  %8 = load i8, i8* %c.i, align 1, !dbg !103
+  %inc2.i = add i8 %8, 1, !dbg !103
+  store i8 %inc2.i, i8* %c.i, align 1, !dbg !103
+  ret void, !dbg !104
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3, !4, !5, !6}
+!llvm.ident = !{!7}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "test.cpp", directory: "/")
+!2 = !{i32 7, !"Dwarf Version", i32 4}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
+!4 = !{i32 1, !"wchar_size", i32 4}
+!5 = !{i32 7, !"uwtable", i32 1}
+!6 = !{i32 7, !"frame-pointer", i32 2}
+!7 = !{!"clang version 14.0.0"}
+!8 = distinct !DISubprogram(name: "not_removed", linkageName: "_Z11not_removedv", scope: !1, file: !1, line: 17, type: !9, scopeLine: 17, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !105)
+!9 = !DISubroutineType(types: !10)
+!10 = !{null}
+!11 = !{}
+!12 = !DILocalVariable(name: "a2", scope: !13, file: !1, line: 23, type: !14)
+!13 = distinct !DILexicalBlock(scope: !8, file: !1, line: 20, column: 3)
+!14 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A2", scope: !8, file: !1, line: 18, size: 32, flags: DIFlagTypePassByValue, elements: !15)
+!15 = !{!16}
+!16 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !14, file: !1, line: 18, baseType: !17, size: 32)
+!17 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!18 = !DILocation(line: 23, column: 8, scope: !13)
+!19 = !DILocation(line: 23, column: 15, scope: !13)
+!20 = !DILocation(line: 23, column: 16, scope: !13)
+!21 = !DILocalVariable(name: "i2", scope: !22, file: !1, line: 25, type: !23)
+!22 = distinct !DILexicalBlock(scope: !13, file: !1, line: 24, column: 5)
+!23 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "I2", scope: !13, file: !1, line: 21, size: 32, flags: DIFlagTypePassByValue, elements: !24)
+!24 = !{!25}
+!25 = !DIDerivedType(tag: DW_TAG_member, name: "j", scope: !23, file: !1, line: 21, baseType: !26, size: 32)
+!26 = !DIDerivedType(tag: DW_TAG_typedef, name: "Int2", scope: !8, file: !1, line: 19, baseType: !17)
+!27 = !DILocation(line: 25, column: 10, scope: !22)
+!28 = !DILocation(line: 25, column: 17, scope: !22)
+!29 = !DILocation(line: 25, column: 18, scope: !22)
+!30 = !DILocalVariable(name: "c2", scope: !22, file: !1, line: 26, type: !31)
+!31 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "C2", scope: !13, file: !1, line: 22, size: 8, flags: DIFlagTypePassByValue, elements: !32)
+!32 = !{!33}
+!33 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !31, file: !1, line: 22, baseType: !34, size: 8)
+!34 = !DIDerivedType(tag: DW_TAG_typedef, name: "Char2", scope: !31, file: !1, line: 22, baseType: !35)
+!35 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!36 = !DILocation(line: 26, column: 10, scope: !22)
+!37 = !DILocation(line: 26, column: 17, scope: !22)
+!38 = !DILocation(line: 26, column: 18, scope: !22)
+!39 = !DILocation(line: 29, column: 1, scope: !8)
+!40 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !1, file: !1, line: 31, type: !9, scopeLine: 31, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !107)
+!41 = !DILocalVariable(name: "a3", scope: !42, file: !1, line: 38, type: !44)
+!42 = distinct !DILexicalBlock(scope: !43, file: !1, line: 36, column: 5)
+!43 = distinct !DILexicalBlock(scope: !40, file: !1, line: 34, column: 3)
+!44 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A3", scope: !40, file: !1, line: 32, size: 32, flags: DIFlagTypePassByValue, elements: !45)
+!45 = !{!46}
+!46 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !44, file: !1, line: 32, baseType: !17, size: 32)
+!47 = !DILocation(line: 38, column: 10, scope: !42)
+!48 = !DILocation(line: 38, column: 17, scope: !42)
+!49 = !DILocation(line: 38, column: 18, scope: !42)
+!50 = !DILocalVariable(name: "i3", scope: !51, file: !1, line: 40, type: !52)
+!51 = distinct !DILexicalBlock(scope: !42, file: !1, line: 39, column: 7)
+!52 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "I3", scope: !43, file: !1, line: 35, size: 32, flags: DIFlagTypePassByValue, elements: !53)
+!53 = !{!54}
+!54 = !DIDerivedType(tag: DW_TAG_member, name: "j", scope: !52, file: !1, line: 35, baseType: !55, size: 32)
+!55 = !DIDerivedType(tag: DW_TAG_typedef, name: "Int3", scope: !40, file: !1, line: 33, baseType: !17)
+!56 = !DILocation(line: 40, column: 12, scope: !51)
+!57 = !DILocation(line: 40, column: 19, scope: !51)
+!58 = !DILocation(line: 40, column: 20, scope: !51)
+!59 = !DILocalVariable(name: "c3", scope: !51, file: !1, line: 41, type: !60)
+!60 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "C3", scope: !42, file: !1, line: 37, size: 8, flags: DIFlagTypePassByValue, elements: !61)
+!61 = !{!62}
+!62 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !60, file: !1, line: 37, baseType: !63, size: 8)
+!63 = !DIDerivedType(tag: DW_TAG_typedef, name: "Char3", scope: !60, file: !1, line: 37, baseType: !35)
+!64 = !DILocation(line: 41, column: 12, scope: !51)
+!65 = !DILocation(line: 41, column: 19, scope: !51)
+!66 = !DILocation(line: 41, column: 20, scope: !51)
+!67 = !DILocalVariable(name: "a1", scope: !68, file: !1, line: 8, type: !70)
+!68 = distinct !DILexicalBlock(scope: !69, file: !1, line: 5, column: 3)
+!69 = distinct !DISubprogram(name: "removed", linkageName: "_Z7removedv", scope: !1, file: !1, line: 2, type: !9, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !110)
+!70 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A1", scope: !69, file: !1, line: 3, size: 32, flags: DIFlagTypePassByValue, elements: !71, identifier: "_ZTSZ7removedvE2A1")
+!71 = !{!72}
+!72 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !70, file: !1, line: 3, baseType: !17, size: 32)
+!73 = !DILocation(line: 8, column: 8, scope: !68, inlinedAt: !74)
+!74 = distinct !DILocation(line: 45, column: 3, scope: !40)
+!75 = !DILocation(line: 8, column: 15, scope: !68, inlinedAt: !74)
+!76 = !DILocation(line: 8, column: 16, scope: !68, inlinedAt: !74)
+!77 = !DILocalVariable(name: "i1", scope: !78, file: !1, line: 10, type: !79)
+!78 = distinct !DILexicalBlock(scope: !68, file: !1, line: 9, column: 5)
+!79 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "I1", scope: !68, file: !1, line: 6, size: 32, flags: DIFlagTypePassByValue, elements: !80, identifier: "_ZTSZ7removedvE2I1")
+!80 = !{!81}
+!81 = !DIDerivedType(tag: DW_TAG_member, name: "j", scope: !79, file: !1, line: 6, baseType: !82, size: 32)
+!82 = !DIDerivedType(tag: DW_TAG_typedef, name: "Int1", scope: !69, file: !1, line: 4, baseType: !17)
+!83 = !DILocation(line: 10, column: 10, scope: !78, inlinedAt: !74)
+!84 = !DILocation(line: 10, column: 17, scope: !78, inlinedAt: !74)
+!85 = !DILocation(line: 10, column: 18, scope: !78, inlinedAt: !74)
+!86 = !DILocalVariable(name: "c1", scope: !78, file: !1, line: 11, type: !87)
+!87 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "C1", scope: !68, file: !1, line: 7, size: 8, flags: DIFlagTypePassByValue, elements: !88, identifier: "_ZTSZ7removedvE2C1")
+!88 = !{!89}
+!89 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !87, file: !1, line: 7, baseType: !90, size: 8)
+!90 = !DIDerivedType(tag: DW_TAG_typedef, name: "Char1", scope: !87, file: !1, line: 7, baseType: !35)
+!91 = !DILocation(line: 11, column: 10, scope: !78, inlinedAt: !74)
+!92 = !DILocation(line: 11, column: 17, scope: !78, inlinedAt: !74)
+!93 = !DILocation(line: 11, column: 18, scope: !78, inlinedAt: !74)
+!94 = !DILocation(line: 23, column: 8, scope: !13, inlinedAt: !95)
+!95 = distinct !DILocation(line: 46, column: 3, scope: !40)
+!96 = !DILocation(line: 23, column: 15, scope: !13, inlinedAt: !95)
+!97 = !DILocation(line: 23, column: 16, scope: !13, inlinedAt: !95)
+!98 = !DILocation(line: 25, column: 10, scope: !22, inlinedAt: !95)
+!99 = !DILocation(line: 25, column: 17, scope: !22, inlinedAt: !95)
+!100 = !DILocation(line: 25, column: 18, scope: !22, inlinedAt: !95)
+!101 = !DILocation(line: 26, column: 10, scope: !22, inlinedAt: !95)
+!102 = !DILocation(line: 26, column: 17, scope: !22, inlinedAt: !95)
+!103 = !DILocation(line: 26, column: 18, scope: !22, inlinedAt: !95)
+!104 = !DILocation(line: 47, column: 1, scope: !40)
+!105 = !{!14, !23, !26, !31}
+!107 = !{!44, !52, !55, !60}
+!110 = !{!70, !79, !82, !87}
diff --git a/llvm/test/DebugInfo/Generic/verifier-invalid-disubprogram.ll b/llvm/test/DebugInfo/Generic/verifier-invalid-disubprogram.ll
index 6d4d0e93d38f9..54ce1c56c6b30 100644
--- a/llvm/test/DebugInfo/Generic/verifier-invalid-disubprogram.ll
+++ b/llvm/test/DebugInfo/Generic/verifier-invalid-disubprogram.ll
@@ -38,7 +38,7 @@ define void @invalid_subprogram_declaration() !dbg !9 { ret void }
 define void @invalid_retained_nodes_list() !dbg !10 { ret void }
 !10 = distinct !DISubprogram(retainedNodes: !0)
 
-; CHECK: invalid retained nodes, expected DILocalVariable, DILabel or DIImportedEntity
+; CHECK: invalid retained nodes, expected DILocalVariable, DILabel, DIImportedEntity or DIType
 define void @invalid_retained_nodes_expected() !dbg !11 { ret void }
 !11 = distinct !DISubprogram(retainedNodes: !{!0})
 
diff --git a/llvm/test/DebugInfo/X86/local-type-as-template-parameter.ll b/llvm/test/DebugInfo/X86/local-type-as-template-parameter.ll
new file mode 100644
index 0000000000000..586574d3e1aaf
--- /dev/null
+++ b/llvm/test/DebugInfo/X86/local-type-as-template-parameter.ll
@@ -0,0 +1,161 @@
+; REQUIERES: system-linux
+; RUN: %llc_dwarf -mtriple=x86_64-linux -O0 -filetype=obj < %s              \
+; RUN:  | llvm-dwarfdump --show-children --name=foo - \
+; RUN:  | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s
+
+; The test ensures that AsmPrinter doesn't crashed compiling this.
+; It also demostrates misplacement for a local type (see PR55680 for details).
+
+; The test compiled from:
+
+; template<typename T>
+; struct A {
+;   A(T &in) : a(in) {}
+;   T a;
+; };
+;
+; __attribute__((always_inline))
+; void foo() {
+;   struct B { int i; };
+;   B objB;
+;   A<B> objA(objB);
+; }
+;
+; int main() {
+;   foo();
+; }
+
+; Concrete out-of-line tree of foo().
+; CHECK: DW_TAG_subprogram
+; CHECK:   DW_AT_abstract_origin {{.*}} "_Z3foov"
+
+; FIXME: 'struct B' should be in the abstract tree below, not here.
+; CHECK:   DW_TAG_structure_type
+; CHECK:     DW_AT_name	("B")
+; CHECK:     DW_TAG_member
+; CHECK:     NULL
+;
+; CHECK:   DW_TAG_variable
+; CHECK:     DW_AT_abstract_origin {{.*}} "objB"
+; CHECK:   DW_TAG_variable
+; CHECK:     DW_AT_abstract_origin {{.*}} "objA"
+
+; CHECK:   NULL
+
+; Abstract tree of foo().
+; CHECK: DW_TAG_subprogram
+; CHECK:   DW_AT_name	("foo")
+; CHECK:   DW_AT_inline	(DW_INL_inlined)
+
+; CHECK:   DW_TAG_variable
+; CHECK:     DW_AT_name	("objB")
+; CHECK:   DW_TAG_variable
+; CHECK:     DW_AT_name	("objA")
+
+; CHECK:   NULL
+
+; CHECK: DW_TAG_inlined_subroutine
+; CHECK:   DW_AT_abstract_origin {{.*}} "_Z3foov"
+; CHECK:   DW_TAG_variable
+; CHECK:     DW_AT_abstract_origin {{.*}} "objB"
+; CHECK:   DW_TAG_variable
+; CHECK:     DW_AT_abstract_origin {{.*}} "objA"
+; CHECK:   NULL
+
+%struct.B = type { i32 }
+%struct.A = type { %struct.B }
+
+define dso_local void @_Z3foov() !dbg !7 {
+entry:
+  %objB = alloca %struct.B, align 4
+  %objA = alloca %struct.A, align 4
+  call void @llvm.dbg.declare(metadata ptr %objB, metadata !30, metadata !DIExpression()), !dbg !31
+  call void @llvm.dbg.declare(metadata ptr %objA, metadata !32, metadata !DIExpression()), !dbg !33
+  call void @_ZN1AIZ3foovE1BEC2ERS0_(ptr noundef nonnull align 4 dereferenceable(4) %objA, ptr noundef nonnull align 4 dereferenceable(4) %objB), !dbg !33
+  ret void, !dbg !34
+}
+
+declare void @llvm.dbg.declare(metadata, metadata, metadata)
+
+define internal void @_ZN1AIZ3foovE1BEC2ERS0_(ptr noundef nonnull align 4 dereferenceable(4) %this, ptr noundef nonnull align 4 dereferenceable(4) %in) unnamed_addr align 2 !dbg !35 {
+entry:
+  %this.addr = alloca ptr, align 8
+  %in.addr = alloca ptr, align 8
+  store ptr %this, ptr %this.addr, align 8
+  call void @llvm.dbg.declare(metadata ptr %this.addr, metadata !36, metadata !DIExpression()), !dbg !38
+  store ptr %in, ptr %in.addr, align 8
+  call void @llvm.dbg.declare(metadata ptr %in.addr, metadata !39, metadata !DIExpression()), !dbg !40
+  %this1 = load ptr, ptr %this.addr, align 8
+  %a = getelementptr inbounds %struct.A, ptr %this1, i32 0, i32 0, !dbg !41
+  %0 = load ptr, ptr %in.addr, align 8, !dbg !42
+  call void @llvm.memcpy.p0.p0.i64(ptr align 4 %a, ptr align 4 %0, i64 4, i1 false), !dbg !41
+  ret void, !dbg !43
+}
+
+define dso_local noundef i32 @main() !dbg !44 {
+entry:
+  %objB.i = alloca %struct.B, align 4
+  %objA.i = alloca %struct.A, align 4
+  call void @llvm.dbg.declare(metadata ptr %objB.i, metadata !30, metadata !DIExpression()), !dbg !47
+  call void @llvm.dbg.declare(metadata ptr %objA.i, metadata !32, metadata !DIExpression()), !dbg !49
+  call void @_ZN1AIZ3foovE1BEC2ERS0_(ptr noundef nonnull align 4 dereferenceable(4) %objA.i, ptr noundef nonnull align 4 dereferenceable(4) %objB.i), !dbg !49
+  ret i32 0, !dbg !50
+}
+
+declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg)
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!22, !23, !24, !25, !26, !27, !28}
+!llvm.ident = !{!29}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 15.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, retainedTypes: !2, splitDebugInlining: false, nameTableKind: None)
+!1 = !DIFile(filename: "test.cpp", directory: "/", checksumkind: CSK_MD5, checksum: "aec7fd397e86f8655ef7f4bb4233b849")
+!2 = !{!3}
+!3 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A<B>", file: !1, line: 2, size: 32, flags: DIFlagTypePassByValue | DIFlagNonTrivial, elements: !4, templateParams: !20)
+!4 = !{!5, !15}
+!5 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !3, file: !1, line: 4, baseType: !6, size: 32)
+!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "B", scope: !7, file: !1, line: 9, size: 32, flags: DIFlagTypePassByValue, elements: !12)
+!7 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !1, file: !1, line: 8, type: !8, scopeLine: 8, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !11)
+!8 = !DISubroutineType(types: !9)
+!9 = !{null}
+!10 = !{}
+!11 = !{!6}
+!12 = !{!13}
+!13 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !6, file: !1, line: 9, baseType: !14, size: 32)
+!14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
+!15 = !DISubprogram(name: "A", scope: !3, file: !1, line: 3, type: !16, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagLocalToUnit)
+!16 = !DISubroutineType(types: !17)
+!17 = !{null, !18, !19}
+!18 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !3, size: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
+!19 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !6, size: 64)
+!20 = !{!21}
+!21 = !DITemplateTypeParameter(name: "T", type: !6)
+!22 = !{i32 7, !"Dwarf Version", i32 5}
+!23 = !{i32 2, !"Debug Info Version", i32 3}
+!24 = !{i32 1, !"wchar_size", i32 4}
+!25 = !{i32 7, !"PIC Level", i32 2}
+!26 = !{i32 7, !"PIE Level", i32 2}
+!27 = !{i32 7, !"uwtable", i32 2}
+!28 = !{i32 7, !"frame-pointer", i32 2}
+!29 = !{!"clang version 15.0.0"}
+!30 = !DILocalVariable(name: "objB", scope: !7, file: !1, line: 10, type: !6)
+!31 = !DILocation(line: 10, column: 5, scope: !7)
+!32 = !DILocalVariable(name: "objA", scope: !7, file: !1, line: 11, type: !3)
+!33 = !DILocation(line: 11, column: 8, scope: !7)
+!34 = !DILocation(line: 12, column: 1, scope: !7)
+!35 = distinct !DISubprogram(name: "A", linkageName: "_ZN1AIZ3foovE1BEC2ERS0_", scope: !3, file: !1, line: 3, type: !16, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !0, declaration: !15, retainedNodes: !10)
+!36 = !DILocalVariable(name: "this", arg: 1, scope: !35, type: !37, flags: DIFlagArtificial | DIFlagObjectPointer)
+!37 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !3, size: 64)
+!38 = !DILocation(line: 0, scope: !35)
+!39 = !DILocalVariable(name: "in", arg: 2, scope: !35, file: !1, line: 3, type: !19)
+!40 = !DILocation(line: 3, column: 8, scope: !35)
+!41 = !DILocation(line: 3, column: 14, scope: !35)
+!42 = !DILocation(line: 3, column: 16, scope: !35)
+!43 = !DILocation(line: 3, column: 21, scope: !35)
+!44 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 14, type: !45, scopeLine: 14, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !10)
+!45 = !DISubroutineType(types: !46)
+!46 = !{!14}
+!47 = !DILocation(line: 10, column: 5, scope: !7, inlinedAt: !48)
+!48 = distinct !DILocation(line: 15, column: 3, scope: !44)
+!49 = !DILocation(line: 11, column: 8, scope: !7, inlinedAt: !48)
+!50 = !DILocation(line: 16, column: 1, scope: !44)
diff --git a/llvm/test/DebugInfo/X86/set.ll b/llvm/test/DebugInfo/X86/set.ll
index 292c7c6e4a577..7dbec4520ed6c 100644
--- a/llvm/test/DebugInfo/X86/set.ll
+++ b/llvm/test/DebugInfo/X86/set.ll
@@ -68,11 +68,11 @@ attributes #1 = { nofree nosync nounwind readnone speculatable willreturn }
 !llvm.module.flags = !{!18, !19, !20}
 
 !0 = !{!"versions- cm3: d5.10.0 llvm: 9.0"}
-!1 = distinct !DICompileUnit(language: DW_LANG_Modula3, file: !2, producer: "cm3", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !3)
+!1 = distinct !DICompileUnit(language: DW_LANG_Modula3, file: !2, producer: "cm3", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug)
 !2 = !DIFile(filename: "Main.m3", directory: "/home/cm3/settest/src")
 !3 = !{!4}
 !4 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum", scope: !5, file: !2, line: 11, size: 8, align: 8, elements: !9)
-!5 = distinct !DISubprogram(name: "Test", linkageName: "Main__Test", scope: !2, file: !2, line: 11, type: !6, scopeLine: 11, spFlags: DISPFlagDefinition, unit: !1, retainedNodes: !8)
+!5 = distinct !DISubprogram(name: "Test", linkageName: "Main__Test", scope: !2, file: !2, line: 11, type: !6, scopeLine: 11, spFlags: DISPFlagDefinition, unit: !1, retainedNodes: !3)
 !6 = !DISubroutineType(types: !7)
 !7 = !{null}
 !8 = !{}
diff --git a/llvm/test/DebugInfo/Generic/split-dwarf-local-import.ll b/llvm/test/DebugInfo/X86/split-dwarf-local-import.ll
similarity index 98%
rename from llvm/test/DebugInfo/Generic/split-dwarf-local-import.ll
rename to llvm/test/DebugInfo/X86/split-dwarf-local-import.ll
index 1a41f3f348ca0..676ef23b679e8 100644
--- a/llvm/test/DebugInfo/Generic/split-dwarf-local-import.ll
+++ b/llvm/test/DebugInfo/X86/split-dwarf-local-import.ll
@@ -1,4 +1,3 @@
-; REQUIRES: x86-registered-target
 ; RUN: %llc_dwarf -O1 -filetype=obj -split-dwarf-file=%t.dwo < %s | llvm-dwarfdump -debug-info - | FileCheck %s --implicit-check-not "{{DW_TAG|NULL}}"
 
 ; CHECK-LABEL: debug_info contents
diff --git a/llvm/test/DebugInfo/Generic/split-dwarf-local-import2.ll b/llvm/test/DebugInfo/X86/split-dwarf-local-import2.ll
similarity index 98%
rename from llvm/test/DebugInfo/Generic/split-dwarf-local-import2.ll
rename to llvm/test/DebugInfo/X86/split-dwarf-local-import2.ll
index 0057f675f9b31..71f80992d1b04 100644
--- a/llvm/test/DebugInfo/Generic/split-dwarf-local-import2.ll
+++ b/llvm/test/DebugInfo/X86/split-dwarf-local-import2.ll
@@ -1,4 +1,3 @@
-; REQUIRES: x86-registered-target
 ; RUN: %llc_dwarf -split-dwarf-file=%t.dwo < %s | FileCheck %s
 
 ; Ensure function-local DW_TAG_imported_declaration get skipped if its parent subprogram was not emitted.
diff --git a/llvm/test/DebugInfo/Generic/split-dwarf-local-import3.ll b/llvm/test/DebugInfo/X86/split-dwarf-local-import3.ll
similarity index 100%
rename from llvm/test/DebugInfo/Generic/split-dwarf-local-import3.ll
rename to llvm/test/DebugInfo/X86/split-dwarf-local-import3.ll

From a132f5eb77a8102f979cb9fc8523f9445fc71645 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Mon, 19 Jun 2023 20:02:47 -0700
Subject: [PATCH 079/130] [BOLT] Fix a warning in release builds

This patch fixes:

  bolt/lib/Core/BinarySection.cpp:120:24: error: unused variable
  'Relocation' [-Werror,-Wunused-variable]
---
 bolt/lib/Core/BinarySection.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bolt/lib/Core/BinarySection.cpp b/bolt/lib/Core/BinarySection.cpp
index e3a1e79b98b15..6742302a98e67 100644
--- a/bolt/lib/Core/BinarySection.cpp
+++ b/bolt/lib/Core/BinarySection.cpp
@@ -117,6 +117,7 @@ void BinarySection::emitAsData(MCStreamer &Streamer,
       if (std::any_of(ROI, ROE, HasUndefSym))
         continue;
 
+#ifndef NDEBUG
       for (const auto &Relocation : make_range(ROI, ROE)) {
         LLVM_DEBUG(
             dbgs() << "BOLT-DEBUG: emitting relocation for symbol "
@@ -126,6 +127,7 @@ void BinarySection::emitAsData(MCStreamer &Streamer,
                    << " with size "
                    << Relocation::getSizeForType(Relocation.Type) << '\n');
       }
+#endif
 
       size_t RelocationSize = Relocation::emit(ROI, ROE, &Streamer);
       SectionOffset += RelocationSize;

From 3fa3cb408d8d0f1365b322262e501b6945f7ead9 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 19 Jun 2023 20:28:39 -0700
Subject: [PATCH 080/130] [XRay] Make llvm.xray.typedevent parameter type match
 __xray_typedevent

The Clang built-in function is void __xray_typedevent(size_t, const void *, size_t),
but the LLVM intrinsics has smaller integer types. Since we only allow
64-bit ELF/Mach-O targets, we can change llvm.xray.typedevent to
i64/ptr/i64.

This allows encoding more information and avoids i16 legalization for
many non-X86 targets.

fdrLoggingHandleTypedEvent only supports uint16_t event type.
---
 .../test/CodeGen/xray-always-emit-typedevent.cpp |  2 +-
 .../CodeGen/xray-instrumentation-bundles.cpp     |  4 ++--
 clang/test/CodeGen/xray-typedevent.cpp           |  8 ++++----
 compiler-rt/include/xray/xray_interface.h        |  5 +++--
 compiler-rt/lib/xray/xray_fdr_logging.cpp        |  9 +++++----
 compiler-rt/lib/xray/xray_interface.cpp          |  4 ++--
 .../xray/TestCases/Posix/typed-event-logging.cpp |  2 +-
 llvm/include/llvm/IR/Intrinsics.td               |  2 +-
 llvm/test/CodeGen/X86/xray-custom-log.ll         | 16 ++++++++--------
 9 files changed, 27 insertions(+), 25 deletions(-)

diff --git a/clang/test/CodeGen/xray-always-emit-typedevent.cpp b/clang/test/CodeGen/xray-always-emit-typedevent.cpp
index 75af669abe28d..745c63f6b3fa5 100644
--- a/clang/test/CodeGen/xray-always-emit-typedevent.cpp
+++ b/clang/test/CodeGen/xray-always-emit-typedevent.cpp
@@ -6,5 +6,5 @@
 [[clang::xray_never_instrument]] void neverInstrument() {
   static constexpr char kPhase[] = "never";
   __xray_typedevent(1, kPhase, 5);
-  // CHECK: call void @llvm.xray.typedevent(i16 {{.*}}, ptr{{.*}}, i32 5)
+  // CHECK: call void @llvm.xray.typedevent(i64 {{.*}}, ptr{{.*}}, i64 5)
 }
diff --git a/clang/test/CodeGen/xray-instrumentation-bundles.cpp b/clang/test/CodeGen/xray-instrumentation-bundles.cpp
index 83ca42ac6d9e9..45ea6e514e6cc 100644
--- a/clang/test/CodeGen/xray-instrumentation-bundles.cpp
+++ b/clang/test/CodeGen/xray-instrumentation-bundles.cpp
@@ -54,8 +54,8 @@
   __xray_typedevent(1, kPhase, 6);
   // CUSTOM: call void @llvm.xray.customevent(ptr{{.*}}, i32 6)
   // NOCUSTOM-NOT: call void @llvm.xray.customevent(ptr{{.*}}, i32 6)
-  // TYPED: call void @llvm.xray.typedevent(i16 {{.*}}, ptr{{.*}}, i32 6)
-  // NOTYPED-NOT: call void @llvm.xray.typedevent(i16 {{.*}}, ptr{{.*}}, i32 6)
+  // TYPED: call void @llvm.xray.typedevent(i64 {{.*}}, ptr{{.*}}, i64 6)
+  // NOTYPED-NOT: call void @llvm.xray.typedevent(
 }
 
 // FUNCTION: attributes #[[ALWAYSATTR]] = {{.*}} "function-instrument"="xray-always" {{.*}}
diff --git a/clang/test/CodeGen/xray-typedevent.cpp b/clang/test/CodeGen/xray-typedevent.cpp
index 57597970cd9e8..2ea4f7c75a7f5 100644
--- a/clang/test/CodeGen/xray-typedevent.cpp
+++ b/clang/test/CodeGen/xray-typedevent.cpp
@@ -7,7 +7,7 @@
   auto EventType = 1;
   static constexpr char kPhase[] = "instrument";
   __xray_typedevent(EventType, kPhase, 10);
-  // CHECK: call void @llvm.xray.typedevent(i16 {{.*}}, ptr{{.*}}, i32 10)
+  // CHECK: call void @llvm.xray.typedevent(i64 {{.*}}, ptr{{.*}}, i64 10)
 }
 
 // CHECK-LABEL: @_Z15neverInstrumentv
@@ -15,7 +15,7 @@
   auto EventType = 2;
   static constexpr char kPhase[] = "never";
   __xray_typedevent(EventType, kPhase, 5);
-  // CHECK-NOT: call void @llvm.xray.typedevent(i16 {{.*}}, ptr{{.*}}, i32 5)
+  // CHECK-NOT: call void @llvm.xray.typedevent(i64 {{.*}}, ptr{{.*}}, i64 5)
 }
 
 // CHECK-LABEL: @_Z21conditionalInstrumenti
@@ -29,6 +29,6 @@
   else
     __xray_typedevent(UntrueEventType, kUntrue, 6);
 
-  // CHECK: call void @llvm.xray.typedevent(i16 {{.*}}, ptr{{.*}}, i32 4)
-  // CHECK: call void @llvm.xray.typedevent(i16 {{.*}}, ptr{{.*}}, i32 6)
+  // CHECK: call void @llvm.xray.typedevent(i64 {{.*}}, ptr{{.*}}, i64 4)
+  // CHECK: call void @llvm.xray.typedevent(i64 {{.*}}, ptr{{.*}}, i64 6)
 }
diff --git a/compiler-rt/include/xray/xray_interface.h b/compiler-rt/include/xray/xray_interface.h
index 410515d2373af..727431c04e4f7 100644
--- a/compiler-rt/include/xray/xray_interface.h
+++ b/compiler-rt/include/xray/xray_interface.h
@@ -76,8 +76,9 @@ extern int __xray_remove_customevent_handler();
 
 /// Set a handler for xray typed event logging. The first parameter is a type
 /// identifier, the second is a payload, and the third is the payload size.
-extern int __xray_set_typedevent_handler(void (*entry)(uint16_t, const void *,
-                                                       std::size_t));
+/// NOTE: fdrLoggingHandleTypedEvent only supports uint16_t event type.
+extern int __xray_set_typedevent_handler(void (*entry)(size_t, const void *,
+                                                       size_t));
 
 /// Removes the currently set typed event handler.
 /// Returns 1 on success, 0 on error.
diff --git a/compiler-rt/lib/xray/xray_fdr_logging.cpp b/compiler-rt/lib/xray/xray_fdr_logging.cpp
index 799814f437f92..6056f8134553b 100644
--- a/compiler-rt/lib/xray/xray_fdr_logging.cpp
+++ b/compiler-rt/lib/xray/xray_fdr_logging.cpp
@@ -580,9 +580,9 @@ void fdrLoggingHandleCustomEvent(void *Event,
   TLD.Controller->customEvent(TSC, CPU, Event, ReducedEventSize);
 }
 
-void fdrLoggingHandleTypedEvent(
-    uint16_t EventType, const void *Event,
-    std::size_t EventSize) noexcept XRAY_NEVER_INSTRUMENT {
+void fdrLoggingHandleTypedEvent(size_t EventType, const void *Event,
+                                size_t EventSize) noexcept
+    XRAY_NEVER_INSTRUMENT {
   auto TC = getTimestamp();
   auto &TSC = TC.TSC;
   auto &CPU = TC.CPU;
@@ -607,7 +607,8 @@ void fdrLoggingHandleTypedEvent(
     return;
 
   int32_t ReducedEventSize = static_cast<int32_t>(EventSize);
-  TLD.Controller->typedEvent(TSC, CPU, EventType, Event, ReducedEventSize);
+  TLD.Controller->typedEvent(TSC, CPU, static_cast<uint16_t>(EventType), Event,
+                             ReducedEventSize);
 }
 
 XRayLogInitStatus fdrLoggingInit(size_t, size_t, void *Options,
diff --git a/compiler-rt/lib/xray/xray_interface.cpp b/compiler-rt/lib/xray/xray_interface.cpp
index 73e67618c9d5f..af9f11e166ac5 100644
--- a/compiler-rt/lib/xray/xray_interface.cpp
+++ b/compiler-rt/lib/xray/xray_interface.cpp
@@ -426,8 +426,8 @@ int __xray_set_customevent_handler(void (*entry)(void *, size_t))
   return 0;
 }
 
-int __xray_set_typedevent_handler(void (*entry)(
-    uint16_t, const void *, size_t)) XRAY_NEVER_INSTRUMENT {
+int __xray_set_typedevent_handler(void (*entry)(size_t, const void *,
+                                                size_t)) XRAY_NEVER_INSTRUMENT {
   if (atomic_load(&XRayInitialized,
                                memory_order_acquire)) {
     atomic_store(&__xray::XRayPatchedTypedEvent,
diff --git a/compiler-rt/test/xray/TestCases/Posix/typed-event-logging.cpp b/compiler-rt/test/xray/TestCases/Posix/typed-event-logging.cpp
index 20c99557acc48..103b11cfb0971 100644
--- a/compiler-rt/test/xray/TestCases/Posix/typed-event-logging.cpp
+++ b/compiler-rt/test/xray/TestCases/Posix/typed-event-logging.cpp
@@ -14,7 +14,7 @@
   printf("after calling the custom logging...\n");
 }
 
-static void myprinter(uint16_t type, const void *ptr, size_t size) {
+static void myprinter(size_t type, const void *ptr, size_t size) {
   assert(type == 42);
   printf("%.*s\n", static_cast<int>(size), static_cast<const char*>(ptr));
 }
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 97bfb8f22fda5..fbb2fa9a65646 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -2258,7 +2258,7 @@ def int_xray_customevent : Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty],
                                       ReadOnly<ArgIndex<0>>]>;
 // Typed event logging for x-ray.
 // Takes a numeric type tag, a pointer to a string and the length of the string.
-def int_xray_typedevent : Intrinsic<[], [llvm_i16_ty, llvm_ptr_ty, llvm_i32_ty],
+def int_xray_typedevent : Intrinsic<[], [llvm_i64_ty, llvm_ptr_ty, llvm_i64_ty],
                                         [IntrWriteMem, NoCapture<ArgIndex<1>>,
                                          ReadOnly<ArgIndex<1>>]>;
 //===----------------------------------------------------------------------===//
diff --git a/llvm/test/CodeGen/X86/xray-custom-log.ll b/llvm/test/CodeGen/X86/xray-custom-log.ll
index 28699e9bb4bcd..ea0d14b9cc7a2 100644
--- a/llvm/test/CodeGen/X86/xray-custom-log.ll
+++ b/llvm/test/CodeGen/X86/xray-custom-log.ll
@@ -37,13 +37,13 @@ define i32 @customevent() nounwind "function-instrument"="xray-always" !dbg !1 {
 
 define i32 @typedevent() nounwind "function-instrument"="xray-always" !dbg !2 {
     %eventptr = alloca i8
-    %eventsize = alloca i32
-    %eventtype = alloca i16
-    store i16 6, ptr %eventtype
-    %type = load i16, ptr %eventtype
-    store i32 3, ptr %eventsize
-    %val = load i32, ptr %eventsize
-    call void @llvm.xray.typedevent(i16 %type, ptr %eventptr, i32 %val), !dbg !9
+    %eventsize = alloca i64
+    %eventtype = alloca i64
+    store i64 6, ptr %eventtype
+    %type = load i64, ptr %eventtype
+    store i64 3, ptr %eventsize
+    %val = load i64, ptr %eventsize
+    call void @llvm.xray.typedevent(i64 %type, ptr %eventptr, i64 %val), !dbg !9
     ; CHECK-LABEL: Lxray_typed_event_sled_0:
     ; CHECK:       .byte 0xeb, 0x14
     ; CHECK-NEXT:  pushq %rdi
@@ -76,7 +76,7 @@ define i32 @typedevent() nounwind "function-instrument"="xray-always" !dbg !2 {
 ; CHECK:       .quad {{.*}}xray_typed_event_sled_0
 
 declare void @llvm.xray.customevent(ptr, i32)
-declare void @llvm.xray.typedevent(i16, ptr, i32)
+declare void @llvm.xray.typedevent(i64, ptr, i64)
 
 ;; Construct call site entries for PATCHABLE_EVENT_CALL.
 ; DBG:     DW_TAG_subprogram

From dafaa8463ef6991055c2fcee799757fa46965625 Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 19 Jun 2023 20:38:16 -0700
Subject: [PATCH 081/130] [XRay] Make llvm.xray.customevent parameter type
 match __xray_customevent

The intrinsic has a smaller integer type than the parameter type of
builtin-function/API. Fix this similar to commit 3fa3cb408d8d0f1365b322262e501b6945f7ead9.
---
 clang/test/CodeGen/xray-always-emit-customevent.cpp |  2 +-
 clang/test/CodeGen/xray-customevent.cpp             |  8 ++++----
 clang/test/CodeGen/xray-instrumentation-bundles.cpp |  4 ++--
 llvm/include/llvm/IR/Intrinsics.td                  |  2 +-
 llvm/test/CodeGen/X86/xray-custom-log.ll            | 10 +++++-----
 5 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/clang/test/CodeGen/xray-always-emit-customevent.cpp b/clang/test/CodeGen/xray-always-emit-customevent.cpp
index 63d4b91f81908..289111c90ff0a 100644
--- a/clang/test/CodeGen/xray-always-emit-customevent.cpp
+++ b/clang/test/CodeGen/xray-always-emit-customevent.cpp
@@ -6,5 +6,5 @@
 [[clang::xray_never_instrument]] void neverInstrument() {
   static constexpr char kPhase[] = "never";
   __xray_customevent(kPhase, 5);
-  // CHECK: call void @llvm.xray.customevent(ptr{{.*}}, i32 5)
+  // CHECK: call void @llvm.xray.customevent(ptr{{.*}}, i64 5)
 }
diff --git a/clang/test/CodeGen/xray-customevent.cpp b/clang/test/CodeGen/xray-customevent.cpp
index efaa26d633109..c64cbbd7c4236 100644
--- a/clang/test/CodeGen/xray-customevent.cpp
+++ b/clang/test/CodeGen/xray-customevent.cpp
@@ -4,14 +4,14 @@
 [[clang::xray_always_instrument]] void alwaysInstrument() {
   static constexpr char kPhase[] = "instrument";
   __xray_customevent(kPhase, 10);
-  // CHECK: call void @llvm.xray.customevent(ptr{{.*}}, i32 10)
+  // CHECK: call void @llvm.xray.customevent(ptr{{.*}}, i64 10)
 }
 
 // CHECK-LABEL: @_Z15neverInstrumentv
 [[clang::xray_never_instrument]] void neverInstrument() {
   static constexpr char kPhase[] = "never";
   __xray_customevent(kPhase, 5);
-  // CHECK-NOT: call void @llvm.xray.customevent(ptr{{.*}}, i32 5)
+  // CHECK-NOT: call void @llvm.xray.customevent(
 }
 
 // CHECK-LABEL: @_Z21conditionalInstrumenti
@@ -23,6 +23,6 @@
   else
     __xray_customevent(kUntrue, 6);
 
-  // CHECK: call void @llvm.xray.customevent(ptr{{.*}}, i32 4)
-  // CHECK: call void @llvm.xray.customevent(ptr{{.*}}, i32 6)
+  // CHECK: call void @llvm.xray.customevent(ptr{{.*}}, i64 4)
+  // CHECK: call void @llvm.xray.customevent(ptr{{.*}}, i64 6)
 }
diff --git a/clang/test/CodeGen/xray-instrumentation-bundles.cpp b/clang/test/CodeGen/xray-instrumentation-bundles.cpp
index 45ea6e514e6cc..7b9ccb25a3b52 100644
--- a/clang/test/CodeGen/xray-instrumentation-bundles.cpp
+++ b/clang/test/CodeGen/xray-instrumentation-bundles.cpp
@@ -52,8 +52,8 @@
   static constexpr char kPhase[] = "always";
   __xray_customevent(kPhase, 6);
   __xray_typedevent(1, kPhase, 6);
-  // CUSTOM: call void @llvm.xray.customevent(ptr{{.*}}, i32 6)
-  // NOCUSTOM-NOT: call void @llvm.xray.customevent(ptr{{.*}}, i32 6)
+  // CUSTOM: call void @llvm.xray.customevent(ptr {{.*}}, i64 6)
+  // NOCUSTOM-NOT: call void @llvm.xray.customevent(
   // TYPED: call void @llvm.xray.typedevent(i64 {{.*}}, ptr{{.*}}, i64 6)
   // NOTYPED-NOT: call void @llvm.xray.typedevent(
 }
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index fbb2fa9a65646..c03eafea2438b 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -2253,7 +2253,7 @@ def int_hwasan_check_memaccess_shortgranules :
 //===----------------------------------------------------------------------===//
 // Custom event logging for x-ray.
 // Takes a pointer to a string and the length of the string.
-def int_xray_customevent : Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty],
+def int_xray_customevent : Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty],
                                      [IntrWriteMem, NoCapture<ArgIndex<0>>,
                                       ReadOnly<ArgIndex<0>>]>;
 // Typed event logging for x-ray.
diff --git a/llvm/test/CodeGen/X86/xray-custom-log.ll b/llvm/test/CodeGen/X86/xray-custom-log.ll
index ea0d14b9cc7a2..c41aa1e986815 100644
--- a/llvm/test/CodeGen/X86/xray-custom-log.ll
+++ b/llvm/test/CodeGen/X86/xray-custom-log.ll
@@ -6,10 +6,10 @@
 
 define i32 @customevent() nounwind "function-instrument"="xray-always" !dbg !1 {
     %eventptr = alloca i8
-    %eventsize = alloca i32
-    store i32 3, ptr %eventsize
-    %val = load i32, ptr %eventsize
-    call void @llvm.xray.customevent(ptr %eventptr, i32 %val), !dbg !8
+    %eventsize = alloca i64
+    store i64 3, ptr %eventsize
+    %val = load i64, ptr %eventsize
+    call void @llvm.xray.customevent(ptr %eventptr, i64 %val), !dbg !8
     ; CHECK-LABEL: Lxray_event_sled_0:
     ; CHECK:       .byte 0xeb, 0x0f
     ; CHECK-NEXT:  pushq %rdi
@@ -75,7 +75,7 @@ define i32 @typedevent() nounwind "function-instrument"="xray-always" !dbg !2 {
 ; CHECK-LABEL: Lxray_sleds_start1:
 ; CHECK:       .quad {{.*}}xray_typed_event_sled_0
 
-declare void @llvm.xray.customevent(ptr, i32)
+declare void @llvm.xray.customevent(ptr, i64)
 declare void @llvm.xray.typedevent(i64, ptr, i64)
 
 ;; Construct call site entries for PATCHABLE_EVENT_CALL.

From 516e32678d87fea013aa972444b987d95eaef8aa Mon Sep 17 00:00:00 2001
From: Bing1 Yu <bing1.yu@intel.com>
Date: Mon, 19 Jun 2023 15:57:01 +0800
Subject: [PATCH 082/130] [X86][AMX] set Stride to Tile's Col when doing
 combine amxcast and store into tilestore

%tile = call x86_amx @llvm.x86.tileloadd64.internal(i16 8, i16 32, i8* %src_ptr, i64 64)
%vec = call <256 x i8> @llvm.x86.cast.tile.to.vector.v256i8(x86_amx...%tile)
store <256 x i8> %vec, <256 x i8>* %dst_ptr, align 256
=>
%tile = call x86_amx @llvm.x86.tileloadd64.internal(i16 8, i16 32, i8* %src_ptr, i64 64)
%stride = sext i16 32 to i64
call void @llvm.x86.tilestored64.internal(i16 8, i16 32, i8* %dst_ptr, i64 32, x86_amx %tile)

Reviewed By: LuoYuanke

Differential Revision: https://reviews.llvm.org/D153002
---
 llvm/include/llvm/IR/IntrinsicsX86.td         |  2 +
 llvm/lib/Target/X86/X86LowerAMXType.cpp       | 18 ++--
 llvm/test/CodeGen/X86/AMX/amx-combine.ll      | 43 +++++++--
 .../X86/AMX/lat-transform-amx-bitcast.ll      | 89 ++++++++++++-------
 4 files changed, 109 insertions(+), 43 deletions(-)

diff --git a/llvm/include/llvm/IR/IntrinsicsX86.td b/llvm/include/llvm/IR/IntrinsicsX86.td
index d8d8bc59987e1..ab735daee93dd 100644
--- a/llvm/include/llvm/IR/IntrinsicsX86.td
+++ b/llvm/include/llvm/IR/IntrinsicsX86.td
@@ -5420,8 +5420,10 @@ let TargetPrefix = "x86" in {
                         [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty,
                          llvm_x86amx_ty, llvm_x86amx_ty,
                          llvm_x86amx_ty], []>;
+  // the vector size can be smaller than AMX register size (1024 bytes)
   def int_x86_cast_vector_to_tile:
       DefaultAttrsIntrinsic<[llvm_x86amx_ty], [llvm_anyvector_ty], [IntrNoMem]>;
+  // the vector size can be smaller than AMX register size (1024 bytes)
   def int_x86_cast_tile_to_vector:
       DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_x86amx_ty], [IntrNoMem]>;
 
diff --git a/llvm/lib/Target/X86/X86LowerAMXType.cpp b/llvm/lib/Target/X86/X86LowerAMXType.cpp
index fa9d367cf7628..0416f0f0d2ec9 100644
--- a/llvm/lib/Target/X86/X86LowerAMXType.cpp
+++ b/llvm/lib/Target/X86/X86LowerAMXType.cpp
@@ -934,9 +934,8 @@ bool X86LowerAMXCast::combineCastStore(IntrinsicInst *Cast, StoreInst *ST) {
   Value *Row = II->getOperand(0);
   Value *Col = II->getOperand(1);
   IRBuilder<> Builder(ST);
-  // Use the maximum column as stride. It must be the same with load
-  // stride.
-  Value *Stride = Builder.getInt64(64);
+  // Stride should be equal to col(measured by bytes)
+  Value *Stride = Builder.CreateSExt(Col, Builder.getInt64Ty());
   Value *I8Ptr =
       Builder.CreateBitCast(ST->getOperand(1), Builder.getInt8PtrTy());
   std::array<Value *, 5> Args = {Row, Col, I8Ptr, Stride, Tile};
@@ -962,8 +961,8 @@ bool X86LowerAMXCast::combineLoadCast(IntrinsicInst *Cast, LoadInst *LD) {
     return false;
   std::tie(Row, Col) = getShape(II, OpNo);
   IRBuilder<> Builder(LD);
-  // Use the maximun column as stride.
-  Value *Stride = Builder.getInt64(64);
+  // Stride should be equal to col(measured by bytes)
+  Value *Stride = Builder.CreateSExt(Col, Builder.getInt64Ty());
   Value *I8Ptr;
 
   // To save compiling time, we create doninator tree when it is really
@@ -1089,8 +1088,14 @@ bool X86LowerAMXCast::combineAMXcast(TargetLibraryInfo *TLI) {
 
   EraseInst(Vec2TileInsts);
   EraseInst(Tile2VecInsts);
+  LLVM_DEBUG(dbgs() << "[LowerAMXTYpe][combineAMXcast] IR dump after combine "
+                       "Vec2Tile and Tile2Vec:\n";
+             Func.dump());
   Change |= combineLdSt(LiveCasts);
   EraseInst(LiveCasts);
+  LLVM_DEBUG(dbgs() << "[LowerAMXTYpe][combineAMXcast] IR dump after combine "
+                       "AMXCast and load/store:\n";
+             Func.dump());
 
   // Handle the A->B->A cast, and there is an intervening PHI node.
   for (BasicBlock &BB : Func) {
@@ -1118,6 +1123,9 @@ bool X86LowerAMXCast::combineAMXcast(TargetLibraryInfo *TLI) {
     Instruction *I = DeadInst.pop_back_val();
     Change |= DCEInstruction(I, DeadInst, TLI);
   }
+  LLVM_DEBUG(dbgs() << "[LowerAMXTYpe][combineAMXcast] IR dump after "
+                       "optimizeAMXCastFromPhi:\n";
+             Func.dump());
   return Change;
 }
 
diff --git a/llvm/test/CodeGen/X86/AMX/amx-combine.ll b/llvm/test/CodeGen/X86/AMX/amx-combine.ll
index 0dc1f4b159838..fe21d64eb7a3a 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-combine.ll
+++ b/llvm/test/CodeGen/X86/AMX/amx-combine.ll
@@ -97,18 +97,21 @@ define void @test_tile_dpbssd(ptr byval(%struct.__tile1024i_str) align 64 %a, pt
 ; CHECK-NEXT:    [[B_ROW_PTR:%.*]] = getelementptr inbounds i8, ptr [[B:%.*]], i64 2
 ; CHECK-NEXT:    [[B_ROW:%.*]] = load i16, ptr [[B_ROW_PTR]], align 2
 ; CHECK-NEXT:    [[B_TILE_PTR:%.*]] = getelementptr inbounds i8, ptr [[B]], i64 64
+; CHECK-NEXT:    [[TMP1:%.*]] = sext i16 [[B_ROW]] to i64
 ; CHECK-NEXT:    [[B_TILE:%.*]] = load <256 x i32>, ptr [[B_TILE_PTR]], align 64
 ; CHECK-NEXT:    store <256 x i32> [[B_TILE]], ptr [[TMP0]], align 1024
 ; CHECK-NEXT:    [[A_ROW:%.*]] = load i16, ptr [[A:%.*]], align 64
 ; CHECK-NEXT:    [[A_COL_PTR:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 2
 ; CHECK-NEXT:    [[A_COL:%.*]] = load i16, ptr [[A_COL_PTR]], align 2
-; CHECK-NEXT:    [[TMP1:%.*]] = udiv i16 [[A_COL]], 4
+; CHECK-NEXT:    [[TMP2:%.*]] = udiv i16 [[A_COL]], 4
 ; CHECK-NEXT:    [[A_TILE_PTR:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 64
-; CHECK-NEXT:    [[TMP2:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[A_ROW]], i16 [[A_COL]], ptr [[A_TILE_PTR]], i64 64)
+; CHECK-NEXT:    [[TMP3:%.*]] = sext i16 [[A_COL]] to i64
+; CHECK-NEXT:    [[TMP4:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[A_ROW]], i16 [[A_COL]], ptr [[A_TILE_PTR]], i64 [[TMP3]])
 ; CHECK-NEXT:    [[C_TILE_PTR:%.*]] = getelementptr inbounds [[STRUCT___TILE1024I_STR:%.*]], ptr [[C:%.*]], i64 0, i32 3
-; CHECK-NEXT:    [[TMP3:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[A_ROW]], i16 [[B_ROW]], ptr [[C_TILE_PTR]], i64 64)
-; CHECK-NEXT:    [[TMP4:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[TMP1]], i16 [[B_ROW]], ptr [[TMP0]], i64 64)
-; CHECK-NEXT:    [[RES:%.*]] = tail call x86_amx @llvm.x86.tdpbssd.internal(i16 [[A_ROW]], i16 [[B_ROW]], i16 [[A_COL]], x86_amx [[TMP3]], x86_amx [[TMP2]], x86_amx [[TMP4]])
+; CHECK-NEXT:    [[TMP5:%.*]] = sext i16 [[B_ROW]] to i64
+; CHECK-NEXT:    [[TMP6:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[A_ROW]], i16 [[B_ROW]], ptr [[C_TILE_PTR]], i64 [[TMP5]])
+; CHECK-NEXT:    [[TMP7:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[TMP2]], i16 [[B_ROW]], ptr [[TMP0]], i64 [[TMP1]])
+; CHECK-NEXT:    [[RES:%.*]] = tail call x86_amx @llvm.x86.tdpbssd.internal(i16 [[A_ROW]], i16 [[B_ROW]], i16 [[A_COL]], x86_amx [[TMP6]], x86_amx [[TMP4]], x86_amx [[TMP7]])
 ; CHECK-NEXT:    ret void
 ;
 entry:
@@ -130,8 +133,38 @@ entry:
   ret void
 }
 
+define void @combine_v256i8amcast_with_store(i8* %src_ptr, <256 x i8>* %dst_ptr) {
+; CHECK-LABEL: @combine_v256i8amcast_with_store(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TILE:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 8, i16 32, ptr [[SRC_PTR:%.*]], i64 64)
+; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 8, i16 32, ptr [[DST_PTR:%.*]], i64 32, x86_amx [[TILE]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %tile = call x86_amx @llvm.x86.tileloadd64.internal(i16 8, i16 32, i8* %src_ptr, i64 64)
+  %vec = call <256 x i8> @llvm.x86.cast.tile.to.vector.v256i8(x86_amx %tile)
+  store <256 x i8> %vec, <256 x i8>* %dst_ptr, align 256
+  ret void
+}
+
+define void @combine_v256i8amcast_with_load(i8* %src_ptr, <256 x i8>* %dst_ptr) {
+; CHECK-LABEL: @combine_v256i8amcast_with_load(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 8, i16 32, ptr [[SRC_PTR:%.*]], i64 32)
+; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 8, i16 32, ptr [[DST_PTR:%.*]], i64 32, x86_amx [[TMP0]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %vec = load <256 x i8>, ptr %src_ptr, align 256
+  %tile = call x86_amx @llvm.x86.cast.vector.to.tile.v256i8(<256 x i8> %vec)
+  call void @llvm.x86.tilestored64.internal(i16 8, i16 32, <256 x i8>* %dst_ptr, i64 32, x86_amx %tile)
+  ret void
+}
+
 declare x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32>)
 declare <256 x i32> @llvm.x86.cast.tile.to.vector.v256i32(x86_amx)
+declare x86_amx @llvm.x86.cast.vector.to.tile.v256i8(<256 x i8>)
+declare <256 x i8> @llvm.x86.cast.tile.to.vector.v256i8(x86_amx)
 declare x86_amx @llvm.x86.tilezero.internal(i16, i16)
 declare x86_amx @llvm.x86.tileloadd64.internal(i16, i16, ptr, i64)
 declare void @llvm.x86.tilestored64.internal(i16, i16, ptr, i64, x86_amx)
diff --git a/llvm/test/CodeGen/X86/AMX/lat-transform-amx-bitcast.ll b/llvm/test/CodeGen/X86/AMX/lat-transform-amx-bitcast.ll
index dc3b15e7c5503..391727d54a03a 100644
--- a/llvm/test/CodeGen/X86/AMX/lat-transform-amx-bitcast.ll
+++ b/llvm/test/CodeGen/X86/AMX/lat-transform-amx-bitcast.ll
@@ -74,7 +74,8 @@ define dso_local <256 x i32> @test_amx_bitcast_store(ptr %out, i16 %m, i16 %n, p
 ; CHECK-NEXT:    [[TMP1:%.*]] = sext i16 [[M]] to i64
 ; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 [[M]], i16 [[M]], ptr [[TMP0]], i64 [[TMP1]], x86_amx [[T1]])
 ; CHECK-NEXT:    [[TMP2:%.*]] = load <256 x i32>, ptr [[TMP0]], align 1024
-; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 [[M]], i16 [[M]], ptr [[OUT:%.*]], i64 64, x86_amx [[T1]])
+; CHECK-NEXT:    [[TMP3:%.*]] = sext i16 [[M]] to i64
+; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 [[M]], i16 [[M]], ptr [[OUT:%.*]], i64 [[TMP3]], x86_amx [[T1]])
 ; CHECK-NEXT:    ret <256 x i32> [[TMP2]]
 ;
 entry:
@@ -129,7 +130,8 @@ define dso_local void @__tile_loadd(ptr nocapture %0, ptr %1, i64 %2) local_unna
 ; CHECK-NEXT:    [[TMP8:%.*]] = ashr exact i64 [[TMP7]], 32
 ; CHECK-NEXT:    [[TMP9:%.*]] = tail call x86_amx @llvm.x86.tileloadd64.internal(i16 [[TMP4]], i16 [[TMP6]], ptr [[TMP1:%.*]], i64 [[TMP8]])
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [[STRUCT___TILE_STR]], ptr [[TMP0]], i64 0, i32 2
-; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 [[TMP4]], i16 [[TMP6]], ptr [[TMP10]], i64 64, x86_amx [[TMP9]])
+; CHECK-NEXT:    [[TMP11:%.*]] = sext i16 [[TMP6]] to i64
+; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 [[TMP4]], i16 [[TMP6]], ptr [[TMP10]], i64 [[TMP11]], x86_amx [[TMP9]])
 ; CHECK-NEXT:    ret void
 ;
   %4 = load i16, ptr %0, align 64
@@ -153,13 +155,17 @@ define dso_local void @__tile_dpbssd(ptr nocapture %0, ptr nocapture readonly by
 ; CHECK-NEXT:    [[TMP8:%.*]] = load i16, ptr [[TMP7]], align 2
 ; CHECK-NEXT:    [[TMP9:%.*]] = udiv i16 [[TMP8]], 4
 ; CHECK-NEXT:    [[TMP10:%.*]] = getelementptr inbounds [[STRUCT___TILE_STR]], ptr [[TMP0:%.*]], i64 0, i32 2
-; CHECK-NEXT:    [[TMP11:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[TMP4]], i16 [[TMP6]], ptr [[TMP10]], i64 64)
-; CHECK-NEXT:    [[TMP12:%.*]] = getelementptr inbounds [[STRUCT___TILE_STR]], ptr [[TMP1]], i64 0, i32 2
-; CHECK-NEXT:    [[TMP13:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[TMP4]], i16 [[TMP8]], ptr [[TMP12]], i64 64)
-; CHECK-NEXT:    [[TMP14:%.*]] = getelementptr inbounds [[STRUCT___TILE_STR]], ptr [[TMP2]], i64 0, i32 2
-; CHECK-NEXT:    [[TMP15:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[TMP9]], i16 [[TMP6]], ptr [[TMP14]], i64 64)
-; CHECK-NEXT:    [[TMP16:%.*]] = tail call x86_amx @llvm.x86.tdpbssd.internal(i16 [[TMP4]], i16 [[TMP6]], i16 [[TMP8]], x86_amx [[TMP11]], x86_amx [[TMP13]], x86_amx [[TMP15]])
-; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 [[TMP4]], i16 [[TMP6]], ptr [[TMP10]], i64 64, x86_amx [[TMP16]])
+; CHECK-NEXT:    [[TMP11:%.*]] = sext i16 [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP12:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[TMP4]], i16 [[TMP6]], ptr [[TMP10]], i64 [[TMP11]])
+; CHECK-NEXT:    [[TMP13:%.*]] = getelementptr inbounds [[STRUCT___TILE_STR]], ptr [[TMP1]], i64 0, i32 2
+; CHECK-NEXT:    [[TMP14:%.*]] = sext i16 [[TMP8]] to i64
+; CHECK-NEXT:    [[TMP15:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[TMP4]], i16 [[TMP8]], ptr [[TMP13]], i64 [[TMP14]])
+; CHECK-NEXT:    [[TMP16:%.*]] = getelementptr inbounds [[STRUCT___TILE_STR]], ptr [[TMP2]], i64 0, i32 2
+; CHECK-NEXT:    [[TMP17:%.*]] = sext i16 [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP18:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[TMP9]], i16 [[TMP6]], ptr [[TMP16]], i64 [[TMP17]])
+; CHECK-NEXT:    [[TMP19:%.*]] = tail call x86_amx @llvm.x86.tdpbssd.internal(i16 [[TMP4]], i16 [[TMP6]], i16 [[TMP8]], x86_amx [[TMP12]], x86_amx [[TMP15]], x86_amx [[TMP18]])
+; CHECK-NEXT:    [[TMP20:%.*]] = sext i16 [[TMP6]] to i64
+; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 [[TMP4]], i16 [[TMP6]], ptr [[TMP10]], i64 [[TMP20]], x86_amx [[TMP19]])
 ; CHECK-NEXT:    ret void
 ;
   %4 = load i16, ptr %1, align 64
@@ -185,11 +191,15 @@ define dso_local void @__tile_dpbssd(ptr nocapture %0, ptr nocapture readonly by
 define dso_local void @__tile_dpbsud(i16 %m, i16 %n, i16 %k, ptr %pc, ptr %pa, ptr %pb) {
 ; CHECK-LABEL: @__tile_dpbsud(
 ; CHECK-NEXT:    [[TMP1:%.*]] = udiv i16 [[K:%.*]], 4
-; CHECK-NEXT:    [[TMP2:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[M:%.*]], i16 [[K]], ptr [[PA:%.*]], i64 64)
-; CHECK-NEXT:    [[TMP3:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[TMP1]], i16 [[N:%.*]], ptr [[PB:%.*]], i64 64)
-; CHECK-NEXT:    [[TMP4:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[M]], i16 [[N]], ptr [[PC:%.*]], i64 64)
-; CHECK-NEXT:    [[T6:%.*]] = tail call x86_amx @llvm.x86.tdpbsud.internal(i16 [[M]], i16 [[N]], i16 [[K]], x86_amx [[TMP4]], x86_amx [[TMP2]], x86_amx [[TMP3]])
-; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 [[M]], i16 [[N]], ptr [[PC]], i64 64, x86_amx [[T6]])
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i16 [[K]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[M:%.*]], i16 [[K]], ptr [[PA:%.*]], i64 [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = sext i16 [[N:%.*]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[TMP1]], i16 [[N]], ptr [[PB:%.*]], i64 [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i16 [[N]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[M]], i16 [[N]], ptr [[PC:%.*]], i64 [[TMP6]])
+; CHECK-NEXT:    [[T6:%.*]] = tail call x86_amx @llvm.x86.tdpbsud.internal(i16 [[M]], i16 [[N]], i16 [[K]], x86_amx [[TMP7]], x86_amx [[TMP3]], x86_amx [[TMP5]])
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i16 [[N]] to i64
+; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 [[M]], i16 [[N]], ptr [[PC]], i64 [[TMP8]], x86_amx [[T6]])
 ; CHECK-NEXT:    ret void
 ;
   %t0 = load <256 x i32>, ptr %pa, align 64
@@ -207,11 +217,15 @@ define dso_local void @__tile_dpbsud(i16 %m, i16 %n, i16 %k, ptr %pc, ptr %pa, p
 define dso_local void @__tile_dpbusd(i16 %m, i16 %n, i16 %k, ptr %pc, ptr %pa, ptr %pb) {
 ; CHECK-LABEL: @__tile_dpbusd(
 ; CHECK-NEXT:    [[TMP1:%.*]] = udiv i16 [[K:%.*]], 4
-; CHECK-NEXT:    [[TMP2:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[M:%.*]], i16 [[K]], ptr [[PA:%.*]], i64 64)
-; CHECK-NEXT:    [[TMP3:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[TMP1]], i16 [[N:%.*]], ptr [[PB:%.*]], i64 64)
-; CHECK-NEXT:    [[TMP4:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[M]], i16 [[N]], ptr [[PC:%.*]], i64 64)
-; CHECK-NEXT:    [[T6:%.*]] = tail call x86_amx @llvm.x86.tdpbusd.internal(i16 [[M]], i16 [[N]], i16 [[K]], x86_amx [[TMP4]], x86_amx [[TMP2]], x86_amx [[TMP3]])
-; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 [[M]], i16 [[N]], ptr [[PC]], i64 64, x86_amx [[T6]])
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i16 [[K]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[M:%.*]], i16 [[K]], ptr [[PA:%.*]], i64 [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = sext i16 [[N:%.*]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[TMP1]], i16 [[N]], ptr [[PB:%.*]], i64 [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i16 [[N]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[M]], i16 [[N]], ptr [[PC:%.*]], i64 [[TMP6]])
+; CHECK-NEXT:    [[T6:%.*]] = tail call x86_amx @llvm.x86.tdpbusd.internal(i16 [[M]], i16 [[N]], i16 [[K]], x86_amx [[TMP7]], x86_amx [[TMP3]], x86_amx [[TMP5]])
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i16 [[N]] to i64
+; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 [[M]], i16 [[N]], ptr [[PC]], i64 [[TMP8]], x86_amx [[T6]])
 ; CHECK-NEXT:    ret void
 ;
   %t0 = load <256 x i32>, ptr %pa, align 64
@@ -229,11 +243,15 @@ define dso_local void @__tile_dpbusd(i16 %m, i16 %n, i16 %k, ptr %pc, ptr %pa, p
 define dso_local void @__tile_dpbuud(i16 %m, i16 %n, i16 %k, ptr %pc, ptr %pa, ptr %pb) {
 ; CHECK-LABEL: @__tile_dpbuud(
 ; CHECK-NEXT:    [[TMP1:%.*]] = udiv i16 [[K:%.*]], 4
-; CHECK-NEXT:    [[TMP2:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[M:%.*]], i16 [[K]], ptr [[PA:%.*]], i64 64)
-; CHECK-NEXT:    [[TMP3:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[TMP1]], i16 [[N:%.*]], ptr [[PB:%.*]], i64 64)
-; CHECK-NEXT:    [[TMP4:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[M]], i16 [[N]], ptr [[PC:%.*]], i64 64)
-; CHECK-NEXT:    [[T6:%.*]] = tail call x86_amx @llvm.x86.tdpbuud.internal(i16 [[M]], i16 [[N]], i16 [[K]], x86_amx [[TMP4]], x86_amx [[TMP2]], x86_amx [[TMP3]])
-; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 [[M]], i16 [[N]], ptr [[PC]], i64 64, x86_amx [[T6]])
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i16 [[K]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[M:%.*]], i16 [[K]], ptr [[PA:%.*]], i64 [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = sext i16 [[N:%.*]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[TMP1]], i16 [[N]], ptr [[PB:%.*]], i64 [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i16 [[N]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[M]], i16 [[N]], ptr [[PC:%.*]], i64 [[TMP6]])
+; CHECK-NEXT:    [[T6:%.*]] = tail call x86_amx @llvm.x86.tdpbuud.internal(i16 [[M]], i16 [[N]], i16 [[K]], x86_amx [[TMP7]], x86_amx [[TMP3]], x86_amx [[TMP5]])
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i16 [[N]] to i64
+; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 [[M]], i16 [[N]], ptr [[PC]], i64 [[TMP8]], x86_amx [[T6]])
 ; CHECK-NEXT:    ret void
 ;
   %t0 = load <256 x i32>, ptr %pa, align 64
@@ -251,11 +269,15 @@ define dso_local void @__tile_dpbuud(i16 %m, i16 %n, i16 %k, ptr %pc, ptr %pa, p
 define dso_local void @__tile_dpbf16ps(i16 %m, i16 %n, i16 %k, ptr %pc, ptr %pa, ptr %pb) {
 ; CHECK-LABEL: @__tile_dpbf16ps(
 ; CHECK-NEXT:    [[TMP1:%.*]] = udiv i16 [[K:%.*]], 4
-; CHECK-NEXT:    [[TMP2:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[M:%.*]], i16 [[K]], ptr [[PA:%.*]], i64 64)
-; CHECK-NEXT:    [[TMP3:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[TMP1]], i16 [[N:%.*]], ptr [[PB:%.*]], i64 64)
-; CHECK-NEXT:    [[TMP4:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[M]], i16 [[N]], ptr [[PC:%.*]], i64 64)
-; CHECK-NEXT:    [[T6:%.*]] = tail call x86_amx @llvm.x86.tdpbf16ps.internal(i16 [[M]], i16 [[N]], i16 [[K]], x86_amx [[TMP4]], x86_amx [[TMP2]], x86_amx [[TMP3]])
-; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 [[M]], i16 [[N]], ptr [[PC]], i64 64, x86_amx [[T6]])
+; CHECK-NEXT:    [[TMP2:%.*]] = sext i16 [[K]] to i64
+; CHECK-NEXT:    [[TMP3:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[M:%.*]], i16 [[K]], ptr [[PA:%.*]], i64 [[TMP2]])
+; CHECK-NEXT:    [[TMP4:%.*]] = sext i16 [[N:%.*]] to i64
+; CHECK-NEXT:    [[TMP5:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[TMP1]], i16 [[N]], ptr [[PB:%.*]], i64 [[TMP4]])
+; CHECK-NEXT:    [[TMP6:%.*]] = sext i16 [[N]] to i64
+; CHECK-NEXT:    [[TMP7:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[M]], i16 [[N]], ptr [[PC:%.*]], i64 [[TMP6]])
+; CHECK-NEXT:    [[T6:%.*]] = tail call x86_amx @llvm.x86.tdpbf16ps.internal(i16 [[M]], i16 [[N]], i16 [[K]], x86_amx [[TMP7]], x86_amx [[TMP3]], x86_amx [[TMP5]])
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i16 [[N]] to i64
+; CHECK-NEXT:    call void @llvm.x86.tilestored64.internal(i16 [[M]], i16 [[N]], ptr [[PC]], i64 [[TMP8]], x86_amx [[T6]])
 ; CHECK-NEXT:    ret void
 ;
   %t0 = load <256 x i32>, ptr %pa, align 64
@@ -276,10 +298,11 @@ define dso_local void @__tile_stored(ptr %0, i64 %1, ptr nocapture readonly byva
 ; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds [[STRUCT___TILE_STR:%.*]], ptr [[TMP2]], i64 0, i32 1
 ; CHECK-NEXT:    [[TMP6:%.*]] = load i16, ptr [[TMP5]], align 2
 ; CHECK-NEXT:    [[TMP7:%.*]] = getelementptr inbounds [[STRUCT___TILE_STR]], ptr [[TMP2]], i64 0, i32 2
-; CHECK-NEXT:    [[TMP8:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[TMP4]], i16 [[TMP6]], ptr [[TMP7]], i64 64)
-; CHECK-NEXT:    [[TMP9:%.*]] = shl i64 [[TMP1:%.*]], 32
-; CHECK-NEXT:    [[TMP10:%.*]] = ashr exact i64 [[TMP9]], 32
-; CHECK-NEXT:    tail call void @llvm.x86.tilestored64.internal(i16 [[TMP4]], i16 [[TMP6]], ptr [[TMP0:%.*]], i64 [[TMP10]], x86_amx [[TMP8]])
+; CHECK-NEXT:    [[TMP8:%.*]] = sext i16 [[TMP6]] to i64
+; CHECK-NEXT:    [[TMP9:%.*]] = call x86_amx @llvm.x86.tileloadd64.internal(i16 [[TMP4]], i16 [[TMP6]], ptr [[TMP7]], i64 [[TMP8]])
+; CHECK-NEXT:    [[TMP10:%.*]] = shl i64 [[TMP1:%.*]], 32
+; CHECK-NEXT:    [[TMP11:%.*]] = ashr exact i64 [[TMP10]], 32
+; CHECK-NEXT:    tail call void @llvm.x86.tilestored64.internal(i16 [[TMP4]], i16 [[TMP6]], ptr [[TMP0:%.*]], i64 [[TMP11]], x86_amx [[TMP9]])
 ; CHECK-NEXT:    ret void
 ;
   %4 = load i16, ptr %2, align 64

From cb353dc74e72b084d54d10515fbc289cc19a2023 Mon Sep 17 00:00:00 2001
From: Zhongyunde <zhongyunde@huawei.com>
Date: Tue, 20 Jun 2023 13:12:02 +0800
Subject: [PATCH 083/130] [LV] Add cost model for simd vector select
 instructions of type float

For simd vector selects, use cmeq + bsl for v2f32/v4f32/v2f64, so their cost are cheep.
Fix https://github.com/llvm/llvm-project/issues/63082

Reviewed By: dmgreen
Differential Revision: https://reviews.llvm.org/D152523
---
 .../AArch64/AArch64TargetTransformInfo.cpp    |  5 ++
 .../CostModel/AArch64/reduce-minmax.ll        | 12 ++---
 .../test/Analysis/CostModel/AArch64/select.ll | 16 ++++++
 .../CostModel/AArch64/vector-select.ll        | 44 ++++++++--------
 ...ting-sinking-required-for-vectorization.ll | 51 ++++++++++++-------
 5 files changed, 81 insertions(+), 47 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 8a00325fff869..bcadd65fc781e 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -2694,6 +2694,11 @@ InstructionCost AArch64TTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
 
     static const TypeConversionCostTblEntry
     VectorSelectTbl[] = {
+      { ISD::SELECT, MVT::v2i1, MVT::v2f32, 2 },
+      { ISD::SELECT, MVT::v2i1, MVT::v2f64, 2 },
+      { ISD::SELECT, MVT::v4i1, MVT::v4f32, 2 },
+      { ISD::SELECT, MVT::v4i1, MVT::v4f16, 2 },
+      { ISD::SELECT, MVT::v8i1, MVT::v8f16, 2 },
       { ISD::SELECT, MVT::v16i1, MVT::v16i16, 16 },
       { ISD::SELECT, MVT::v8i1, MVT::v8i32, 8 },
       { ISD::SELECT, MVT::v16i1, MVT::v16i32, 16 },
diff --git a/llvm/test/Analysis/CostModel/AArch64/reduce-minmax.ll b/llvm/test/Analysis/CostModel/AArch64/reduce-minmax.ll
index c10f87fcef658..afa1d5bdd3161 100644
--- a/llvm/test/Analysis/CostModel/AArch64/reduce-minmax.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/reduce-minmax.ll
@@ -163,9 +163,9 @@ define void @reduce_smax() {
 define void @reduce_fmin16() {
 ; CHECK-NOF16-LABEL: 'reduce_fmin16'
 ; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2f16 = call half @llvm.vector.reduce.fmin.v2f16(<2 x half> undef)
-; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V4f16 = call half @llvm.vector.reduce.fmin.v4f16(<4 x half> undef)
-; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V8f16 = call half @llvm.vector.reduce.fmin.v8f16(<8 x half> undef)
-; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 298 for instruction: %V16f16 = call half @llvm.vector.reduce.fmin.v16f16(<16 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4f16 = call half @llvm.vector.reduce.fmin.v4f16(<4 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 117 for instruction: %V8f16 = call half @llvm.vector.reduce.fmin.v8f16(<8 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 190 for instruction: %V16f16 = call half @llvm.vector.reduce.fmin.v16f16(<16 x half> undef)
 ; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2f16m = call half @llvm.vector.reduce.fminimum.v2f16(<2 x half> undef)
 ; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4f16m = call half @llvm.vector.reduce.fminimum.v4f16(<4 x half> undef)
 ; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V8f16m = call half @llvm.vector.reduce.fminimum.v8f16(<8 x half> undef)
@@ -197,9 +197,9 @@ define void @reduce_fmin16() {
 define void @reduce_fmax16() {
 ; CHECK-NOF16-LABEL: 'reduce_fmax16'
 ; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 12 for instruction: %V2f16 = call half @llvm.vector.reduce.fmax.v2f16(<2 x half> undef)
-; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 40 for instruction: %V4f16 = call half @llvm.vector.reduce.fmax.v4f16(<4 x half> undef)
-; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 198 for instruction: %V8f16 = call half @llvm.vector.reduce.fmax.v8f16(<8 x half> undef)
-; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 298 for instruction: %V16f16 = call half @llvm.vector.reduce.fmax.v16f16(<16 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 18 for instruction: %V4f16 = call half @llvm.vector.reduce.fmax.v4f16(<4 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 117 for instruction: %V8f16 = call half @llvm.vector.reduce.fmax.v8f16(<8 x half> undef)
+; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 190 for instruction: %V16f16 = call half @llvm.vector.reduce.fmax.v16f16(<16 x half> undef)
 ; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 5 for instruction: %V2f16m = call half @llvm.vector.reduce.fmaximum.v2f16(<2 x half> undef)
 ; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 13 for instruction: %V4f16m = call half @llvm.vector.reduce.fmaximum.v4f16(<4 x half> undef)
 ; CHECK-NOF16-NEXT:  Cost Model: Found an estimated cost of 29 for instruction: %V8f16m = call half @llvm.vector.reduce.fmaximum.v8f16(<8 x half> undef)
diff --git a/llvm/test/Analysis/CostModel/AArch64/select.ll b/llvm/test/Analysis/CostModel/AArch64/select.ll
index d493ebb349ba0..a69d02bb562ef 100644
--- a/llvm/test/Analysis/CostModel/AArch64/select.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/select.ll
@@ -18,6 +18,11 @@ define void @select() {
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 80 for instruction: %v16a = select <4 x i1> undef, <4 x i64> undef, <4 x i64> undef
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 160 for instruction: %v16b = select <8 x i1> undef, <8 x i64> undef, <8 x i64> undef
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 320 for instruction: %v16c = select <16 x i1> undef, <16 x i64> undef, <16 x i64> undef
+; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2float = select <2 x i1> undef, <2 x float> undef, <2 x float> undef
+; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4float = select <4 x i1> undef, <4 x float> undef, <4 x float> undef
+; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v2double = select <2 x i1> undef, <2 x double> undef, <2 x double> undef
+; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v4half = select <4 x i1> undef, <4 x half> undef, <4 x half> undef
+; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %v8half = select <8 x i1> undef, <8 x half> undef, <8 x half> undef
 ; CHECK-THROUGHPUT-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
 ;
 ; CHECK-SIZE-LABEL: 'select'
@@ -33,6 +38,11 @@ define void @select() {
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16a = select <4 x i1> undef, <4 x i64> undef, <4 x i64> undef
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16b = select <8 x i1> undef, <8 x i64> undef, <8 x i64> undef
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v16c = select <16 x i1> undef, <16 x i64> undef, <16 x i64> undef
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2float = select <2 x i1> undef, <2 x float> undef, <2 x float> undef
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4float = select <4 x i1> undef, <4 x float> undef, <4 x float> undef
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v2double = select <2 x i1> undef, <2 x double> undef, <2 x double> undef
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v4half = select <4 x i1> undef, <4 x half> undef, <4 x half> undef
+; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: %v8half = select <8 x i1> undef, <8 x half> undef, <8 x half> undef
 ; CHECK-SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
 ;
   %v1 = select i1 undef, i8 undef, i8 undef
@@ -53,5 +63,11 @@ define void @select() {
   %v16b = select <8 x i1> undef, <8 x i64> undef, <8 x i64> undef
   %v16c = select <16 x i1> undef, <16 x i64> undef, <16 x i64> undef
 
+  ; simd vector float
+  %v2float = select <2 x i1> undef, <2 x float> undef, <2 x float> undef
+  %v4float = select <4 x i1> undef, <4 x float> undef, <4 x float> undef
+  %v2double = select <2 x i1> undef, <2 x double> undef, <2 x double> undef
+  %v4half = select <4 x i1> undef, <4 x half> undef, <4 x half> undef
+  %v8half = select <8 x i1> undef, <8 x half> undef, <8 x half> undef
   ret void
 }
diff --git a/llvm/test/Analysis/CostModel/AArch64/vector-select.ll b/llvm/test/Analysis/CostModel/AArch64/vector-select.ll
index 7d36ec2281978..40062605c47c2 100644
--- a/llvm/test/Analysis/CostModel/AArch64/vector-select.ll
+++ b/llvm/test/Analysis/CostModel/AArch64/vector-select.ll
@@ -158,7 +158,7 @@ define <2 x i64> @v2i64_select_no_cmp(<2 x i64> %a, <2 x i64> %b, <2 x i1> %cond
 define <4 x half> @v4f16_select_ogt(<4 x half> %a, <4 x half> %b, <4 x half> %c) {
 ; COST-LABEL: v4f16_select_ogt
 ; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction:   %cmp.1 = fcmp ogt <4 x half> %a, %b
-; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 13 for instruction:   %s.1 = select <4 x i1> %cmp.1, <4 x half> %a, <4 x half> %c
+; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction:   %s.1 = select <4 x i1> %cmp.1, <4 x half> %a, <4 x half> %c
 ; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:  %cmp.1 = fcmp ogt <4 x half> %a, %b
 ; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:  %s.1 = select <4 x i1> %cmp.1, <4 x half> %a, <4 x half> %c
 ;
@@ -176,7 +176,7 @@ define <4 x half> @v4f16_select_ogt(<4 x half> %a, <4 x half> %b, <4 x half> %c)
 define <8 x half> @v8f16_select_ogt(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
 ; COST-LABEL: v8f16_select_ogt
 ; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 29 for instruction:   %cmp.1 = fcmp ogt <8 x half> %a, %b
-; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 29 for instruction:   %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
+; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction:   %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
 ; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:  %cmp.1 = fcmp ogt <8 x half> %a, %b
 ; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:  %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
 ;
@@ -242,7 +242,7 @@ define <2 x double> @v2f64_select_ogt(<2 x double> %a, <2 x double> %b, <2 x dou
 define <4 x half> @v4f16_select_oge(<4 x half> %a, <4 x half> %b, <4 x half> %c) {
 ; COST-LABEL: v4f16_select_oge
 ; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction:   %cmp.1 = fcmp oge <4 x half> %a, %b
-; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 13 for instruction:   %s.1 = select <4 x i1> %cmp.1, <4 x half> %a, <4 x half> %c
+; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction:   %s.1 = select <4 x i1> %cmp.1, <4 x half> %a, <4 x half> %c
 ; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:  %cmp.1 = fcmp oge <4 x half> %a, %b
 ; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:  %s.1 = select <4 x i1> %cmp.1, <4 x half> %a, <4 x half> %c
 ;
@@ -260,7 +260,7 @@ define <4 x half> @v4f16_select_oge(<4 x half> %a, <4 x half> %b, <4 x half> %c)
 define <8 x half> @v8f16_select_oge(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
 ; COST-LABEL: v8f16_select_oge
 ; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 29 for instruction:   %cmp.1 = fcmp oge <8 x half> %a, %b
-; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 29 for instruction:   %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
+; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction:   %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
 ; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:  %cmp.1 = fcmp oge <8 x half> %a, %b
 ; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:  %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
 ;
@@ -326,7 +326,7 @@ define <2 x double> @v2f64_select_oge(<2 x double> %a, <2 x double> %b, <2 x dou
 define <4 x half> @v4f16_select_olt(<4 x half> %a, <4 x half> %b, <4 x half> %c) {
 ; COST-LABEL: v4f16_select_olt
 ; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction:   %cmp.1 = fcmp olt <4 x half> %a, %b
-; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 13 for instruction:   %s.1 = select <4 x i1> %cmp.1, <4 x half> %a, <4 x half> %c
+; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction:   %s.1 = select <4 x i1> %cmp.1, <4 x half> %a, <4 x half> %c
 ; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:  %cmp.1 = fcmp olt <4 x half> %a, %b
 ; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:  %s.1 = select <4 x i1> %cmp.1, <4 x half> %a, <4 x half> %c
 ;
@@ -344,7 +344,7 @@ define <4 x half> @v4f16_select_olt(<4 x half> %a, <4 x half> %b, <4 x half> %c)
 define <8 x half> @v8f16_select_olt(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
 ; COST-LABEL: v8f16_select_olt
 ; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 29 for instruction:   %cmp.1 = fcmp olt <8 x half> %a, %b
-; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 29 for instruction:   %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
+; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction:   %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
 ; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:  %cmp.1 = fcmp olt <8 x half> %a, %b
 ; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:  %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
 ;
@@ -410,7 +410,7 @@ define <2 x double> @v2f64_select_olt(<2 x double> %a, <2 x double> %b, <2 x dou
 define <4 x half> @v4f16_select_ole(<4 x half> %a, <4 x half> %b, <4 x half> %c) {
 ; COST-LABEL: v4f16_select_ole
 ; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction:   %cmp.1 = fcmp ole <4 x half> %a, %b
-; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 13 for instruction:   %s.1 = select <4 x i1> %cmp.1, <4 x half> %a, <4 x half> %c
+; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction:   %s.1 = select <4 x i1> %cmp.1, <4 x half> %a, <4 x half> %c
 ; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:  %cmp.1 = fcmp ole <4 x half> %a, %b
 ; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:  %s.1 = select <4 x i1> %cmp.1, <4 x half> %a, <4 x half> %c
 ;
@@ -428,7 +428,7 @@ define <4 x half> @v4f16_select_ole(<4 x half> %a, <4 x half> %b, <4 x half> %c)
 define <8 x half> @v8f16_select_ole(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
 ; COST-LABEL: v8f16_select_ole
 ; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 29 for instruction:   %cmp.1 = fcmp ole <8 x half> %a, %b
-; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 29 for instruction:   %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
+; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction:   %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
 ; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:  %cmp.1 = fcmp ole <8 x half> %a, %b
 ; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:  %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
 ;
@@ -494,7 +494,7 @@ define <2 x double> @v2f64_select_ole(<2 x double> %a, <2 x double> %b, <2 x dou
 define <4 x half> @v4f16_select_oeq(<4 x half> %a, <4 x half> %b, <4 x half> %c) {
 ; COST-LABEL: v4f16_select_oeq
 ; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction:   %cmp.1 = fcmp oeq <4 x half> %a, %b
-; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 13 for instruction:   %s.1 = select <4 x i1> %cmp.1, <4 x half> %a, <4 x half> %c
+; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction:   %s.1 = select <4 x i1> %cmp.1, <4 x half> %a, <4 x half> %c
 ; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:  %cmp.1 = fcmp oeq <4 x half> %a, %b
 ; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:  %s.1 = select <4 x i1> %cmp.1, <4 x half> %a, <4 x half> %c
 ;
@@ -512,7 +512,7 @@ define <4 x half> @v4f16_select_oeq(<4 x half> %a, <4 x half> %b, <4 x half> %c)
 define <8 x half> @v8f16_select_oeq(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
 ; COST-LABEL: v8f16_select_oeq
 ; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 29 for instruction:   %cmp.1 = fcmp oeq <8 x half> %a, %b
-; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 29 for instruction:   %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
+; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction:   %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
 ; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:  %cmp.1 = fcmp oeq <8 x half> %a, %b
 ; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:  %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
 ;
@@ -578,9 +578,9 @@ define <2 x double> @v2f64_select_oeq(<2 x double> %a, <2 x double> %b, <2 x dou
 define <4 x half> @v4f16_select_one(<4 x half> %a, <4 x half> %b, <4 x half> %c) {
 ; COST-LABEL: v4f16_select_one
 ; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction:   %cmp.1 = fcmp one <4 x half> %a, %b
-; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 13 for instruction:   %s.1 = select <4 x i1> %cmp.1, <4 x half> %a, <4 x half> %c
+; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction:   %s.1 = select <4 x i1> %cmp.1, <4 x half> %a, <4 x half> %c
 ; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:  %cmp.1 = fcmp one <4 x half> %a, %b
-; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 13 for instruction:  %s.1 = select <4 x i1> %cmp.1, <4 x half> %a, <4 x half> %c
+; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction:  %s.1 = select <4 x i1> %cmp.1, <4 x half> %a, <4 x half> %c
 ;
 ; CODE-LABEL: v4f16_select_one
 ; CODE:       bb.0
@@ -598,9 +598,9 @@ define <4 x half> @v4f16_select_one(<4 x half> %a, <4 x half> %b, <4 x half> %c)
 define <8 x half> @v8f16_select_one(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
 ; COST-LABEL: v8f16_select_one
 ; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 29 for instruction:  %cmp.1 = fcmp one <8 x half> %a, %b
-; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 29 for instruction:  %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
+; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction:  %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
 ; COST-FULLFP16-NEXT: Cost Model: Found an estimated cost of 1 for instruction:  %cmp.1 = fcmp one <8 x half> %a, %b
-; COST-FULLFP16-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
+; COST-FULLFP16-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
 ;
 ; CODE-LABEL: v8f16_select_one
 ; CODE:       bb.0
@@ -618,7 +618,7 @@ define <8 x half> @v8f16_select_one(<8 x half> %a, <8 x half> %b, <8 x half> %c)
 define <2 x float> @v2f32_select_one(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
 ; COST-LABEL: v2f32_select_one
 ; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:   %cmp.1 = fcmp one <2 x float> %a, %b
-; COST-NEXT:  Cost Model: Found an estimated cost of 5 for instruction:   %s.1 = select <2 x i1> %cmp.1, <2 x float> %a, <2 x float> %c
+; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction:   %s.1 = select <2 x i1> %cmp.1, <2 x float> %a, <2 x float> %c
 
 ; CODE-LABEL: v2f32_select_one
 ; CODE:       bb.0
@@ -636,7 +636,7 @@ define <2 x float> @v2f32_select_one(<2 x float> %a, <2 x float> %b, <2 x float>
 define <4 x float> @v4f32_select_one(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
 ; COST-LABEL: v4f32_select_one
 ; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:   %cmp.1 = fcmp one <4 x float> %a, %b
-; COST-NEXT:  Cost Model: Found an estimated cost of 13 for instruction:  %s.1 = select <4 x i1> %cmp.1, <4 x float> %a, <4 x float> %c
+; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction:  %s.1 = select <4 x i1> %cmp.1, <4 x float> %a, <4 x float> %c
 
 ; CODE-LABEL: v4f32_select_one
 ; CODE:       bb.0
@@ -654,7 +654,7 @@ define <4 x float> @v4f32_select_one(<4 x float> %a, <4 x float> %b, <4 x float>
 define <2 x double> @v2f64_select_one(<2 x double> %a, <2 x double> %b, <2 x double> %c) {
 ; COST-LABEL: v2f64_select_one
 ; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:   %cmp.1 = fcmp one <2 x double> %a, %b
-; COST-NEXT:  Cost Model: Found an estimated cost of 5 for instruction:   %s.1 = select <2 x i1> %cmp.1, <2 x double> %a, <2 x double> %c
+; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction:   %s.1 = select <2 x i1> %cmp.1, <2 x double> %a, <2 x double> %c
 ;
 ; CODE-LABEL: v2f64_select_one
 ; CODE:       bb.0
@@ -672,7 +672,7 @@ define <2 x double> @v2f64_select_one(<2 x double> %a, <2 x double> %b, <2 x dou
 define <4 x half> @v4f16_select_une(<4 x half> %a, <4 x half> %b, <4 x half> %c) {
 ; COST-LABEL: v4f16_select_une
 ; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 4 for instruction:   %cmp.1 = fcmp une <4 x half> %a, %b
-; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 13 for instruction:   %s.1 = select <4 x i1> %cmp.1, <4 x half> %a, <4 x half> %c
+; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction:   %s.1 = select <4 x i1> %cmp.1, <4 x half> %a, <4 x half> %c
 ; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:  %cmp.1 = fcmp une <4 x half> %a, %b
 ; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:  %s.1 = select <4 x i1> %cmp.1, <4 x half> %a, <4 x half> %c
 ;
@@ -690,7 +690,7 @@ define <4 x half> @v4f16_select_une(<4 x half> %a, <4 x half> %b, <4 x half> %c)
 define <8 x half> @v8f16_select_une(<8 x half> %a, <8 x half> %b, <8 x half> %c) {
 ; COST-LABEL: v8f16_select_une
 ; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 29 for instruction:   %cmp.1 = fcmp une <8 x half> %a, %b
-; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 29 for instruction:   %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
+; COST-NOFP16-NEXT:  Cost Model: Found an estimated cost of 2 for instruction:   %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
 ; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:  %cmp.1 = fcmp une <8 x half> %a, %b
 ; COST-FULLFP16-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:  %s.1 = select <8 x i1> %cmp.1, <8 x half> %a, <8 x half> %c
 ;
@@ -756,7 +756,7 @@ define <2 x double> @v2f64_select_une(<2 x double> %a, <2 x double> %b, <2 x dou
 define <2 x float> @v2f32_select_ord(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
 ; COST-LABEL: v2f32_select_ord
 ; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:   %cmp.1 = fcmp ord <2 x float> %a, %b
-; COST-NEXT:  Cost Model: Found an estimated cost of 5 for instruction:   %s.1 = select <2 x i1> %cmp.1, <2 x float> %a, <2 x float> %c
+; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction:   %s.1 = select <2 x i1> %cmp.1, <2 x float> %a, <2 x float> %c
 ;
 ; CODE-LABEL: v2f32_select_ord
 ; CODE:       bb.0
@@ -774,7 +774,7 @@ define <2 x float> @v2f32_select_ord(<2 x float> %a, <2 x float> %b, <2 x float>
 define <4 x float> @v4f32_select_ord(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
 ; COST-LABEL: v4f32_select_ord
 ; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:   %cmp.1 = fcmp ord <4 x float> %a, %b
-; COST-NEXT:  Cost Model: Found an estimated cost of 13 for instruction:  %s.1 = select <4 x i1> %cmp.1, <4 x float> %a, <4 x float> %c
+; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction:  %s.1 = select <4 x i1> %cmp.1, <4 x float> %a, <4 x float> %c
 
 ; CODE-LABEL: v4f32_select_ord
 ; CODE:       bb.0
@@ -792,7 +792,7 @@ define <4 x float> @v4f32_select_ord(<4 x float> %a, <4 x float> %b, <4 x float>
 define <2 x double> @v2f64_select_ord(<2 x double> %a, <2 x double> %b, <2 x double> %c) {
 ; COST-LABEL: v2f64_select_ord
 ; COST-NEXT:  Cost Model: Found an estimated cost of 1 for instruction:   %cmp.1 = fcmp ord <2 x double> %a, %b
-; COST-NEXT:  Cost Model: Found an estimated cost of 5 for instruction:   %s.1 = select <2 x i1> %cmp.1, <2 x double> %a, <2 x double> %c
+; COST-NEXT:  Cost Model: Found an estimated cost of 2 for instruction:   %s.1 = select <2 x i1> %cmp.1, <2 x double> %a, <2 x double> %c
 ;
 ; CODE-LABEL: v2f64_select_ord
 ; CODE:       bb.0
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/hoisting-sinking-required-for-vectorization.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/hoisting-sinking-required-for-vectorization.ll
index c25d68c9b67ad..b98352524e185 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/hoisting-sinking-required-for-vectorization.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/hoisting-sinking-required-for-vectorization.ll
@@ -133,37 +133,50 @@ for.end:                                          ; preds = %for.cond.cleanup
 define void @loop2(ptr %A, ptr %B, ptr %C, float %x) {
 ; CHECK-LABEL: @loop2(
 ; CHECK-NEXT:  entry:
-; CHECK-NEXT:    [[UGLYGEP:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 40000
-; CHECK-NEXT:    [[UGLYGEP2:%.*]] = getelementptr i8, ptr [[C:%.*]], i64 40000
-; CHECK-NEXT:    [[UGLYGEP3:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 40000
-; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ugt ptr [[UGLYGEP2]], [[B]]
-; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ugt ptr [[UGLYGEP]], [[C]]
+; CHECK-NEXT:    [[SCEVGEP:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 40000
+; CHECK-NEXT:    [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[C:%.*]], i64 40000
+; CHECK-NEXT:    [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 40000
+; CHECK-NEXT:    [[BOUND0:%.*]] = icmp ugt ptr [[SCEVGEP2]], [[B]]
+; CHECK-NEXT:    [[BOUND1:%.*]] = icmp ugt ptr [[SCEVGEP]], [[C]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]]
-; CHECK-NEXT:    [[BOUND04:%.*]] = icmp ugt ptr [[UGLYGEP3]], [[B]]
-; CHECK-NEXT:    [[BOUND15:%.*]] = icmp ugt ptr [[UGLYGEP]], [[A]]
+; CHECK-NEXT:    [[BOUND04:%.*]] = icmp ugt ptr [[SCEVGEP3]], [[B]]
+; CHECK-NEXT:    [[BOUND15:%.*]] = icmp ugt ptr [[SCEVGEP]], [[A]]
 ; CHECK-NEXT:    [[FOUND_CONFLICT6:%.*]] = and i1 [[BOUND04]], [[BOUND15]]
 ; CHECK-NEXT:    [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT6]]
 ; CHECK-NEXT:    br i1 [[CONFLICT_RDX]], label [[LOOP_BODY:%.*]], label [[VECTOR_PH:%.*]]
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x float> poison, float [[X:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT]], <4 x float> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT10:%.*]] = insertelement <4 x float> poison, float [[X]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT11:%.*]] = shufflevector <4 x float> [[BROADCAST_SPLATINSERT10]], <4 x float> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]]
 ; CHECK-NEXT:    [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4, !alias.scope !4
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], <i32 20, i32 20, i32 20, i32 20>
-; CHECK-NEXT:    [[TMP2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD7:%.*]] = load <4 x float>, ptr [[TMP2]], align 4, !alias.scope !7
-; CHECK-NEXT:    [[TMP3:%.*]] = fmul <4 x float> [[WIDE_LOAD7]], [[BROADCAST_SPLAT]]
-; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr float, ptr [[B]], i64 [[INDEX]]
-; CHECK-NEXT:    [[WIDE_LOAD8:%.*]] = load <4 x float>, ptr [[TMP4]], align 4, !alias.scope !9, !noalias !11
-; CHECK-NEXT:    [[TMP5:%.*]] = select <4 x i1> [[TMP1]], <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, <4 x float> [[WIDE_LOAD8]]
-; CHECK-NEXT:    [[PREDPHI:%.*]] = fadd <4 x float> [[TMP3]], [[TMP5]]
-; CHECK-NEXT:    store <4 x float> [[PREDPHI]], ptr [[TMP4]], align 4, !alias.scope !9, !noalias !11
-; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
-; CHECK-NEXT:    [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10000
-; CHECK-NEXT:    br i1 [[TMP6]], label [[EXIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
+; CHECK-NEXT:    [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 4
+; CHECK-NEXT:    [[WIDE_LOAD7:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4, !alias.scope !4
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], <i32 20, i32 20, i32 20, i32 20>
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD7]], <i32 20, i32 20, i32 20, i32 20>
+; CHECK-NEXT:    [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD8:%.*]] = load <4 x float>, ptr [[TMP4]], align 4, !alias.scope !7
+; CHECK-NEXT:    [[TMP5:%.*]] = getelementptr inbounds float, ptr [[TMP4]], i64 4
+; CHECK-NEXT:    [[WIDE_LOAD9:%.*]] = load <4 x float>, ptr [[TMP5]], align 4, !alias.scope !7
+; CHECK-NEXT:    [[TMP6:%.*]] = fmul <4 x float> [[WIDE_LOAD8]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP7:%.*]] = fmul <4 x float> [[WIDE_LOAD9]], [[BROADCAST_SPLAT11]]
+; CHECK-NEXT:    [[TMP8:%.*]] = getelementptr float, ptr [[B]], i64 [[INDEX]]
+; CHECK-NEXT:    [[WIDE_LOAD12:%.*]] = load <4 x float>, ptr [[TMP8]], align 4, !alias.scope !9, !noalias !11
+; CHECK-NEXT:    [[TMP9:%.*]] = getelementptr float, ptr [[TMP8]], i64 4
+; CHECK-NEXT:    [[WIDE_LOAD13:%.*]] = load <4 x float>, ptr [[TMP9]], align 4, !alias.scope !9, !noalias !11
+; CHECK-NEXT:    [[TMP10:%.*]] = select <4 x i1> [[TMP2]], <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, <4 x float> [[WIDE_LOAD12]]
+; CHECK-NEXT:    [[PREDPHI:%.*]] = fadd <4 x float> [[TMP6]], [[TMP10]]
+; CHECK-NEXT:    [[TMP11:%.*]] = select <4 x i1> [[TMP3]], <4 x float> <float -0.000000e+00, float -0.000000e+00, float -0.000000e+00, float -0.000000e+00>, <4 x float> [[WIDE_LOAD13]]
+; CHECK-NEXT:    [[PREDPHI14:%.*]] = fadd <4 x float> [[TMP7]], [[TMP11]]
+; CHECK-NEXT:    store <4 x float> [[PREDPHI]], ptr [[TMP8]], align 4, !alias.scope !9, !noalias !11
+; CHECK-NEXT:    store <4 x float> [[PREDPHI14]], ptr [[TMP9]], align 4, !alias.scope !9, !noalias !11
+; CHECK-NEXT:    [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
+; CHECK-NEXT:    [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 10000
+; CHECK-NEXT:    br i1 [[TMP12]], label [[EXIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]]
 ; CHECK:       loop.body:
 ; CHECK-NEXT:    [[IV1:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ], [ 0, [[ENTRY:%.*]] ]
 ; CHECK-NEXT:    [[C_GEP:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IV1]]

From 7ac0ff562ab8b0c2134b17333f4efa6fbbeab48c Mon Sep 17 00:00:00 2001
From: Jaroslav Sevcik <jarin@google.com>
Date: Tue, 20 Jun 2023 07:19:37 +0200
Subject: [PATCH 084/130] [lldb] Make test for D153043 independent of external
 symbols

This removes dependence on the libc abort function.
---
 .../Shell/Unwind/Inputs/eh-frame-dwarf-unwind-abort.s     | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/lldb/test/Shell/Unwind/Inputs/eh-frame-dwarf-unwind-abort.s b/lldb/test/Shell/Unwind/Inputs/eh-frame-dwarf-unwind-abort.s
index 660bb14395e8d..95099ce42d3f5 100644
--- a/lldb/test/Shell/Unwind/Inputs/eh-frame-dwarf-unwind-abort.s
+++ b/lldb/test/Shell/Unwind/Inputs/eh-frame-dwarf-unwind-abort.s
@@ -10,7 +10,7 @@ asm_main:
         .cfi_offset 6, -16
         movq    %rsp, %rbp
         .cfi_def_cfa_register 6
-        callq   abort
+        callq   abort_function
 .L:
         .cfi_def_cfa 7, 8
         .cfi_restore 6
@@ -18,6 +18,12 @@ asm_main:
         ud2
         .cfi_endproc
 
+        .globl abort_function
+abort_function:
+        .cfi_startproc
+        ud2
+        .cfi_endproc
+
 	.data
 	.globl  g_hard_abort
 g_hard_abort:

From a18b17b1befd833165ec34da581cb83ebab43ace Mon Sep 17 00:00:00 2001
From: Jaroslav Sevcik <jarin@google.com>
Date: Tue, 20 Jun 2023 07:56:16 +0200
Subject: [PATCH 085/130] [lldb] Make the test for D153043 linux-only

---
 lldb/test/Shell/Unwind/eh-frame-dwarf-unwind-abort.test | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lldb/test/Shell/Unwind/eh-frame-dwarf-unwind-abort.test b/lldb/test/Shell/Unwind/eh-frame-dwarf-unwind-abort.test
index ccf973d9313c2..477a656a711f9 100644
--- a/lldb/test/Shell/Unwind/eh-frame-dwarf-unwind-abort.test
+++ b/lldb/test/Shell/Unwind/eh-frame-dwarf-unwind-abort.test
@@ -1,7 +1,6 @@
 # Test restoring of register values.
 
-# UNSUPPORTED: system-windows
-# REQUIRES: target-x86_64, native
+# REQUIRES: target-x86_64, system-linux, native
 
 # RUN: %clang_host %p/Inputs/call-asm.c %p/Inputs/eh-frame-dwarf-unwind-abort.s -o %t
 # RUN: %lldb %t -s %s -o exit | FileCheck %s

From f8a4cd0f07e0f16b1a09c6c746cf627fc71f16bd Mon Sep 17 00:00:00 2001
From: Fangrui Song <i@maskray.me>
Date: Mon, 19 Jun 2023 23:02:45 -0700
Subject: [PATCH 086/130] [xray][AArch64] Rewrite trampoline

Optimize (cmp+beq => cbz), duduplicate code (SAVE_REGISTERS/RESTORE_REGISTERS),
improve portability (use ASM_SYMBOL to be compatible with Mach-O), and fix style
issues.
Also, port D37965 (x86 tail call) to __xray_FunctionTailExit.
---
 .../lib/xray/xray_trampoline_AArch64.S        | 223 +++++++-----------
 1 file changed, 83 insertions(+), 140 deletions(-)

diff --git a/compiler-rt/lib/xray/xray_trampoline_AArch64.S b/compiler-rt/lib/xray/xray_trampoline_AArch64.S
index 3bf52cef60fed..eea56d7f0a871 100644
--- a/compiler-rt/lib/xray/xray_trampoline_AArch64.S
+++ b/compiler-rt/lib/xray/xray_trampoline_AArch64.S
@@ -1,163 +1,106 @@
 #include "../builtins/assembly.h"
+#include "../sanitizer_common/sanitizer_asm.h"
 
-    .text
-    /* The variable containing the handler function pointer */
-    .global _ZN6__xray19XRayPatchedFunctionE
-    /* Word-aligned function entry point */
-    .p2align 2
-    /* Let C/C++ see the symbol */
-    .global __xray_FunctionEntry
-    .hidden __xray_FunctionEntry
-    .type __xray_FunctionEntry, %function
-    /* In C++ it is void extern "C" __xray_FunctionEntry(uint32_t FuncId) with
-         FuncId passed in W0 register. */
-__xray_FunctionEntry:
+.macro SAVE_REGISTERS
+  stp x1, x2, [sp, #-16]!
+  stp x3, x4, [sp, #-16]!
+  stp x5, x6, [sp, #-16]!
+  stp x7, x30, [sp, #-16]!
+  stp q0, q1, [sp, #-32]!
+  stp q2, q3, [sp, #-32]!
+  stp q4, q5, [sp, #-32]!
+  stp q6, q7, [sp, #-32]!
+  // x8 is the indirect result register and needs to be preserved for the body of the function to use.
+  stp x8, x0, [sp, #-16]!
+.endm
+
+.macro RESTORE_REGISTERS
+  ldp x8, x0, [sp], #16
+  ldp q6, q7, [sp], #32
+  ldp q4, q5, [sp], #32
+  ldp q2, q3, [sp], #32
+  ldp q0, q1, [sp], #32
+  ldp x7, x30, [sp], #16
+  ldp x5, x6, [sp], #16
+  ldp x3, x4, [sp], #16
+  ldp x1, x2, [sp], #16
+.endm
+
+.text
+.p2align 2
+.global ASM_SYMBOL(__xray_FunctionEntry)
+ASM_HIDDEN(__xray_FunctionEntry)
+ASM_TYPE_FUNCTION(__xray_FunctionEntry)
+ASM_SYMBOL(__xray_FunctionEntry):
     /* Move the return address beyond the end of sled data. The 12 bytes of
          data are inserted in the code of the runtime patch, between the call
          instruction and the instruction returned into. The data contains 32
          bits of instrumented function ID and 64 bits of the address of
          the current trampoline. */
-    ADD X30, X30, #12
-    /* Push the registers which may be modified by the handler function */
-    STP X1, X2, [SP, #-16]!
-    STP X3, X4, [SP, #-16]!
-    STP X5, X6, [SP, #-16]!
-    STP X7, X30, [SP, #-16]!
-    STP Q0, Q1, [SP, #-32]!
-    STP Q2, Q3, [SP, #-32]!
-    STP Q4, Q5, [SP, #-32]!
-    STP Q6, Q7, [SP, #-32]!
-    /* X8 is the indirect result register and needs to be preserved for the body
-     of the function to use */
-    STP X8, X0, [SP, #-16]!
+  add x30, x30, #12
+  // Push the registers which may be modified by the handler function.
+  SAVE_REGISTERS
 
-    /* Load the page address of _ZN6__xray19XRayPatchedFunctionE into X1 */
-    ADRP X1, _ZN6__xray19XRayPatchedFunctionE
-    /* Load the handler function pointer into X2 */
-    LDR X2, [X1, #:lo12:_ZN6__xray19XRayPatchedFunctionE]
-    /* Handler address is nullptr if handler is not set */
-    CMP X2, #0
-    BEQ FunctionEntry_restore
-    /* Function ID is already in W0 (the first parameter).
-         X1=0 means that we are tracing an entry event */
-    MOV X1, #0
-    /* Call the handler with 2 parameters in W0 and X1 */
-    BLR X2
-FunctionEntry_restore:
-    /* Pop the saved registers */
-    LDP X8, X0, [SP], #16
-    LDP Q6, Q7, [SP], #32
-    LDP Q4, Q5, [SP], #32
-    LDP Q2, Q3, [SP], #32
-    LDP Q0, Q1, [SP], #32
-    LDP X7, X30, [SP], #16
-    LDP X5, X6, [SP], #16
-    LDP X3, X4, [SP], #16
-    LDP X1, X2, [SP], #16
-    RET
+  // Load the handler function pointer.
+  adrp x2, ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)
+  ldr x2, [x2, #:lo12:ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)]
+  cbz x2, 1f
+  // Function ID is already in w0 (the first parameter). x1=0 means that we are tracing an entry event.
+  mov x1, #0
+  // Call the handler with 2 parameters.
+  blr x2
+1:
+  RESTORE_REGISTERS
+  ret
 
-    /* Word-aligned function entry point */
-    .p2align 2
-    /* Let C/C++ see the symbol */
-    .global __xray_FunctionExit
-    .hidden __xray_FunctionExit
-    .type __xray_FunctionExit, %function
-    /* In C++ it is void extern "C" __xray_FunctionExit(uint32_t FuncId) with
-         FuncId passed in W0 register. */
-__xray_FunctionExit:
+.p2align 2
+.global ASM_SYMBOL(__xray_FunctionExit)
+ASM_HIDDEN(__xray_FunctionExit)
+ASM_TYPE_FUNCTION(__xray_FunctionExit)
+ASM_SYMBOL(__xray_FunctionExit):
     /* Move the return address beyond the end of sled data. The 12 bytes of
          data are inserted in the code of the runtime patch, between the call
          instruction and the instruction returned into. The data contains 32
          bits of instrumented function ID and 64 bits of the address of
          the current trampoline. */
-    ADD X30, X30, #12
-    /* Push the registers which may be modified by the handler function */
-    STP X1, X2, [SP, #-16]!
-    STP X3, X4, [SP, #-16]!
-    STP X5, X6, [SP, #-16]!
-    STP X7, X30, [SP, #-16]!
-    STP Q0, Q1, [SP, #-32]!
-    STP Q2, Q3, [SP, #-32]!
-    STP Q4, Q5, [SP, #-32]!
-    STP Q6, Q7, [SP, #-32]!
-    /* X8 is the indirect result register and needs to be preserved for the body
-     of the function to use */
-    STP X8, X0, [SP, #-16]!
+  add x30, x30, #12
+  SAVE_REGISTERS
 
-    /* Load the page address of _ZN6__xray19XRayPatchedFunctionE into X1 */
-    ADRP X1, _ZN6__xray19XRayPatchedFunctionE
-    /* Load the handler function pointer into X2 */
-    LDR X2, [X1, #:lo12:_ZN6__xray19XRayPatchedFunctionE]
-    /* Handler address is nullptr if handler is not set */
-    CMP X2, #0
-    BEQ FunctionExit_restore
-    /* Function ID is already in W0 (the first parameter).
-         X1=1 means that we are tracing an exit event */
-    MOV X1, #1
-    /* Call the handler with 2 parameters in W0 and X1 */
-    BLR X2
-FunctionExit_restore:
-    LDP X8, X0, [SP], #16
-    LDP Q6, Q7, [SP], #32
-    LDP Q4, Q5, [SP], #32
-    LDP Q2, Q3, [SP], #32
-    LDP Q0, Q1, [SP], #32
-    LDP X7, X30, [SP], #16
-    LDP X5, X6, [SP], #16
-    LDP X3, X4, [SP], #16
-    LDP X1, X2, [SP], #16
-    RET
+  // Load the handler function pointer into x2.
+  adrp x2, ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)
+  ldr x2, [x2, #:lo12:ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)]
+  cbz x2, 1f
+  // Function ID is already in w0 (the first parameter). x1=1 means that we are tracing an exit event.
+  mov x1, #1
+  // Call the handler with 2 parameters.
+  blr x2
+1:
+  RESTORE_REGISTERS
+  ret
 
-    /* Word-aligned function entry point */
-    .p2align 2
-    /* Let C/C++ see the symbol */
-    .global __xray_FunctionTailExit
-    .hidden __xray_FunctionTailExit
-    .type __xray_FunctionTailExit, %function
-    /* In C++ it is void extern "C" __xray_FunctionTailExit(uint32_t FuncId)
-         with FuncId passed in W0 register. */
-__xray_FunctionTailExit:
+.p2align 2
+.global ASM_SYMBOL(__xray_FunctionTailExit)
+ASM_HIDDEN(__xray_FunctionTailExit)
+ASM_TYPE_FUNCTION(__xray_FunctionTailExit)
+ASM_SYMBOL(__xray_FunctionTailExit):
     /* Move the return address beyond the end of sled data. The 12 bytes of
          data are inserted in the code of the runtime patch, between the call
          instruction and the instruction returned into. The data contains 32
          bits of instrumented function ID and 64 bits of the address of
          the current trampoline. */
-    ADD X30, X30, #12
-    /* Push the registers which may be modified by the handler function */
-    STP X1, X2, [SP, #-16]!
-    STP X3, X4, [SP, #-16]!
-    STP X5, X6, [SP, #-16]!
-    STP X7, X30, [SP, #-16]!
-    /* Push the parameters of the tail called function */
-    STP Q0, Q1, [SP, #-32]!
-    STP Q2, Q3, [SP, #-32]!
-    STP Q4, Q5, [SP, #-32]!
-    STP Q6, Q7, [SP, #-32]!
-    /* Load the page address of _ZN6__xray19XRayPatchedFunctionE into X1 */
-    ADRP X1, _ZN6__xray19XRayPatchedFunctionE
-    /* Load the handler function pointer into X2 */
-    LDR X2, [X1, #:lo12:_ZN6__xray19XRayPatchedFunctionE]
-    /* Handler address is nullptr if handler is not set */
-    CMP X2, #0
-    BEQ FunctionTailExit_restore
-    /* Function ID is already in W0 (the first parameter).
-         X1=2 means that we are tracing a tail exit event, but before the
-         logging part of XRay is ready, we pretend that here a normal function
-         exit happens, so we give the handler code 1 */
-    MOV X1, #1
-    /* Call the handler with 2 parameters in W0 and X1 */
-    BLR X2
-FunctionTailExit_restore:
-    /* Pop the parameters of the tail called function */
-    LDP Q6, Q7, [SP], #32
-    LDP Q4, Q5, [SP], #32
-    LDP Q2, Q3, [SP], #32
-    LDP Q0, Q1, [SP], #32
-    /* Pop the registers which may be modified by the handler function */
-    LDP X7, X30, [SP], #16
-    LDP X5, X6, [SP], #16
-    LDP X3, X4, [SP], #16
-    LDP X1, X2, [SP], #16
-    RET
+  add x30, x30, #12
+  // Save the registers which may be modified by the handler function.
+  SAVE_REGISTERS
+  // Load the handler function pointer into x2.
+  adrp x2, ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)
+  ldr x2, [x2, #:lo12:ASM_SYMBOL(_ZN6__xray19XRayPatchedFunctionE)]
+  cbz x2, 1f
+  // Function ID is already in w0 (the first parameter). x1=2 means that we are tracing a tail exit event.
+  mov x1, #2
+  // Call the handler with 2 parameters.
+  blr x2
+1:
+  RESTORE_REGISTERS
+  ret
 
 NO_EXEC_STACK_DIRECTIVE

From a3b9c1533e9e83c6f966205d0a0000fe72624864 Mon Sep 17 00:00:00 2001
From: Kazu Hirata <kazu@google.com>
Date: Mon, 19 Jun 2023 23:36:14 -0700
Subject: [PATCH 087/130] [tools] Use llvm::is_contained (NFC)

---
 llvm/tools/llvm-dwarfutil/DebugInfoLinker.cpp    | 3 +--
 llvm/tools/llvm-profgen/MissingFrameInferrer.cpp | 6 ++----
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/llvm/tools/llvm-dwarfutil/DebugInfoLinker.cpp b/llvm/tools/llvm-dwarfutil/DebugInfoLinker.cpp
index b4f3c5aa79852..cd719f2101679 100644
--- a/llvm/tools/llvm-dwarfutil/DebugInfoLinker.cpp
+++ b/llvm/tools/llvm-dwarfutil/DebugInfoLinker.cpp
@@ -377,8 +377,7 @@ Error linkDebugInfoImpl(object::ObjectFile &File, const Options &Options,
 
           if (Options.AccelTableKind == DwarfUtilAccelKind::None)
             AccelTableNamesToDelete.push_back(Sec.Name);
-          else if (std::find(AccelTables.begin(), AccelTables.end(),
-                             *SrcAccelTableKind) == AccelTables.end())
+          else if (!llvm::is_contained(AccelTables, *SrcAccelTableKind))
             AccelTableNamesToReplace.push_back(Sec.Name);
         } else if (!knownByDWARFUtil(Sec.Name)) {
           assert(!SrcAccelTableKind);
diff --git a/llvm/tools/llvm-profgen/MissingFrameInferrer.cpp b/llvm/tools/llvm-profgen/MissingFrameInferrer.cpp
index 4127fdc54bd18..ee49950f39ca4 100644
--- a/llvm/tools/llvm-profgen/MissingFrameInferrer.cpp
+++ b/llvm/tools/llvm-profgen/MissingFrameInferrer.cpp
@@ -242,10 +242,8 @@ bool MissingFrameInferrer::inferMissingFrames(
 
   // Done with the inference if the calle is reachable via a single callsite.
   // This may not be accurate but it improves the search throughput.
-  for (auto Target : CallEdgesF[From]) {
-    if (Target == ToFRange->Func)
-      return true;
-  }
+  if (llvm::is_contained(CallEdgesF[From], ToFRange->Func))
+    return true;
 
   // Bail out if callee is not tailcall reachable at all.
   if (!TailCallTargetFuncs.contains(ToFRange->Func))

From 18ec2030844273ea055494bf93c8b8e8d172bec0 Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Tue, 20 Jun 2023 08:54:49 +0200
Subject: [PATCH 088/130] [mlir][transform] Add ApplyRegisteredPassOp transform
 op

This transform op runs a pass on the target op.

Differential Revision: https://reviews.llvm.org/D153143
---
 .../mlir/Dialect/Transform/IR/TransformOps.td | 34 ++++++++
 mlir/lib/Dialect/Transform/IR/CMakeLists.txt  |  1 +
 .../lib/Dialect/Transform/IR/TransformOps.cpp | 80 ++++++++++++++++---
 .../Transform/test-pass-application.mlir      | 72 +++++++++++++++++
 .../llvm-project-overlay/mlir/BUILD.bazel     |  1 +
 5 files changed, 177 insertions(+), 11 deletions(-)
 create mode 100644 mlir/test/Dialect/Transform/test-pass-application.mlir

diff --git a/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td b/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td
index ca427ab7367ce..6f9aaa6f379d7 100644
--- a/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td
+++ b/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td
@@ -194,6 +194,40 @@ def ApplyCanonicalizationPatternsOp
   let assemblyFormat = "attr-dict";
 }
 
+def ApplyRegisteredPassOp : TransformDialectOp<"apply_registered_pass",
+    [TransformOpInterface, TransformEachOpTrait,
+     FunctionalStyleTransformOpTrait, MemoryEffectsOpInterface]> {
+  let summary = "Applies the specified registered pass";
+  let description = [{
+    This transform applies the specified pass to the targeted ops. The name of
+    the pass is specified as a string attribute, as set during pass
+    registration. Optionally, pass options may be specified as a string
+    attribute. The pass options syntax is identical to the one used with
+    "mlir-opt".
+
+    This transform consumes the target handle and produces a new handle that is
+    mapped to the same op. Passes are not allowed to remove/modify the operation
+    that they operate on, so the target op is guaranteed to still exist. The
+    target handle is invalidated because a pass may arbitrarily modify the body
+    of targeted ops.
+  }];
+
+  let arguments = (ins TransformHandleTypeInterface:$target,
+                       StrAttr:$pass_name,
+                       DefaultValuedAttr<StrAttr, "\"\"">:$options);
+  let results = (outs TransformHandleTypeInterface:$result);
+  let assemblyFormat = [{
+    $pass_name `to` $target attr-dict `:` functional-type(operands, results)
+  }];
+
+  let extraClassDeclaration = [{
+    ::mlir::DiagnosedSilenceableFailure applyToOne(
+      ::mlir::Operation *target,
+      ::mlir::transform::ApplyToEachResultList &results,
+      ::mlir::transform::TransformState &state);
+  }];
+}
+
 def CastOp : TransformDialectOp<"cast",
     [TransformOpInterface, TransformEachOpTrait,
      DeclareOpInterfaceMethods<CastOpInterface>,
diff --git a/mlir/lib/Dialect/Transform/IR/CMakeLists.txt b/mlir/lib/Dialect/Transform/IR/CMakeLists.txt
index 62a1f3da1d8fd..a5eace53cd6d1 100644
--- a/mlir/lib/Dialect/Transform/IR/CMakeLists.txt
+++ b/mlir/lib/Dialect/Transform/IR/CMakeLists.txt
@@ -15,6 +15,7 @@ add_mlir_dialect_library(MLIRTransformDialect
   MLIRCastInterfaces
   MLIRIR
   MLIRParser
+  MLIRPass
   MLIRRewrite
   MLIRSideEffectInterfaces
   MLIRTransforms
diff --git a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
index d4852eebf706e..3049930b0ced1 100644
--- a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
+++ b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
@@ -16,6 +16,9 @@
 #include "mlir/IR/FunctionImplementation.h"
 #include "mlir/IR/PatternMatch.h"
 #include "mlir/Interfaces/ControlFlowInterfaces.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Pass/PassManager.h"
+#include "mlir/Pass/PassRegistry.h"
 #include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/ScopeExit.h"
@@ -47,6 +50,26 @@ static ParseResult parseForeachMatchSymbols(OpAsmParser &parser,
                                             ArrayAttr &matchers,
                                             ArrayAttr &actions);
 
+/// Helper function to check if the given transform op is contained in (or
+/// equal to) the given payload target op. In that case, an error is returned.
+/// Transforming transform IR that is currently executing is generally unsafe.
+static DiagnosedSilenceableFailure
+ensurePayloadIsSeparateFromTransform(transform::TransformOpInterface transform,
+                                     Operation *payload) {
+  Operation *transformAncestor = transform.getOperation();
+  while (transformAncestor) {
+    if (transformAncestor == payload) {
+      DiagnosedDefiniteFailure diag =
+          transform.emitDefiniteFailure()
+          << "cannot apply transform to itself (or one of its ancestors)";
+      diag.attachNote(payload->getLoc()) << "target payload op";
+      return diag;
+    }
+    transformAncestor = transformAncestor->getParentOp();
+  }
+  return DiagnosedSilenceableFailure::success();
+}
+
 #define GET_OP_CLASSES
 #include "mlir/Dialect/Transform/IR/TransformOps.cpp.inc"
 
@@ -243,17 +266,10 @@ transform::ApplyPatternsOp::applyToOne(Operation *target,
   // transform IR while it is being interpreted is generally dangerous. Even
   // more so for the ApplyPatternsOp because the GreedyPatternRewriteDriver
   // performs many additional simplifications such as dead code elimination.
-  Operation *transformAncestor = getOperation();
-  while (transformAncestor) {
-    if (transformAncestor == target) {
-      DiagnosedDefiniteFailure diag =
-          emitDefiniteFailure()
-          << "cannot apply transform to itself (or one of its ancestors)";
-      diag.attachNote(target->getLoc()) << "target payload op";
-      return diag;
-    }
-    transformAncestor = transformAncestor->getParentOp();
-  }
+  DiagnosedSilenceableFailure payloadCheck =
+      ensurePayloadIsSeparateFromTransform(*this, target);
+  if (!payloadCheck.succeeded())
+    return payloadCheck;
 
   // Gather all specified patterns.
   MLIRContext *ctx = target->getContext();
@@ -357,6 +373,48 @@ void transform::ApplyCanonicalizationPatternsOp::populatePatterns(
     op.getCanonicalizationPatterns(patterns, ctx);
 }
 
+//===----------------------------------------------------------------------===//
+// ApplyRegisteredPassOp
+//===----------------------------------------------------------------------===//
+
+DiagnosedSilenceableFailure
+transform::ApplyRegisteredPassOp::applyToOne(Operation *target,
+                                             ApplyToEachResultList &results,
+                                             transform::TransformState &state) {
+  // Make sure that this transform is not applied to itself. Modifying the
+  // transform IR while it is being interpreted is generally dangerous. Even
+  // more so when applying passes because they may perform a wide range of IR
+  // modifications.
+  DiagnosedSilenceableFailure payloadCheck =
+      ensurePayloadIsSeparateFromTransform(*this, target);
+  if (!payloadCheck.succeeded())
+    return payloadCheck;
+
+  // Get pass from registry.
+  const PassInfo *passInfo = Pass::lookupPassInfo(getPassName());
+  if (!passInfo) {
+    return emitDefiniteFailure() << "unknown pass: " << getPassName();
+  }
+
+  // Create pass manager with a single pass and run it.
+  PassManager pm(getContext());
+  if (failed(passInfo->addToPipeline(pm, getOptions(), [&](const Twine &msg) {
+        emitError(msg);
+        return failure();
+      }))) {
+    return emitDefiniteFailure()
+           << "failed to add pass to pipeline: " << getPassName();
+  }
+  if (failed(pm.run(target))) {
+    auto diag = emitSilenceableError() << "pass pipeline failed";
+    diag.attachNote(target->getLoc()) << "target op";
+    return diag;
+  }
+
+  results.push_back(target);
+  return DiagnosedSilenceableFailure::success();
+}
+
 //===----------------------------------------------------------------------===//
 // CastOp
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Dialect/Transform/test-pass-application.mlir b/mlir/test/Dialect/Transform/test-pass-application.mlir
new file mode 100644
index 0000000000000..720f1eb79a1a0
--- /dev/null
+++ b/mlir/test/Dialect/Transform/test-pass-application.mlir
@@ -0,0 +1,72 @@
+// RUN: mlir-opt %s --test-transform-dialect-interpreter -allow-unregistered-dialect --split-input-file --verify-diagnostics | FileCheck %s
+
+// CHECK-LABEL: func @successful_pass_application(
+//       CHECK:   %[[c5:.*]] = arith.constant 5 : index
+//       CHECK:   return %[[c5]]
+func.func @successful_pass_application(%t: tensor<5xf32>) -> index {
+  %c0 = arith.constant 0 : index
+  %dim = tensor.dim %t, %c0 : tensor<5xf32>
+  return %dim : index
+}
+
+transform.sequence failures(propagate) {
+^bb1(%arg1: !transform.any_op):
+  %1 = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+  transform.apply_registered_pass "canonicalize" to %1 : (!transform.any_op) -> !transform.any_op
+}
+
+// -----
+
+func.func @invalid_pass_name() {
+  return
+}
+
+transform.sequence failures(propagate) {
+^bb1(%arg1: !transform.any_op):
+  %1 = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+  // expected-error @below {{unknown pass: non-existing-pass}}
+  transform.apply_registered_pass "non-existing-pass" to %1 : (!transform.any_op) -> !transform.any_op
+}
+
+// -----
+
+func.func @not_isolated_from_above(%t: tensor<5xf32>) -> index {
+  %c0 = arith.constant 0 : index
+  // expected-note @below {{target op}}
+  // expected-error @below {{trying to schedule a pass on an operation not marked as 'IsolatedFromAbove'}}
+  %dim = tensor.dim %t, %c0 : tensor<5xf32>
+  return %dim : index
+}
+
+transform.sequence failures(propagate) {
+^bb1(%arg1: !transform.any_op):
+  %1 = transform.structured.match ops{["tensor.dim"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+  // expected-error @below {{pass pipeline failed}}
+  transform.apply_registered_pass "canonicalize" to %1 : (!transform.any_op) -> !transform.any_op
+}
+
+// -----
+
+func.func @invalid_pass_option() {
+  return
+}
+
+transform.sequence failures(propagate) {
+^bb1(%arg1: !transform.any_op):
+  %1 = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+  // expected-error @below {{failed to add pass to pipeline: canonicalize}}
+  transform.apply_registered_pass "canonicalize" to %1 {options = "invalid-option=1"} : (!transform.any_op) -> !transform.any_op
+}
+
+// -----
+
+// CHECK-LABEL: func @valid_pass_option()
+func.func @valid_pass_option() {
+  return
+}
+
+transform.sequence failures(propagate) {
+^bb1(%arg1: !transform.any_op):
+  %1 = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+  transform.apply_registered_pass "canonicalize" to %1 {options = "top-down=false"} : (!transform.any_op) -> !transform.any_op
+}
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index f4865eac5cfc6..2e6a8c76645cf 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -10149,6 +10149,7 @@ cc_library(
         ":CastInterfaces",
         ":ControlFlowInterfaces",
         ":IR",
+        ":Pass",
         ":Rewrite",
         ":SideEffectInterfaces",
         ":Support",

From ab32cc6c02f539bc87e71fefc9670f28a7fc7f56 Mon Sep 17 00:00:00 2001
From: Haojian Wu <hokein.wu@gmail.com>
Date: Mon, 19 Jun 2023 14:57:09 +0200
Subject: [PATCH 089/130] [include-cleaner] Bailout on invalid code for the
 command-line tool

The binary tool only works on working source code, if the source code is
not compilable, don't perform any analysis and edits.

Differential Revision: https://reviews.llvm.org/D153271
---
 .../include-cleaner/tool/IncludeCleaner.cpp            | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/clang-tools-extra/include-cleaner/tool/IncludeCleaner.cpp b/clang-tools-extra/include-cleaner/tool/IncludeCleaner.cpp
index 918f7c968ef90..008da47164092 100644
--- a/clang-tools-extra/include-cleaner/tool/IncludeCleaner.cpp
+++ b/clang-tools-extra/include-cleaner/tool/IncludeCleaner.cpp
@@ -108,10 +108,18 @@ class Action : public clang::ASTFrontendAction {
   }
 
   void EndSourceFile() override {
+    const auto &SM = getCompilerInstance().getSourceManager();
+    if (SM.getDiagnostics().hasUncompilableErrorOccurred()) {
+      llvm::errs()
+          << "Skipping file " << getCurrentFile()
+          << " due to compiler errors. clang-include-cleaner expects to "
+             "work on compilable source code.\n";
+      return;
+    }
+
     if (!HTMLReportPath.empty())
       writeHTML();
 
-    const auto &SM = getCompilerInstance().getSourceManager();
     auto &HS = getCompilerInstance().getPreprocessor().getHeaderSearchInfo();
     llvm::StringRef Path =
         SM.getFileEntryForID(SM.getMainFileID())->tryGetRealPathName();

From c3075023850bbb7276085198d42b69938d48380d Mon Sep 17 00:00:00 2001
From: Nathan Ridge <zeratul976@hotmail.com>
Date: Tue, 20 Jun 2023 02:57:36 -0400
Subject: [PATCH 090/130] [clangd] Index the type of a non-type template
 parameter

Fixes https://github.com/clangd/clangd/issues/1666

Differential Revision: https://reviews.llvm.org/D153251
---
 clang/lib/Index/IndexDecl.cpp        |  1 +
 clang/unittests/Index/IndexTests.cpp | 14 ++++++++++++++
 2 files changed, 15 insertions(+)

diff --git a/clang/lib/Index/IndexDecl.cpp b/clang/lib/Index/IndexDecl.cpp
index 882e02836d4fb..1c04aa17d53fb 100644
--- a/clang/lib/Index/IndexDecl.cpp
+++ b/clang/lib/Index/IndexDecl.cpp
@@ -705,6 +705,7 @@ class IndexingDeclVisitor : public ConstDeclVisitor<IndexingDeclVisitor, bool> {
           IndexCtx.handleReference(C->getNamedConcept(), C->getConceptNameLoc(),
                                    Parent, TTP->getLexicalDeclContext());
       } else if (const auto *NTTP = dyn_cast<NonTypeTemplateParmDecl>(TP)) {
+        IndexCtx.indexTypeSourceInfo(NTTP->getTypeSourceInfo(), Parent);
         if (NTTP->hasDefaultArgument())
           IndexCtx.indexBody(NTTP->getDefaultArgument(), Parent);
       } else if (const auto *TTPD = dyn_cast<TemplateTemplateParmDecl>(TP)) {
diff --git a/clang/unittests/Index/IndexTests.cpp b/clang/unittests/Index/IndexTests.cpp
index 88ad63b97b92a..690673a1072b0 100644
--- a/clang/unittests/Index/IndexTests.cpp
+++ b/clang/unittests/Index/IndexTests.cpp
@@ -392,6 +392,20 @@ TEST(IndexTest, EnumBase) {
             Contains(AllOf(QName("MyTypedef"), HasRole(SymbolRole::Reference),
                            WrittenAt(Position(4, 16))))));
 }
+
+TEST(IndexTest, NonTypeTemplateParameter) {
+  std::string Code = R"cpp(
+    enum class Foobar { foo };
+    template <Foobar f>
+    constexpr void func() {}
+  )cpp";
+  auto Index = std::make_shared<Indexer>();
+  tooling::runToolOnCode(std::make_unique<IndexAction>(Index), Code);
+  EXPECT_THAT(Index->Symbols,
+              Contains(AllOf(QName("Foobar"), HasRole(SymbolRole::Reference),
+                             WrittenAt(Position(3, 15)))));
+}
+
 } // namespace
 } // namespace index
 } // namespace clang

From aa7b127cb7314e326457d7f790d36db1cb74f63c Mon Sep 17 00:00:00 2001
From: Diana Picus <Diana-Magda.Picus@amd.com>
Date: Wed, 7 Jun 2023 13:52:36 +0200
Subject: [PATCH 091/130] [AMDGPU] Start documenting calling conventions. NFC

Add a section to AMDGPUUsage.rst about calling conventions and list the
ones from the CallingConv enum. Full descriptions can come later (help
appreciated).

Differential Revision: https://reviews.llvm.org/D151996
---
 llvm/docs/AMDGPUUsage.rst | 61 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)

diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 90f50e487d8ff..ce1f16a6bf95c 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -1064,6 +1064,67 @@ The AMDGPU backend supports the following LLVM IR attributes.
 
      ======================================= ==========================================================
 
+Calling Conventions
+-------------------
+
+The AMDGPU backend supports the following calling conventions:
+
+  .. table:: AMDGPU Calling Conventions
+     :name: amdgpu-cc
+
+     =============================== ==========================================================
+     Calling Convention              Description
+     =============================== ==========================================================
+     ``ccc``                         The C calling convention. Used by default.
+                                     See :ref:`amdgpu-amdhsa-function-call-convention-non-kernel-functions`
+                                     for more details.
+
+     ``fastcc``                      The fast calling convention. Mostly the same as the ``ccc``.
+
+     ``coldcc``                      The cold calling convention. Mostly the same as the ``ccc`.
+
+     ``amdgpu_cs``                   Used for Mesa/AMDPAL compute shaders.
+                                     ..TODO::
+                                     Describe.
+
+     ``amdgpu_es``                   Used for AMDPAL shader stage before geometry shader if geometry is in
+                                     use. So either the domain (= tessellation evaluation) shader if
+                                     tessellation is in use, or otherwise the vertex shader.
+                                     ..TODO::
+                                     Describe.
+
+     ``amdgpu_gfx``                  Used for AMD graphics targets. Functions with this calling convention
+                                     cannot be used as entry points.
+                                     ..TODO::
+                                     Describe.
+
+     ``amdgpu_gs``                   Used for Mesa/AMDPAL geometry shaders.
+                                     ..TODO::
+                                     Describe.
+
+     ``amdgpu_hs``                   Used for Mesa/AMDPAL hull shaders (= tessellation control shaders).
+                                     ..TODO::
+                                     Describe.
+
+     ``amdgpu_kernel``               See :ref:`_amdgpu-amdhsa-function-call-convention-kernel-functions`
+
+     ``amdgpu_ls``                   Used for AMDPAL vertex shader if tessellation is in use.
+                                     ..TODO::
+                                     Describe.
+
+     ``amdgpu_ps``                   Used for Mesa/AMDPAL pixel shaders.
+                                     ..TODO::
+                                     Describe.
+
+     ``amdgpu_vs``                   Used for Mesa/AMDPAL last shader stage before rasterization (vertex
+                                     shader if tessellation and geometry are not in use, or otherwise
+                                     copy shader if one is needed).
+                                     ..TODO::
+                                     Describe.
+
+     =============================== ==========================================================
+
+
 .. _amdgpu-elf-code-object:
 
 ELF Code Object

From d116f20e37d96bf5ee847053ad77aec02f7167bd Mon Sep 17 00:00:00 2001
From: Diana Picus <Diana-Magda.Picus@amd.com>
Date: Tue, 20 Jun 2023 09:37:16 +0200
Subject: [PATCH 092/130] Fixup D151996

---
 llvm/docs/AMDGPUUsage.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index ce1f16a6bf95c..7aea864ddfc82 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -1081,7 +1081,7 @@ The AMDGPU backend supports the following calling conventions:
 
      ``fastcc``                      The fast calling convention. Mostly the same as the ``ccc``.
 
-     ``coldcc``                      The cold calling convention. Mostly the same as the ``ccc`.
+     ``coldcc``                      The cold calling convention. Mostly the same as the ``ccc``.
 
      ``amdgpu_cs``                   Used for Mesa/AMDPAL compute shaders.
                                      ..TODO::

From aad51417452432092258034c55537a7092fa6295 Mon Sep 17 00:00:00 2001
From: Martin Braenne <mboehme@google.com>
Date: Tue, 20 Jun 2023 07:15:31 +0000
Subject: [PATCH 093/130] Revert "Prevent deadlocks in death tests."

This reverts commit dfbcee286b9b96751014ebc5ba5290e42796be37.

This was causing unit tests to fail on Gentoo, see comments on
https://reviews.llvm.org/D152696.
---
 third-party/unittest/UnitTestMain/TestMain.cpp | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/third-party/unittest/UnitTestMain/TestMain.cpp b/third-party/unittest/UnitTestMain/TestMain.cpp
index c1fb848491e84..35ba72ba3fcd3 100644
--- a/third-party/unittest/UnitTestMain/TestMain.cpp
+++ b/third-party/unittest/UnitTestMain/TestMain.cpp
@@ -29,10 +29,6 @@ int main(int argc, char **argv) {
                                             true /* Disable crash reporting */);
   }
 
-  // Use the "threadsafe" test style for death tests -- the "fast" test style
-  // can cause deadlocks.
-  testing::GTEST_FLAG(death_test_style) = "threadsafe";
-
   // Initialize both gmock and gtest.
   testing::InitGoogleMock(&argc, argv);
 

From 99610fa77df46fbc38d80e0b7ee02acfddbd032d Mon Sep 17 00:00:00 2001
From: Diana Picus <Diana-Magda.Picus@amd.com>
Date: Tue, 20 Jun 2023 09:54:40 +0200
Subject: [PATCH 094/130] Revert "[AMDGPU] Start documenting calling
 conventions. NFC"

This reverts commit aa7b127cb7314e326457d7f790d36db1cb74f63c.

...because I really ought to install sphinx.
---
 llvm/docs/AMDGPUUsage.rst | 61 ---------------------------------------
 1 file changed, 61 deletions(-)

diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 7aea864ddfc82..90f50e487d8ff 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -1064,67 +1064,6 @@ The AMDGPU backend supports the following LLVM IR attributes.
 
      ======================================= ==========================================================
 
-Calling Conventions
--------------------
-
-The AMDGPU backend supports the following calling conventions:
-
-  .. table:: AMDGPU Calling Conventions
-     :name: amdgpu-cc
-
-     =============================== ==========================================================
-     Calling Convention              Description
-     =============================== ==========================================================
-     ``ccc``                         The C calling convention. Used by default.
-                                     See :ref:`amdgpu-amdhsa-function-call-convention-non-kernel-functions`
-                                     for more details.
-
-     ``fastcc``                      The fast calling convention. Mostly the same as the ``ccc``.
-
-     ``coldcc``                      The cold calling convention. Mostly the same as the ``ccc``.
-
-     ``amdgpu_cs``                   Used for Mesa/AMDPAL compute shaders.
-                                     ..TODO::
-                                     Describe.
-
-     ``amdgpu_es``                   Used for AMDPAL shader stage before geometry shader if geometry is in
-                                     use. So either the domain (= tessellation evaluation) shader if
-                                     tessellation is in use, or otherwise the vertex shader.
-                                     ..TODO::
-                                     Describe.
-
-     ``amdgpu_gfx``                  Used for AMD graphics targets. Functions with this calling convention
-                                     cannot be used as entry points.
-                                     ..TODO::
-                                     Describe.
-
-     ``amdgpu_gs``                   Used for Mesa/AMDPAL geometry shaders.
-                                     ..TODO::
-                                     Describe.
-
-     ``amdgpu_hs``                   Used for Mesa/AMDPAL hull shaders (= tessellation control shaders).
-                                     ..TODO::
-                                     Describe.
-
-     ``amdgpu_kernel``               See :ref:`_amdgpu-amdhsa-function-call-convention-kernel-functions`
-
-     ``amdgpu_ls``                   Used for AMDPAL vertex shader if tessellation is in use.
-                                     ..TODO::
-                                     Describe.
-
-     ``amdgpu_ps``                   Used for Mesa/AMDPAL pixel shaders.
-                                     ..TODO::
-                                     Describe.
-
-     ``amdgpu_vs``                   Used for Mesa/AMDPAL last shader stage before rasterization (vertex
-                                     shader if tessellation and geometry are not in use, or otherwise
-                                     copy shader if one is needed).
-                                     ..TODO::
-                                     Describe.
-
-     =============================== ==========================================================
-
-
 .. _amdgpu-elf-code-object:
 
 ELF Code Object

From 56e33d98814586ac6ca5a24b2978b489d1b67ccb Mon Sep 17 00:00:00 2001
From: Ben Shi <2283975856@qq.com>
Date: Mon, 19 Jun 2023 17:12:25 +0800
Subject: [PATCH 095/130] [CSKY][test][NFC] Add more tests of multiplication
 with immediates

Reviewed By: zixuan-wu

Differential Revision: https://reviews.llvm.org/D153105
---
 llvm/lib/Target/CSKY/CSKY.td              |   8 +-
 llvm/lib/Target/CSKY/CSKYISelLowering.cpp |   6 +-
 llvm/test/CodeGen/CSKY/mul-imm.ll         | 399 ++++++++++++++++++++++
 3 files changed, 406 insertions(+), 7 deletions(-)
 create mode 100644 llvm/test/CodeGen/CSKY/mul-imm.ll

diff --git a/llvm/lib/Target/CSKY/CSKY.td b/llvm/lib/Target/CSKY/CSKY.td
index e5ac106c9b59a..9809caa8bd8f6 100644
--- a/llvm/lib/Target/CSKY/CSKY.td
+++ b/llvm/lib/Target/CSKY/CSKY.td
@@ -40,17 +40,17 @@ def HasFdivdu : Predicate<"Subtarget->hasFdivdu()">,
 
 def FeatureFPUV3_HI
     : SubtargetFeature<"fpuv3_hi", "HasFPUv3HalfWord", "true",
-                       "Enable FPUv3 harf word converting instructions">;
+                       "Enable FPUv3 half word converting instructions">;
 def HasFPUv3_HI : Predicate<"Subtarget->hasFPUv3HalfWord()">,
                   AssemblerPredicate<(all_of FeatureFPUV3_HI),
-                  "Enable FPUv3 harf word converting instructions">;
+                  "Enable FPUv3 half word converting instructions">;
 
 def FeatureFPUV3_HF
     : SubtargetFeature<"fpuv3_hf", "HasFPUv3HalfFloat", "true",
-                       "Enable FPUv3 harf precision operate instructions">;
+                       "Enable FPUv3 half precision operate instructions">;
 def HasFPUv3_HF : Predicate<"Subtarget->hasFPUv3HalfFloat()">,
                   AssemblerPredicate<(all_of FeatureFPUV3_HF),
-                  "Enable FPUv3 harf precision operate instructions">;
+                  "Enable FPUv3 half precision operate instructions">;
 
 def FeatureFPUV3_SF
     : SubtargetFeature<"fpuv3_sf", "HasFPUv3SingleFloat", "true",
diff --git a/llvm/lib/Target/CSKY/CSKYISelLowering.cpp b/llvm/lib/Target/CSKY/CSKYISelLowering.cpp
index 2c8649a3b6ad1..4a2df7cd4d021 100644
--- a/llvm/lib/Target/CSKY/CSKYISelLowering.cpp
+++ b/llvm/lib/Target/CSKY/CSKYISelLowering.cpp
@@ -116,9 +116,9 @@ CSKYTargetLowering::CSKYTargetLowering(const TargetMachine &TM,
       ISD::SETUGE, ISD::SETULT, ISD::SETULE,
   };
 
-  ISD::NodeType FPOpToExpand[] = {ISD::FSIN, ISD::FCOS, ISD::FSINCOS,
-                                  ISD::FPOW, ISD::FREM, ISD::FCOPYSIGN,
-                                  ISD::FP16_TO_FP, ISD::FP_TO_FP16};
+  ISD::NodeType FPOpToExpand[] = {
+      ISD::FSIN, ISD::FCOS,      ISD::FSINCOS,    ISD::FPOW,
+      ISD::FREM, ISD::FCOPYSIGN, ISD::FP16_TO_FP, ISD::FP_TO_FP16};
 
   if (STI.useHardFloat()) {
 
diff --git a/llvm/test/CodeGen/CSKY/mul-imm.ll b/llvm/test/CodeGen/CSKY/mul-imm.ll
new file mode 100644
index 0000000000000..c87e527430de5
--- /dev/null
+++ b/llvm/test/CodeGen/CSKY/mul-imm.ll
@@ -0,0 +1,399 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -verify-machineinstrs -csky-no-aliases -mattr=+e2 < %s -mtriple=csky | FileCheck %s
+; RUN: llc -verify-machineinstrs -csky-no-aliases < %s -mtriple=csky | FileCheck %s --check-prefix=GENERIC
+
+;; This file shows if a multiplication can be simplified to an addition/subtraction
+;; of left shifts.
+
+define i32 @mul_i32_4097(i32 %x) {
+; CHECK-LABEL: mul_i32_4097:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movi32 a1, 4097
+; CHECK-NEXT:    mult16 a0, a1
+; CHECK-NEXT:    rts16
+;
+; GENERIC-LABEL: mul_i32_4097:
+; GENERIC:       # %bb.0: # %entry
+; GENERIC-NEXT:    .cfi_def_cfa_offset 0
+; GENERIC-NEXT:    subi16 sp, sp, 4
+; GENERIC-NEXT:    .cfi_def_cfa_offset 4
+; GENERIC-NEXT:    movi16 a1, 0
+; GENERIC-NEXT:    lsli16 a2, a1, 24
+; GENERIC-NEXT:    lsli16 a1, a1, 16
+; GENERIC-NEXT:    or16 a1, a2
+; GENERIC-NEXT:    movi16 a2, 16
+; GENERIC-NEXT:    lsli16 a2, a2, 8
+; GENERIC-NEXT:    or16 a2, a1
+; GENERIC-NEXT:    movi16 a1, 1
+; GENERIC-NEXT:    or16 a1, a2
+; GENERIC-NEXT:    mult16 a0, a1
+; GENERIC-NEXT:    addi16 sp, sp, 4
+; GENERIC-NEXT:    rts16
+entry:
+  %y = mul nsw i32 %x, 4097
+  ret i32 %y
+}
+
+define i32 @mul_i32_4095(i32 %x) {
+; CHECK-LABEL: mul_i32_4095:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movi32 a1, 4095
+; CHECK-NEXT:    mult16 a0, a1
+; CHECK-NEXT:    rts16
+;
+; GENERIC-LABEL: mul_i32_4095:
+; GENERIC:       # %bb.0: # %entry
+; GENERIC-NEXT:    .cfi_def_cfa_offset 0
+; GENERIC-NEXT:    subi16 sp, sp, 4
+; GENERIC-NEXT:    .cfi_def_cfa_offset 4
+; GENERIC-NEXT:    movi16 a1, 0
+; GENERIC-NEXT:    lsli16 a2, a1, 24
+; GENERIC-NEXT:    lsli16 a1, a1, 16
+; GENERIC-NEXT:    or16 a1, a2
+; GENERIC-NEXT:    movi16 a2, 15
+; GENERIC-NEXT:    lsli16 a2, a2, 8
+; GENERIC-NEXT:    or16 a2, a1
+; GENERIC-NEXT:    movi16 a1, 255
+; GENERIC-NEXT:    or16 a1, a2
+; GENERIC-NEXT:    mult16 a0, a1
+; GENERIC-NEXT:    addi16 sp, sp, 4
+; GENERIC-NEXT:    rts16
+entry:
+  %y = mul nsw i32 %x, 4095
+  ret i32 %y
+}
+
+define i32 @mul_i32_minus_4095(i32 %x) {
+; CHECK-LABEL: mul_i32_minus_4095:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movih32 a1, 65535
+; CHECK-NEXT:    ori32 a1, a1, 61441
+; CHECK-NEXT:    mult16 a0, a1
+; CHECK-NEXT:    rts16
+;
+; GENERIC-LABEL: mul_i32_minus_4095:
+; GENERIC:       # %bb.0: # %entry
+; GENERIC-NEXT:    .cfi_def_cfa_offset 0
+; GENERIC-NEXT:    subi16 sp, sp, 4
+; GENERIC-NEXT:    .cfi_def_cfa_offset 4
+; GENERIC-NEXT:    movi16 a1, 255
+; GENERIC-NEXT:    lsli16 a2, a1, 24
+; GENERIC-NEXT:    lsli16 a1, a1, 16
+; GENERIC-NEXT:    or16 a1, a2
+; GENERIC-NEXT:    movi16 a2, 240
+; GENERIC-NEXT:    lsli16 a2, a2, 8
+; GENERIC-NEXT:    or16 a2, a1
+; GENERIC-NEXT:    movi16 a1, 1
+; GENERIC-NEXT:    or16 a1, a2
+; GENERIC-NEXT:    mult16 a0, a1
+; GENERIC-NEXT:    addi16 sp, sp, 4
+; GENERIC-NEXT:    rts16
+entry:
+  %y = mul nsw i32 %x, -4095
+  ret i32 %y
+}
+
+define i16 @mul_i16_4097(i16 %x) {
+; CHECK-LABEL: mul_i16_4097:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movi32 a1, 4097
+; CHECK-NEXT:    mult16 a0, a1
+; CHECK-NEXT:    rts16
+;
+; GENERIC-LABEL: mul_i16_4097:
+; GENERIC:       # %bb.0: # %entry
+; GENERIC-NEXT:    .cfi_def_cfa_offset 0
+; GENERIC-NEXT:    subi16 sp, sp, 4
+; GENERIC-NEXT:    .cfi_def_cfa_offset 4
+; GENERIC-NEXT:    movi16 a1, 0
+; GENERIC-NEXT:    lsli16 a2, a1, 24
+; GENERIC-NEXT:    lsli16 a1, a1, 16
+; GENERIC-NEXT:    or16 a1, a2
+; GENERIC-NEXT:    movi16 a2, 16
+; GENERIC-NEXT:    lsli16 a2, a2, 8
+; GENERIC-NEXT:    or16 a2, a1
+; GENERIC-NEXT:    movi16 a1, 1
+; GENERIC-NEXT:    or16 a1, a2
+; GENERIC-NEXT:    mult16 a0, a1
+; GENERIC-NEXT:    addi16 sp, sp, 4
+; GENERIC-NEXT:    rts16
+entry:
+  %y = mul nsw i16 %x, 4097
+  ret i16 %y
+}
+
+define i16 @mul_i16_4095(i16 %x) {
+; CHECK-LABEL: mul_i16_4095:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movi32 a1, 4095
+; CHECK-NEXT:    mult16 a0, a1
+; CHECK-NEXT:    rts16
+;
+; GENERIC-LABEL: mul_i16_4095:
+; GENERIC:       # %bb.0: # %entry
+; GENERIC-NEXT:    .cfi_def_cfa_offset 0
+; GENERIC-NEXT:    subi16 sp, sp, 4
+; GENERIC-NEXT:    .cfi_def_cfa_offset 4
+; GENERIC-NEXT:    movi16 a1, 0
+; GENERIC-NEXT:    lsli16 a2, a1, 24
+; GENERIC-NEXT:    lsli16 a1, a1, 16
+; GENERIC-NEXT:    or16 a1, a2
+; GENERIC-NEXT:    movi16 a2, 15
+; GENERIC-NEXT:    lsli16 a2, a2, 8
+; GENERIC-NEXT:    or16 a2, a1
+; GENERIC-NEXT:    movi16 a1, 255
+; GENERIC-NEXT:    or16 a1, a2
+; GENERIC-NEXT:    mult16 a0, a1
+; GENERIC-NEXT:    addi16 sp, sp, 4
+; GENERIC-NEXT:    rts16
+entry:
+  %y = mul nsw i16 %x, 4095
+  ret i16 %y
+}
+
+define i16 @mul_i16_minus_4095(i16 %x) {
+; CHECK-LABEL: mul_i16_minus_4095:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movih32 a1, 65535
+; CHECK-NEXT:    ori32 a1, a1, 61441
+; CHECK-NEXT:    mult16 a0, a1
+; CHECK-NEXT:    rts16
+;
+; GENERIC-LABEL: mul_i16_minus_4095:
+; GENERIC:       # %bb.0: # %entry
+; GENERIC-NEXT:    .cfi_def_cfa_offset 0
+; GENERIC-NEXT:    subi16 sp, sp, 4
+; GENERIC-NEXT:    .cfi_def_cfa_offset 4
+; GENERIC-NEXT:    movi16 a1, 255
+; GENERIC-NEXT:    lsli16 a2, a1, 24
+; GENERIC-NEXT:    lsli16 a1, a1, 16
+; GENERIC-NEXT:    or16 a1, a2
+; GENERIC-NEXT:    movi16 a2, 240
+; GENERIC-NEXT:    lsli16 a2, a2, 8
+; GENERIC-NEXT:    or16 a2, a1
+; GENERIC-NEXT:    movi16 a1, 1
+; GENERIC-NEXT:    or16 a1, a2
+; GENERIC-NEXT:    mult16 a0, a1
+; GENERIC-NEXT:    addi16 sp, sp, 4
+; GENERIC-NEXT:    rts16
+entry:
+  %y = mul nsw i16 %x, -4095
+  ret i16 %y
+}
+
+define i8 @mul_i8_65(i8 %x) {
+; CHECK-LABEL: mul_i8_65:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movi16 a1, 65
+; CHECK-NEXT:    mult16 a0, a1
+; CHECK-NEXT:    rts16
+;
+; GENERIC-LABEL: mul_i8_65:
+; GENERIC:       # %bb.0: # %entry
+; GENERIC-NEXT:    .cfi_def_cfa_offset 0
+; GENERIC-NEXT:    subi16 sp, sp, 4
+; GENERIC-NEXT:    .cfi_def_cfa_offset 4
+; GENERIC-NEXT:    movi16 a1, 65
+; GENERIC-NEXT:    mult16 a0, a1
+; GENERIC-NEXT:    addi16 sp, sp, 4
+; GENERIC-NEXT:    rts16
+entry:
+  %y = mul nsw i8 %x, 65
+  ret i8 %y
+}
+
+define i8 @mul_i8_63(i8 %x) {
+; CHECK-LABEL: mul_i8_63:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movi16 a1, 63
+; CHECK-NEXT:    mult16 a0, a1
+; CHECK-NEXT:    rts16
+;
+; GENERIC-LABEL: mul_i8_63:
+; GENERIC:       # %bb.0: # %entry
+; GENERIC-NEXT:    .cfi_def_cfa_offset 0
+; GENERIC-NEXT:    subi16 sp, sp, 4
+; GENERIC-NEXT:    .cfi_def_cfa_offset 4
+; GENERIC-NEXT:    movi16 a1, 63
+; GENERIC-NEXT:    mult16 a0, a1
+; GENERIC-NEXT:    addi16 sp, sp, 4
+; GENERIC-NEXT:    rts16
+entry:
+  %y = mul nsw i8 %x, 63
+  ret i8 %y
+}
+
+define i8 @mul_i8_minus_63(i8 %x) {
+; CHECK-LABEL: mul_i8_minus_63:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movih32 a1, 65535
+; CHECK-NEXT:    ori32 a1, a1, 65473
+; CHECK-NEXT:    mult16 a0, a1
+; CHECK-NEXT:    rts16
+;
+; GENERIC-LABEL: mul_i8_minus_63:
+; GENERIC:       # %bb.0: # %entry
+; GENERIC-NEXT:    .cfi_def_cfa_offset 0
+; GENERIC-NEXT:    subi16 sp, sp, 4
+; GENERIC-NEXT:    .cfi_def_cfa_offset 4
+; GENERIC-NEXT:    movi16 a1, 255
+; GENERIC-NEXT:    lsli16 a2, a1, 24
+; GENERIC-NEXT:    lsli16 a3, a1, 16
+; GENERIC-NEXT:    or16 a3, a2
+; GENERIC-NEXT:    lsli16 a1, a1, 8
+; GENERIC-NEXT:    or16 a1, a3
+; GENERIC-NEXT:    movi16 a2, 193
+; GENERIC-NEXT:    or16 a2, a1
+; GENERIC-NEXT:    mult16 a0, a2
+; GENERIC-NEXT:    addi16 sp, sp, 4
+; GENERIC-NEXT:    rts16
+entry:
+  %y = mul nsw i8 %x, -63
+  ret i8 %y
+}
+
+define i32 @mul_i32_minus_4097(i32 %x) {
+; CHECK-LABEL: mul_i32_minus_4097:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movih32 a1, 65535
+; CHECK-NEXT:    ori32 a1, a1, 61439
+; CHECK-NEXT:    mult16 a0, a1
+; CHECK-NEXT:    rts16
+;
+; GENERIC-LABEL: mul_i32_minus_4097:
+; GENERIC:       # %bb.0: # %entry
+; GENERIC-NEXT:    .cfi_def_cfa_offset 0
+; GENERIC-NEXT:    subi16 sp, sp, 4
+; GENERIC-NEXT:    .cfi_def_cfa_offset 4
+; GENERIC-NEXT:    movi16 a1, 255
+; GENERIC-NEXT:    lsli16 a2, a1, 24
+; GENERIC-NEXT:    lsli16 a3, a1, 16
+; GENERIC-NEXT:    or16 a3, a2
+; GENERIC-NEXT:    movi16 a2, 239
+; GENERIC-NEXT:    lsli16 a2, a2, 8
+; GENERIC-NEXT:    or16 a2, a3
+; GENERIC-NEXT:    or16 a2, a1
+; GENERIC-NEXT:    mult16 a0, a2
+; GENERIC-NEXT:    addi16 sp, sp, 4
+; GENERIC-NEXT:    rts16
+entry:
+  %y = mul nsw i32 %x, -4097
+  ret i32 %y
+}
+
+define i16 @mul_i16_minus_4097(i16 %x) {
+; CHECK-LABEL: mul_i16_minus_4097:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movih32 a1, 65535
+; CHECK-NEXT:    ori32 a1, a1, 61439
+; CHECK-NEXT:    mult16 a0, a1
+; CHECK-NEXT:    rts16
+;
+; GENERIC-LABEL: mul_i16_minus_4097:
+; GENERIC:       # %bb.0: # %entry
+; GENERIC-NEXT:    .cfi_def_cfa_offset 0
+; GENERIC-NEXT:    subi16 sp, sp, 4
+; GENERIC-NEXT:    .cfi_def_cfa_offset 4
+; GENERIC-NEXT:    movi16 a1, 255
+; GENERIC-NEXT:    lsli16 a2, a1, 24
+; GENERIC-NEXT:    lsli16 a3, a1, 16
+; GENERIC-NEXT:    or16 a3, a2
+; GENERIC-NEXT:    movi16 a2, 239
+; GENERIC-NEXT:    lsli16 a2, a2, 8
+; GENERIC-NEXT:    or16 a2, a3
+; GENERIC-NEXT:    or16 a2, a1
+; GENERIC-NEXT:    mult16 a0, a2
+; GENERIC-NEXT:    addi16 sp, sp, 4
+; GENERIC-NEXT:    rts16
+entry:
+  %y = mul nsw i16 %x, -4097
+  ret i16 %y
+}
+
+define i8 @mul_i8_minus_65(i8 %x) {
+; CHECK-LABEL: mul_i8_minus_65:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movih32 a1, 65535
+; CHECK-NEXT:    ori32 a1, a1, 65471
+; CHECK-NEXT:    mult16 a0, a1
+; CHECK-NEXT:    rts16
+;
+; GENERIC-LABEL: mul_i8_minus_65:
+; GENERIC:       # %bb.0: # %entry
+; GENERIC-NEXT:    .cfi_def_cfa_offset 0
+; GENERIC-NEXT:    subi16 sp, sp, 4
+; GENERIC-NEXT:    .cfi_def_cfa_offset 4
+; GENERIC-NEXT:    movi16 a1, 255
+; GENERIC-NEXT:    lsli16 a2, a1, 24
+; GENERIC-NEXT:    lsli16 a3, a1, 16
+; GENERIC-NEXT:    or16 a3, a2
+; GENERIC-NEXT:    lsli16 a1, a1, 8
+; GENERIC-NEXT:    or16 a1, a3
+; GENERIC-NEXT:    movi16 a2, 191
+; GENERIC-NEXT:    or16 a2, a1
+; GENERIC-NEXT:    mult16 a0, a2
+; GENERIC-NEXT:    addi16 sp, sp, 4
+; GENERIC-NEXT:    rts16
+entry:
+  %y = mul nsw i8 %x, -65
+  ret i8 %y
+}
+
+;; This case can not be optimized, due to the data type exceeds.
+define i64 @mul_i64_4097(i64 %x) {
+; CHECK-LABEL: mul_i64_4097:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    subi16 sp, sp, 4
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    st32.w lr, (sp, 0) # 4-byte Folded Spill
+; CHECK-NEXT:    .cfi_offset lr, -4
+; CHECK-NEXT:    .cfi_def_cfa_offset 4
+; CHECK-NEXT:    lrw32 t0, [.LCPI12_0]
+; CHECK-NEXT:    movi32 a2, 4097
+; CHECK-NEXT:    movi16 a3, 0
+; CHECK-NEXT:    jsr16 t0
+; CHECK-NEXT:    ld32.w lr, (sp, 0) # 4-byte Folded Reload
+; CHECK-NEXT:    addi16 sp, sp, 4
+; CHECK-NEXT:    rts16
+; CHECK-NEXT:    .p2align 1
+; CHECK-NEXT:  # %bb.1:
+; CHECK-NEXT:    .p2align 2, 0x0
+; CHECK-NEXT:  .LCPI12_0:
+; CHECK-NEXT:    .long __muldi3
+;
+; GENERIC-LABEL: mul_i64_4097:
+; GENERIC:       # %bb.0: # %entry
+; GENERIC-NEXT:    subi16 sp, sp, 8
+; GENERIC-NEXT:    .cfi_def_cfa_offset 8
+; GENERIC-NEXT:    st16.w l0, (sp, 4) # 4-byte Folded Spill
+; GENERIC-NEXT:    st32.w lr, (sp, 0) # 4-byte Folded Spill
+; GENERIC-NEXT:    .cfi_offset l0, -4
+; GENERIC-NEXT:    .cfi_offset lr, -8
+; GENERIC-NEXT:    subi16 sp, sp, 4
+; GENERIC-NEXT:    .cfi_def_cfa_offset 12
+; GENERIC-NEXT:    movi16 a2, 0
+; GENERIC-NEXT:    lsli16 a3, a2, 24
+; GENERIC-NEXT:    lsli16 a2, a2, 16
+; GENERIC-NEXT:    or16 a2, a3
+; GENERIC-NEXT:    movi16 a3, 16
+; GENERIC-NEXT:    lsli16 a3, a3, 8
+; GENERIC-NEXT:    or16 a3, a2
+; GENERIC-NEXT:    movi16 a2, 1
+; GENERIC-NEXT:    or16 a2, a3
+; GENERIC-NEXT:    lrw32 l0, [.LCPI12_0]
+; GENERIC-NEXT:    movi16 a3, 0
+; GENERIC-NEXT:    jsr16 l0
+; GENERIC-NEXT:    addi16 sp, sp, 4
+; GENERIC-NEXT:    ld32.w lr, (sp, 0) # 4-byte Folded Reload
+; GENERIC-NEXT:    ld16.w l0, (sp, 4) # 4-byte Folded Reload
+; GENERIC-NEXT:    addi16 sp, sp, 8
+; GENERIC-NEXT:    rts16
+; GENERIC-NEXT:    .p2align 1
+; GENERIC-NEXT:  # %bb.1:
+; GENERIC-NEXT:    .p2align 2, 0x0
+; GENERIC-NEXT:  .LCPI12_0:
+; GENERIC-NEXT:    .long __muldi3
+entry:
+  %y = mul nsw i64 %x, 4097
+  ret i64 %y
+}

From 6d05f3f56e22ab9420c71062e2a0c8923aa2a2d7 Mon Sep 17 00:00:00 2001
From: Ben Shi <2283975856@qq.com>
Date: Mon, 19 Jun 2023 17:14:41 +0800
Subject: [PATCH 096/130] [CSKY] Optimize multiplication with immediates

Try to break a multiplication with a specific immediate to
an/a addition/subtraction of left shifts.

Reviewed By: zixuan-wu

Differential Revision: https://reviews.llvm.org/D153106
---
 llvm/lib/Target/CSKY/CSKYISelLowering.cpp |  25 ++++
 llvm/lib/Target/CSKY/CSKYISelLowering.h   |   3 +
 llvm/lib/Target/CSKY/CSKYSubtarget.h      |   2 +
 llvm/test/CodeGen/CSKY/base-i.ll          |  62 +++-----
 llvm/test/CodeGen/CSKY/mul-imm.ll         | 169 ++++++----------------
 5 files changed, 102 insertions(+), 159 deletions(-)

diff --git a/llvm/lib/Target/CSKY/CSKYISelLowering.cpp b/llvm/lib/Target/CSKY/CSKYISelLowering.cpp
index 4a2df7cd4d021..5d21aab513dde 100644
--- a/llvm/lib/Target/CSKY/CSKYISelLowering.cpp
+++ b/llvm/lib/Target/CSKY/CSKYISelLowering.cpp
@@ -1376,3 +1376,28 @@ SDValue CSKYTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
 
   return V;
 }
+
+bool CSKYTargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT,
+                                                SDValue C) const {
+  if (!VT.isScalarInteger())
+    return false;
+
+  // Omit if data size exceeds.
+  if (VT.getSizeInBits() > Subtarget.XLen)
+    return false;
+
+  if (auto *ConstNode = dyn_cast<ConstantSDNode>(C.getNode())) {
+    const APInt &Imm = ConstNode->getAPIntValue();
+    // Break MULT to LSLI + ADDU/SUBU.
+    if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
+        (1 - Imm).isPowerOf2())
+      return true;
+    // Only break MULT for sub targets without MULT32, since an extra
+    // instruction will be generated against the above 3 cases. We leave it
+    // unchanged on sub targets with MULT32, since not sure it is better.
+    if (!Subtarget.hasE2() && (-1 - Imm).isPowerOf2())
+      return true;
+  }
+
+  return false;
+}
diff --git a/llvm/lib/Target/CSKY/CSKYISelLowering.h b/llvm/lib/Target/CSKY/CSKYISelLowering.h
index d0abc7e9a7e41..c724882c6042f 100644
--- a/llvm/lib/Target/CSKY/CSKYISelLowering.h
+++ b/llvm/lib/Target/CSKY/CSKYISelLowering.h
@@ -173,6 +173,9 @@ class CSKYTargetLowering : public TargetLowering {
 
   CCAssignFn *CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg) const;
   CCAssignFn *CCAssignFnForReturn(CallingConv::ID CC, bool IsVarArg) const;
+
+  bool decomposeMulByConstant(LLVMContext &Context, EVT VT,
+                              SDValue C) const override;
 };
 
 } // namespace llvm
diff --git a/llvm/lib/Target/CSKY/CSKYSubtarget.h b/llvm/lib/Target/CSKY/CSKYSubtarget.h
index 9e7ad00c0a504..b8be347935ac7 100644
--- a/llvm/lib/Target/CSKY/CSKYSubtarget.h
+++ b/llvm/lib/Target/CSKY/CSKYSubtarget.h
@@ -206,6 +206,8 @@ class CSKYSubtarget : public CSKYGenSubtargetInfo {
   bool isCK810V() const { return CSKYProcFamily == CK810V; }
   bool isCK860() const { return CSKYProcFamily == CK860; }
   bool isCK860V() const { return CSKYProcFamily == CK860V; }
+
+  const unsigned XLen = 32;
 };
 } // namespace llvm
 
diff --git a/llvm/test/CodeGen/CSKY/base-i.ll b/llvm/test/CodeGen/CSKY/base-i.ll
index 132a39f92c4b4..6e612cb3f5c4b 100644
--- a/llvm/test/CodeGen/CSKY/base-i.ll
+++ b/llvm/test/CodeGen/CSKY/base-i.ll
@@ -480,8 +480,8 @@ entry:
 define i32 @mulRI_X(i32 %x) {
 ; CHECK-LABEL: mulRI_X:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movi32 a1, 4097
-; CHECK-NEXT:    mult16 a0, a1
+; CHECK-NEXT:    lsli16 a1, a0, 12
+; CHECK-NEXT:    addu16 a0, a1
 ; CHECK-NEXT:    rts16
 ;
 ; GENERIC-LABEL: mulRI_X:
@@ -489,16 +489,8 @@ define i32 @mulRI_X(i32 %x) {
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 0
 ; GENERIC-NEXT:    subi16 sp, sp, 4
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 4
-; GENERIC-NEXT:    movi16 a1, 0
-; GENERIC-NEXT:    lsli16 a2, a1, 24
-; GENERIC-NEXT:    lsli16 a1, a1, 16
-; GENERIC-NEXT:    or16 a1, a2
-; GENERIC-NEXT:    movi16 a2, 16
-; GENERIC-NEXT:    lsli16 a2, a2, 8
-; GENERIC-NEXT:    or16 a2, a1
-; GENERIC-NEXT:    movi16 a1, 1
-; GENERIC-NEXT:    or16 a1, a2
-; GENERIC-NEXT:    mult16 a0, a1
+; GENERIC-NEXT:    lsli16 a1, a0, 12
+; GENERIC-NEXT:    addu16 a0, a1, a0
 ; GENERIC-NEXT:    addi16 sp, sp, 4
 ; GENERIC-NEXT:    rts16
 entry:
@@ -528,8 +520,7 @@ entry:
 define i16 @MUL_SHORT_I(i16 %x) {
 ; CHECK-LABEL: MUL_SHORT_I:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movi16 a1, 3
-; CHECK-NEXT:    mult16 a0, a1
+; CHECK-NEXT:    ixh32 a0, a0, a0
 ; CHECK-NEXT:    rts16
 ;
 ; GENERIC-LABEL: MUL_SHORT_I:
@@ -537,8 +528,8 @@ define i16 @MUL_SHORT_I(i16 %x) {
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 0
 ; GENERIC-NEXT:    subi16 sp, sp, 4
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 4
-; GENERIC-NEXT:    movi16 a1, 3
-; GENERIC-NEXT:    mult16 a0, a1
+; GENERIC-NEXT:    lsli16 a1, a0, 1
+; GENERIC-NEXT:    addu16 a0, a1, a0
 ; GENERIC-NEXT:    addi16 sp, sp, 4
 ; GENERIC-NEXT:    rts16
 entry:
@@ -568,9 +559,9 @@ entry:
 define i8 @MUL_CHAR_I(i8 %x) {
 ; CHECK-LABEL: MUL_CHAR_I:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movih32 a1, 65535
-; CHECK-NEXT:    ori32 a1, a1, 65533
-; CHECK-NEXT:    mult16 a0, a1
+; CHECK-NEXT:    ixh32 a0, a0, a0
+; CHECK-NEXT:    movi16 a1, 0
+; CHECK-NEXT:    subu16 a0, a1, a0
 ; CHECK-NEXT:    rts16
 ;
 ; GENERIC-LABEL: MUL_CHAR_I:
@@ -578,15 +569,10 @@ define i8 @MUL_CHAR_I(i8 %x) {
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 0
 ; GENERIC-NEXT:    subi16 sp, sp, 4
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 4
-; GENERIC-NEXT:    movi16 a1, 255
-; GENERIC-NEXT:    lsli16 a2, a1, 24
-; GENERIC-NEXT:    lsli16 a3, a1, 16
-; GENERIC-NEXT:    or16 a3, a2
-; GENERIC-NEXT:    lsli16 a1, a1, 8
-; GENERIC-NEXT:    or16 a1, a3
-; GENERIC-NEXT:    movi16 a2, 253
-; GENERIC-NEXT:    or16 a2, a1
-; GENERIC-NEXT:    mult16 a0, a2
+; GENERIC-NEXT:    lsli16 a1, a0, 1
+; GENERIC-NEXT:    addu16 a0, a1, a0
+; GENERIC-NEXT:    movi16 a1, 0
+; GENERIC-NEXT:    subu16 a0, a1, a0
 ; GENERIC-NEXT:    addi16 sp, sp, 4
 ; GENERIC-NEXT:    rts16
 entry:
@@ -619,7 +605,7 @@ define i32 @udivRR(i32 %x, i32 %y) {
 ; GENERIC-NEXT:    rts16
 ; GENERIC-NEXT:    .p2align 1
 ; GENERIC-NEXT:  # %bb.1:
-; GENERIC-NEXT:    .p2align 2
+; GENERIC-NEXT:    .p2align 2, 0x0
 ; GENERIC-NEXT:  .LCPI25_0:
 ; GENERIC-NEXT:    .long __udivsi3
 entry:
@@ -651,7 +637,7 @@ define i32 @udivRI(i32 %x) {
 ; GENERIC-NEXT:    rts16
 ; GENERIC-NEXT:    .p2align 1
 ; GENERIC-NEXT:  # %bb.1:
-; GENERIC-NEXT:    .p2align 2
+; GENERIC-NEXT:    .p2align 2, 0x0
 ; GENERIC-NEXT:  .LCPI26_0:
 ; GENERIC-NEXT:    .long __udivsi3
 entry:
@@ -691,7 +677,7 @@ define i32 @udivRI_X(i32 %x) {
 ; GENERIC-NEXT:    rts16
 ; GENERIC-NEXT:    .p2align 1
 ; GENERIC-NEXT:  # %bb.1:
-; GENERIC-NEXT:    .p2align 2
+; GENERIC-NEXT:    .p2align 2, 0x0
 ; GENERIC-NEXT:  .LCPI27_0:
 ; GENERIC-NEXT:    .long __udivsi3
 entry:
@@ -738,7 +724,7 @@ define i16 @UDIV_SHORT(i16 %x, i16 %y) {
 ; GENERIC-NEXT:    rts16
 ; GENERIC-NEXT:    .p2align 1
 ; GENERIC-NEXT:  # %bb.1:
-; GENERIC-NEXT:    .p2align 2
+; GENERIC-NEXT:    .p2align 2, 0x0
 ; GENERIC-NEXT:  .LCPI28_0:
 ; GENERIC-NEXT:    .long __udivsi3
 entry:
@@ -812,7 +798,7 @@ define i8 @UDIV_CHAR(i8 %x, i8 %y) {
 ; GENERIC-NEXT:    rts16
 ; GENERIC-NEXT:    .p2align 1
 ; GENERIC-NEXT:  # %bb.1:
-; GENERIC-NEXT:    .p2align 2
+; GENERIC-NEXT:    .p2align 2, 0x0
 ; GENERIC-NEXT:  .LCPI30_0:
 ; GENERIC-NEXT:    .long __udivsi3
 entry:
@@ -871,7 +857,7 @@ define i32 @sdivRR(i32 %x, i32 %y) {
 ; GENERIC-NEXT:    rts16
 ; GENERIC-NEXT:    .p2align 1
 ; GENERIC-NEXT:  # %bb.1:
-; GENERIC-NEXT:    .p2align 2
+; GENERIC-NEXT:    .p2align 2, 0x0
 ; GENERIC-NEXT:  .LCPI32_0:
 ; GENERIC-NEXT:    .long __divsi3
 entry:
@@ -903,7 +889,7 @@ define i32 @sdivRI(i32 %x) {
 ; GENERIC-NEXT:    rts16
 ; GENERIC-NEXT:    .p2align 1
 ; GENERIC-NEXT:  # %bb.1:
-; GENERIC-NEXT:    .p2align 2
+; GENERIC-NEXT:    .p2align 2, 0x0
 ; GENERIC-NEXT:  .LCPI33_0:
 ; GENERIC-NEXT:    .long __divsi3
 entry:
@@ -943,7 +929,7 @@ define i32 @sdivRI_X(i32 %x) {
 ; GENERIC-NEXT:    rts16
 ; GENERIC-NEXT:    .p2align 1
 ; GENERIC-NEXT:  # %bb.1:
-; GENERIC-NEXT:    .p2align 2
+; GENERIC-NEXT:    .p2align 2, 0x0
 ; GENERIC-NEXT:  .LCPI34_0:
 ; GENERIC-NEXT:    .long __divsi3
 entry:
@@ -978,7 +964,7 @@ define i16 @SDIV_SHORT(i16 %x, i16 %y) {
 ; GENERIC-NEXT:    rts16
 ; GENERIC-NEXT:    .p2align 1
 ; GENERIC-NEXT:  # %bb.1:
-; GENERIC-NEXT:    .p2align 2
+; GENERIC-NEXT:    .p2align 2, 0x0
 ; GENERIC-NEXT:  .LCPI35_0:
 ; GENERIC-NEXT:    .long __divsi3
 entry:
@@ -1050,7 +1036,7 @@ define i8 @SDIV_CHAR(i8 %x, i8 %y) {
 ; GENERIC-NEXT:    rts16
 ; GENERIC-NEXT:    .p2align 1
 ; GENERIC-NEXT:  # %bb.1:
-; GENERIC-NEXT:    .p2align 2
+; GENERIC-NEXT:    .p2align 2, 0x0
 ; GENERIC-NEXT:  .LCPI37_0:
 ; GENERIC-NEXT:    .long __divsi3
 entry:
diff --git a/llvm/test/CodeGen/CSKY/mul-imm.ll b/llvm/test/CodeGen/CSKY/mul-imm.ll
index c87e527430de5..f9491e4bae75e 100644
--- a/llvm/test/CodeGen/CSKY/mul-imm.ll
+++ b/llvm/test/CodeGen/CSKY/mul-imm.ll
@@ -8,8 +8,8 @@
 define i32 @mul_i32_4097(i32 %x) {
 ; CHECK-LABEL: mul_i32_4097:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movi32 a1, 4097
-; CHECK-NEXT:    mult16 a0, a1
+; CHECK-NEXT:    lsli16 a1, a0, 12
+; CHECK-NEXT:    addu16 a0, a1
 ; CHECK-NEXT:    rts16
 ;
 ; GENERIC-LABEL: mul_i32_4097:
@@ -17,16 +17,8 @@ define i32 @mul_i32_4097(i32 %x) {
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 0
 ; GENERIC-NEXT:    subi16 sp, sp, 4
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 4
-; GENERIC-NEXT:    movi16 a1, 0
-; GENERIC-NEXT:    lsli16 a2, a1, 24
-; GENERIC-NEXT:    lsli16 a1, a1, 16
-; GENERIC-NEXT:    or16 a1, a2
-; GENERIC-NEXT:    movi16 a2, 16
-; GENERIC-NEXT:    lsli16 a2, a2, 8
-; GENERIC-NEXT:    or16 a2, a1
-; GENERIC-NEXT:    movi16 a1, 1
-; GENERIC-NEXT:    or16 a1, a2
-; GENERIC-NEXT:    mult16 a0, a1
+; GENERIC-NEXT:    lsli16 a1, a0, 12
+; GENERIC-NEXT:    addu16 a0, a1, a0
 ; GENERIC-NEXT:    addi16 sp, sp, 4
 ; GENERIC-NEXT:    rts16
 entry:
@@ -37,8 +29,8 @@ entry:
 define i32 @mul_i32_4095(i32 %x) {
 ; CHECK-LABEL: mul_i32_4095:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movi32 a1, 4095
-; CHECK-NEXT:    mult16 a0, a1
+; CHECK-NEXT:    lsli16 a1, a0, 12
+; CHECK-NEXT:    subu16 a0, a1, a0
 ; CHECK-NEXT:    rts16
 ;
 ; GENERIC-LABEL: mul_i32_4095:
@@ -46,16 +38,8 @@ define i32 @mul_i32_4095(i32 %x) {
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 0
 ; GENERIC-NEXT:    subi16 sp, sp, 4
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 4
-; GENERIC-NEXT:    movi16 a1, 0
-; GENERIC-NEXT:    lsli16 a2, a1, 24
-; GENERIC-NEXT:    lsli16 a1, a1, 16
-; GENERIC-NEXT:    or16 a1, a2
-; GENERIC-NEXT:    movi16 a2, 15
-; GENERIC-NEXT:    lsli16 a2, a2, 8
-; GENERIC-NEXT:    or16 a2, a1
-; GENERIC-NEXT:    movi16 a1, 255
-; GENERIC-NEXT:    or16 a1, a2
-; GENERIC-NEXT:    mult16 a0, a1
+; GENERIC-NEXT:    lsli16 a1, a0, 12
+; GENERIC-NEXT:    subu16 a0, a1, a0
 ; GENERIC-NEXT:    addi16 sp, sp, 4
 ; GENERIC-NEXT:    rts16
 entry:
@@ -66,9 +50,8 @@ entry:
 define i32 @mul_i32_minus_4095(i32 %x) {
 ; CHECK-LABEL: mul_i32_minus_4095:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movih32 a1, 65535
-; CHECK-NEXT:    ori32 a1, a1, 61441
-; CHECK-NEXT:    mult16 a0, a1
+; CHECK-NEXT:    lsli16 a1, a0, 12
+; CHECK-NEXT:    subu16 a0, a1
 ; CHECK-NEXT:    rts16
 ;
 ; GENERIC-LABEL: mul_i32_minus_4095:
@@ -76,16 +59,8 @@ define i32 @mul_i32_minus_4095(i32 %x) {
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 0
 ; GENERIC-NEXT:    subi16 sp, sp, 4
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 4
-; GENERIC-NEXT:    movi16 a1, 255
-; GENERIC-NEXT:    lsli16 a2, a1, 24
-; GENERIC-NEXT:    lsli16 a1, a1, 16
-; GENERIC-NEXT:    or16 a1, a2
-; GENERIC-NEXT:    movi16 a2, 240
-; GENERIC-NEXT:    lsli16 a2, a2, 8
-; GENERIC-NEXT:    or16 a2, a1
-; GENERIC-NEXT:    movi16 a1, 1
-; GENERIC-NEXT:    or16 a1, a2
-; GENERIC-NEXT:    mult16 a0, a1
+; GENERIC-NEXT:    lsli16 a1, a0, 12
+; GENERIC-NEXT:    subu16 a0, a0, a1
 ; GENERIC-NEXT:    addi16 sp, sp, 4
 ; GENERIC-NEXT:    rts16
 entry:
@@ -96,8 +71,8 @@ entry:
 define i16 @mul_i16_4097(i16 %x) {
 ; CHECK-LABEL: mul_i16_4097:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movi32 a1, 4097
-; CHECK-NEXT:    mult16 a0, a1
+; CHECK-NEXT:    lsli16 a1, a0, 12
+; CHECK-NEXT:    addu16 a0, a1
 ; CHECK-NEXT:    rts16
 ;
 ; GENERIC-LABEL: mul_i16_4097:
@@ -105,16 +80,8 @@ define i16 @mul_i16_4097(i16 %x) {
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 0
 ; GENERIC-NEXT:    subi16 sp, sp, 4
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 4
-; GENERIC-NEXT:    movi16 a1, 0
-; GENERIC-NEXT:    lsli16 a2, a1, 24
-; GENERIC-NEXT:    lsli16 a1, a1, 16
-; GENERIC-NEXT:    or16 a1, a2
-; GENERIC-NEXT:    movi16 a2, 16
-; GENERIC-NEXT:    lsli16 a2, a2, 8
-; GENERIC-NEXT:    or16 a2, a1
-; GENERIC-NEXT:    movi16 a1, 1
-; GENERIC-NEXT:    or16 a1, a2
-; GENERIC-NEXT:    mult16 a0, a1
+; GENERIC-NEXT:    lsli16 a1, a0, 12
+; GENERIC-NEXT:    addu16 a0, a1, a0
 ; GENERIC-NEXT:    addi16 sp, sp, 4
 ; GENERIC-NEXT:    rts16
 entry:
@@ -125,8 +92,8 @@ entry:
 define i16 @mul_i16_4095(i16 %x) {
 ; CHECK-LABEL: mul_i16_4095:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movi32 a1, 4095
-; CHECK-NEXT:    mult16 a0, a1
+; CHECK-NEXT:    lsli16 a1, a0, 12
+; CHECK-NEXT:    subu16 a0, a1, a0
 ; CHECK-NEXT:    rts16
 ;
 ; GENERIC-LABEL: mul_i16_4095:
@@ -134,16 +101,8 @@ define i16 @mul_i16_4095(i16 %x) {
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 0
 ; GENERIC-NEXT:    subi16 sp, sp, 4
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 4
-; GENERIC-NEXT:    movi16 a1, 0
-; GENERIC-NEXT:    lsli16 a2, a1, 24
-; GENERIC-NEXT:    lsli16 a1, a1, 16
-; GENERIC-NEXT:    or16 a1, a2
-; GENERIC-NEXT:    movi16 a2, 15
-; GENERIC-NEXT:    lsli16 a2, a2, 8
-; GENERIC-NEXT:    or16 a2, a1
-; GENERIC-NEXT:    movi16 a1, 255
-; GENERIC-NEXT:    or16 a1, a2
-; GENERIC-NEXT:    mult16 a0, a1
+; GENERIC-NEXT:    lsli16 a1, a0, 12
+; GENERIC-NEXT:    subu16 a0, a1, a0
 ; GENERIC-NEXT:    addi16 sp, sp, 4
 ; GENERIC-NEXT:    rts16
 entry:
@@ -154,9 +113,8 @@ entry:
 define i16 @mul_i16_minus_4095(i16 %x) {
 ; CHECK-LABEL: mul_i16_minus_4095:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movih32 a1, 65535
-; CHECK-NEXT:    ori32 a1, a1, 61441
-; CHECK-NEXT:    mult16 a0, a1
+; CHECK-NEXT:    lsli16 a1, a0, 12
+; CHECK-NEXT:    subu16 a0, a1
 ; CHECK-NEXT:    rts16
 ;
 ; GENERIC-LABEL: mul_i16_minus_4095:
@@ -164,16 +122,8 @@ define i16 @mul_i16_minus_4095(i16 %x) {
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 0
 ; GENERIC-NEXT:    subi16 sp, sp, 4
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 4
-; GENERIC-NEXT:    movi16 a1, 255
-; GENERIC-NEXT:    lsli16 a2, a1, 24
-; GENERIC-NEXT:    lsli16 a1, a1, 16
-; GENERIC-NEXT:    or16 a1, a2
-; GENERIC-NEXT:    movi16 a2, 240
-; GENERIC-NEXT:    lsli16 a2, a2, 8
-; GENERIC-NEXT:    or16 a2, a1
-; GENERIC-NEXT:    movi16 a1, 1
-; GENERIC-NEXT:    or16 a1, a2
-; GENERIC-NEXT:    mult16 a0, a1
+; GENERIC-NEXT:    lsli16 a1, a0, 12
+; GENERIC-NEXT:    subu16 a0, a0, a1
 ; GENERIC-NEXT:    addi16 sp, sp, 4
 ; GENERIC-NEXT:    rts16
 entry:
@@ -184,8 +134,8 @@ entry:
 define i8 @mul_i8_65(i8 %x) {
 ; CHECK-LABEL: mul_i8_65:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movi16 a1, 65
-; CHECK-NEXT:    mult16 a0, a1
+; CHECK-NEXT:    lsli16 a1, a0, 6
+; CHECK-NEXT:    addu16 a0, a1
 ; CHECK-NEXT:    rts16
 ;
 ; GENERIC-LABEL: mul_i8_65:
@@ -193,8 +143,8 @@ define i8 @mul_i8_65(i8 %x) {
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 0
 ; GENERIC-NEXT:    subi16 sp, sp, 4
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 4
-; GENERIC-NEXT:    movi16 a1, 65
-; GENERIC-NEXT:    mult16 a0, a1
+; GENERIC-NEXT:    lsli16 a1, a0, 6
+; GENERIC-NEXT:    addu16 a0, a1, a0
 ; GENERIC-NEXT:    addi16 sp, sp, 4
 ; GENERIC-NEXT:    rts16
 entry:
@@ -205,8 +155,8 @@ entry:
 define i8 @mul_i8_63(i8 %x) {
 ; CHECK-LABEL: mul_i8_63:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movi16 a1, 63
-; CHECK-NEXT:    mult16 a0, a1
+; CHECK-NEXT:    lsli16 a1, a0, 6
+; CHECK-NEXT:    subu16 a0, a1, a0
 ; CHECK-NEXT:    rts16
 ;
 ; GENERIC-LABEL: mul_i8_63:
@@ -214,8 +164,8 @@ define i8 @mul_i8_63(i8 %x) {
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 0
 ; GENERIC-NEXT:    subi16 sp, sp, 4
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 4
-; GENERIC-NEXT:    movi16 a1, 63
-; GENERIC-NEXT:    mult16 a0, a1
+; GENERIC-NEXT:    lsli16 a1, a0, 6
+; GENERIC-NEXT:    subu16 a0, a1, a0
 ; GENERIC-NEXT:    addi16 sp, sp, 4
 ; GENERIC-NEXT:    rts16
 entry:
@@ -226,9 +176,8 @@ entry:
 define i8 @mul_i8_minus_63(i8 %x) {
 ; CHECK-LABEL: mul_i8_minus_63:
 ; CHECK:       # %bb.0: # %entry
-; CHECK-NEXT:    movih32 a1, 65535
-; CHECK-NEXT:    ori32 a1, a1, 65473
-; CHECK-NEXT:    mult16 a0, a1
+; CHECK-NEXT:    lsli16 a1, a0, 6
+; CHECK-NEXT:    subu16 a0, a1
 ; CHECK-NEXT:    rts16
 ;
 ; GENERIC-LABEL: mul_i8_minus_63:
@@ -236,15 +185,8 @@ define i8 @mul_i8_minus_63(i8 %x) {
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 0
 ; GENERIC-NEXT:    subi16 sp, sp, 4
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 4
-; GENERIC-NEXT:    movi16 a1, 255
-; GENERIC-NEXT:    lsli16 a2, a1, 24
-; GENERIC-NEXT:    lsli16 a3, a1, 16
-; GENERIC-NEXT:    or16 a3, a2
-; GENERIC-NEXT:    lsli16 a1, a1, 8
-; GENERIC-NEXT:    or16 a1, a3
-; GENERIC-NEXT:    movi16 a2, 193
-; GENERIC-NEXT:    or16 a2, a1
-; GENERIC-NEXT:    mult16 a0, a2
+; GENERIC-NEXT:    lsli16 a1, a0, 6
+; GENERIC-NEXT:    subu16 a0, a0, a1
 ; GENERIC-NEXT:    addi16 sp, sp, 4
 ; GENERIC-NEXT:    rts16
 entry:
@@ -265,15 +207,10 @@ define i32 @mul_i32_minus_4097(i32 %x) {
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 0
 ; GENERIC-NEXT:    subi16 sp, sp, 4
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 4
-; GENERIC-NEXT:    movi16 a1, 255
-; GENERIC-NEXT:    lsli16 a2, a1, 24
-; GENERIC-NEXT:    lsli16 a3, a1, 16
-; GENERIC-NEXT:    or16 a3, a2
-; GENERIC-NEXT:    movi16 a2, 239
-; GENERIC-NEXT:    lsli16 a2, a2, 8
-; GENERIC-NEXT:    or16 a2, a3
-; GENERIC-NEXT:    or16 a2, a1
-; GENERIC-NEXT:    mult16 a0, a2
+; GENERIC-NEXT:    lsli16 a1, a0, 12
+; GENERIC-NEXT:    addu16 a0, a1, a0
+; GENERIC-NEXT:    movi16 a1, 0
+; GENERIC-NEXT:    subu16 a0, a1, a0
 ; GENERIC-NEXT:    addi16 sp, sp, 4
 ; GENERIC-NEXT:    rts16
 entry:
@@ -294,15 +231,10 @@ define i16 @mul_i16_minus_4097(i16 %x) {
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 0
 ; GENERIC-NEXT:    subi16 sp, sp, 4
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 4
-; GENERIC-NEXT:    movi16 a1, 255
-; GENERIC-NEXT:    lsli16 a2, a1, 24
-; GENERIC-NEXT:    lsli16 a3, a1, 16
-; GENERIC-NEXT:    or16 a3, a2
-; GENERIC-NEXT:    movi16 a2, 239
-; GENERIC-NEXT:    lsli16 a2, a2, 8
-; GENERIC-NEXT:    or16 a2, a3
-; GENERIC-NEXT:    or16 a2, a1
-; GENERIC-NEXT:    mult16 a0, a2
+; GENERIC-NEXT:    lsli16 a1, a0, 12
+; GENERIC-NEXT:    addu16 a0, a1, a0
+; GENERIC-NEXT:    movi16 a1, 0
+; GENERIC-NEXT:    subu16 a0, a1, a0
 ; GENERIC-NEXT:    addi16 sp, sp, 4
 ; GENERIC-NEXT:    rts16
 entry:
@@ -323,15 +255,10 @@ define i8 @mul_i8_minus_65(i8 %x) {
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 0
 ; GENERIC-NEXT:    subi16 sp, sp, 4
 ; GENERIC-NEXT:    .cfi_def_cfa_offset 4
-; GENERIC-NEXT:    movi16 a1, 255
-; GENERIC-NEXT:    lsli16 a2, a1, 24
-; GENERIC-NEXT:    lsli16 a3, a1, 16
-; GENERIC-NEXT:    or16 a3, a2
-; GENERIC-NEXT:    lsli16 a1, a1, 8
-; GENERIC-NEXT:    or16 a1, a3
-; GENERIC-NEXT:    movi16 a2, 191
-; GENERIC-NEXT:    or16 a2, a1
-; GENERIC-NEXT:    mult16 a0, a2
+; GENERIC-NEXT:    lsli16 a1, a0, 6
+; GENERIC-NEXT:    addu16 a0, a1, a0
+; GENERIC-NEXT:    movi16 a1, 0
+; GENERIC-NEXT:    subu16 a0, a1, a0
 ; GENERIC-NEXT:    addi16 sp, sp, 4
 ; GENERIC-NEXT:    rts16
 entry:

From 189d9447b182e80e284561382f43bbf5ae0bb031 Mon Sep 17 00:00:00 2001
From: Nuno Lopes <nuno.lopes@tecnico.ulisboa.pt>
Date: Tue, 20 Jun 2023 09:07:08 +0100
Subject: [PATCH 097/130] [docs] Fix GEP faq references to undefined behavior
 [NFC]

---
 llvm/docs/GetElementPtr.rst | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/llvm/docs/GetElementPtr.rst b/llvm/docs/GetElementPtr.rst
index 49076ab6f5997..b08c99c6cee69 100644
--- a/llvm/docs/GetElementPtr.rst
+++ b/llvm/docs/GetElementPtr.rst
@@ -132,8 +132,8 @@ The obtuse part of this example is in the cases of ``%idx2`` and ``%idx3``. They
 result in the computation of addresses that point to memory past the end of the
 ``@MyVar`` global, which is only one ``i32`` long, not three ``i32``\s long.
 While this is legal in LLVM, it is inadvisable because any load or store with
-the pointer that results from these GEP instructions would produce undefined
-results.
+the pointer that results from these GEP instructions would trigger undefined
+behavior (UB).
 
 Why is the extra 0 index required?
 ----------------------------------
@@ -345,7 +345,7 @@ the static array type bounds are respected.
 The second sense of being out of bounds is computing an address that's beyond
 the actual underlying allocated object.
 
-With the ``inbounds`` keyword, the result value of the GEP is undefined if the
+With the ``inbounds`` keyword, the result value of the GEP is ``poison`` if the
 address is outside the actual underlying allocated object and not the address
 one-past-the-end.
 
@@ -428,8 +428,8 @@ evaluating the implied two's complement integer computation. However, since
 there's no guarantee of where an object will be allocated in the address space,
 such values have limited meaning.
 
-If the GEP has the ``inbounds`` keyword, the result value is undefined (a "trap
-value") if the GEP overflows (i.e. wraps around the end of the address space).
+If the GEP has the ``inbounds`` keyword, the result value is ``poison``
+if the GEP overflows (i.e. wraps around the end of the address space).
 
 As such, there are some ramifications of this for inbounds GEPs: scales implied
 by array/vector/pointer indices are always known to be "nsw" since they are

From c146df9618764d5548c38ce61edc45147094b26f Mon Sep 17 00:00:00 2001
From: Michael Buch <michaelbuch12@gmail.com>
Date: Mon, 19 Jun 2023 13:59:32 +0100
Subject: [PATCH 098/130] [clang][DebugInfo] Emit DW_AT_deleted on any deleted
 member function

Currently we emit `DW_AT_deleted` for `deleted` special-member
functions (i.e., ctors/dtors). However, in C++ one can mark any
member function as deleted. This patch expands the set of member
functions for which we emit `DW_AT_deleted`.

The DWARFv5 spec section 5.7.8 says:
```
<non-normative>
In C++, a member function may be declared as deleted. This prevents the compiler from
generating a default implementation of a special member function such as a constructor
or destructor, and can affect overload resolution when used on other member functions.
</non-normative>

If the member function entry has been declared as deleted, then that entry has a
DW_AT_deleted attribute.
```

Thus this change is conforming.

Differential Revision: https://reviews.llvm.org/D153282
---
 clang/lib/CodeGen/CGDebugInfo.cpp            | 23 ++--------------
 clang/test/CodeGenCXX/debug-info-deleted.cpp |  5 ++++
 llvm/test/DebugInfo/X86/DW_AT_deleted.ll     | 28 ++++++++++++++++----
 3 files changed, 30 insertions(+), 26 deletions(-)

diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index 2fd2227720a2a..f049a682cfed6 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -1945,27 +1945,8 @@ llvm::DISubprogram *CGDebugInfo::CreateCXXMemberFunction(
     ContainingType = RecordTy;
   }
 
-  // We're checking for deleted C++ special member functions
-  // [Ctors,Dtors, Copy/Move]
-  auto checkAttrDeleted = [&](const auto *Method) {
-    if (Method->getCanonicalDecl()->isDeleted())
-      SPFlags |= llvm::DISubprogram::SPFlagDeleted;
-  };
-
-  switch (Method->getKind()) {
-
-  case Decl::CXXConstructor:
-  case Decl::CXXDestructor:
-    checkAttrDeleted(Method);
-    break;
-  case Decl::CXXMethod:
-    if (Method->isCopyAssignmentOperator() ||
-        Method->isMoveAssignmentOperator())
-      checkAttrDeleted(Method);
-    break;
-  default:
-    break;
-  }
+  if (Method->getCanonicalDecl()->isDeleted())
+    SPFlags |= llvm::DISubprogram::SPFlagDeleted;
 
   if (Method->isNoReturn())
     Flags |= llvm::DINode::FlagNoReturn;
diff --git a/clang/test/CodeGenCXX/debug-info-deleted.cpp b/clang/test/CodeGenCXX/debug-info-deleted.cpp
index d7d0b6dba49ea..564f7fff00fb9 100644
--- a/clang/test/CodeGenCXX/debug-info-deleted.cpp
+++ b/clang/test/CodeGenCXX/debug-info-deleted.cpp
@@ -11,6 +11,8 @@
 // ATTR: DISubprogram(name: "operator=", linkageName: "_ZN7deletedaSERKS_", {{.*}}, flags: DIFlagPublic | DIFlagPrototyped, spFlags: DISPFlagDeleted
 // ATTR: DISubprogram(name: "deleted", {{.*}}, flags: DIFlagPublic | DIFlagPrototyped, spFlags: DISPFlagDeleted
 // ATTR: DISubprogram(name: "operator=", linkageName: "_ZN7deletedaSEOS_", {{.*}}, flags: DIFlagPublic | DIFlagPrototyped, spFlags: DISPFlagDeleted
+// ATTR: DISubprogram(name: "func", {{.*}}, flags: DIFlagPublic | DIFlagPrototyped | DIFlagRValueReference, spFlags: DISPFlagDeleted
+// ATTR: DISubprogram(name: "bar", {{.*}}, flags: DIFlagPublic | DIFlagPrototyped | DIFlagStaticMember, spFlags: DISPFlagDeleted
 // ATTR: DISubprogram(name: "~deleted", {{.*}}, flags: DIFlagPublic | DIFlagPrototyped,
 class deleted {
 public:
@@ -23,6 +25,9 @@ class deleted {
   deleted(deleted &&) = delete;
   deleted &operator=(deleted &&) = delete;
 
+  void func() && = delete;
+  static int bar() = delete;
+
   ~deleted() = default;
 };
 
diff --git a/llvm/test/DebugInfo/X86/DW_AT_deleted.ll b/llvm/test/DebugInfo/X86/DW_AT_deleted.ll
index 805655f25813f..0311458ee5223 100644
--- a/llvm/test/DebugInfo/X86/DW_AT_deleted.ll
+++ b/llvm/test/DebugInfo/X86/DW_AT_deleted.ll
@@ -12,6 +12,9 @@
 ; 
 ;   deleted(deleted &&) = delete;
 ;   deleted &operator=(deleted &&) = delete;
+;
+;   void func() && = delete;
+;   static void bar() = delete;
 ; 
 ;   ~deleted() = default;
 ; };
@@ -48,6 +51,16 @@
 ; CHECK-NEXT: DW_AT_name [DW_FORM_strx1]    (indexed (00000008) string = "operator=")
 ; CHECK:  DW_AT_deleted [DW_FORM_flag_present]  (true)
 
+; CHECK: DW_TAG_subprogram [10]
+; CHECK-NEXT: DW_AT_linkage_name [DW_FORM_strx1]    (indexed (0000000b) string = "_ZNO7deleted4funcEv")
+; CHECK-NEXT: DW_AT_name [DW_FORM_strx1]    (indexed (0000000c) string = "func")
+; CHECK:  DW_AT_deleted [DW_FORM_flag_present]  (true)
+
+; CHECK: DW_TAG_subprogram [11]
+; CHECK-NEXT: DW_AT_linkage_name [DW_FORM_strx1]    (indexed (0000000d) string = "_ZN7deleted3barEv")
+; CHECK-NEXT: DW_AT_name [DW_FORM_strx1]    (indexed (0000000e) string = "bar")
+; CHECK:  DW_AT_deleted [DW_FORM_flag_present]  (true)
+
 ; ModuleID = 'debug-info-deleted.cpp'
 source_filename = "debug-info-deleted.cpp"
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
@@ -58,8 +71,8 @@ target triple = "x86_64-unknown-linux-gnu"
 ; Function Attrs: noinline nounwind optnone uwtable
 define dso_local void @_Z3foov() #0 !dbg !7 {
   %1 = alloca %class.deleted, align 1
-  call void @llvm.dbg.declare(metadata ptr %1, metadata !10, metadata !DIExpression()), !dbg !34
-  ret void, !dbg !35
+  call void @llvm.dbg.declare(metadata ptr %1, metadata !10, metadata !DIExpression()), !dbg !39
+  ret void, !dbg !40
 }
 
 ; Function Attrs: nounwind readnone speculatable willreturn
@@ -84,7 +97,7 @@ attributes #1 = { nounwind readnone speculatable willreturn }
 !9 = !{null}
 !10 = !DILocalVariable(name: "obj1", scope: !7, file: !1, line: 15, type: !11)
 !11 = distinct !DICompositeType(tag: DW_TAG_class_type, name: "deleted", file: !1, line: 1, size: 8, flags: DIFlagTypePassByReference, elements: !12, identifier: "_ZTS7deleted")
-!12 = !{!13, !17, !22, !26, !30, !33}
+!12 = !{!13, !17, !22, !26, !30, !33, !34, !36}
 !13 = !DISubprogram(name: "deleted", scope: !11, file: !1, line: 3, type: !14, scopeLine: 3, flags: DIFlagPublic | DIFlagPrototyped, spFlags: 0)
 !14 = !DISubroutineType(types: !15)
 !15 = !{null, !16}
@@ -106,5 +119,10 @@ attributes #1 = { nounwind readnone speculatable willreturn }
 !31 = !DISubroutineType(types: !32)
 !32 = !{!25, !16, !29}
 !33 = !DISubprogram(name: "~deleted", scope: !11, file: !1, line: 11, type: !14, scopeLine: 11, flags: DIFlagPublic | DIFlagPrototyped, spFlags: 0)
-!34 = !DILocation(line: 15, column: 13, scope: !7)
-!35 = !DILocation(line: 16, column: 3, scope: !7)
+!34 = !DISubprogram(name: "func", linkageName: "_ZNO7deleted4funcEv", scope: !11, file: !1, line: 13, type: !35, scopeLine: 13, flags: DIFlagPublic | DIFlagPrototyped | DIFlagRValueReference, spFlags: DISPFlagDeleted)                                                                                                                                                                                                           
+!35 = !DISubroutineType(flags: DIFlagRValueReference, types: !15)                                                                                                                                                 
+!36 = !DISubprogram(name: "bar", linkageName: "_ZN7deleted3barEv", scope: !11, file: !1, line: 15, type: !37, scopeLine: 15, flags: DIFlagPublic | DIFlagPrototyped | DIFlagStaticMember, spFlags: DISPFlagDeleted)                                                                                                                                                                                                                 
+!37 = !DISubroutineType(types: !38)
+!38 = !{null}
+!39 = !DILocation(line: 15, column: 13, scope: !7)
+!40 = !DILocation(line: 16, column: 3, scope: !7)

From f6be0814c3a5d1b957164be0af16018dc2b9232d Mon Sep 17 00:00:00 2001
From: serge-sans-paille <sguelton@mozilla.com>
Date: Mon, 19 Jun 2023 18:35:03 +0200
Subject: [PATCH 099/130] [llvm-profdata] Fix llvm-profdata help and make sure
 it remains in sync

This makes the new `order` subcommand part of the help.
As a side effect, also make llvm::map_range compatible with plain
arrays.

Differential Revision: https://reviews.llvm.org/D153303
---
 llvm/include/llvm/ADT/STLExtras.h          |  3 +-
 llvm/tools/llvm-profdata/llvm-profdata.cpp | 35 +++++++++++++++-------
 2 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/llvm/include/llvm/ADT/STLExtras.h b/llvm/include/llvm/ADT/STLExtras.h
index 6475ac7976072..6f4c06bf44330 100644
--- a/llvm/include/llvm/ADT/STLExtras.h
+++ b/llvm/include/llvm/ADT/STLExtras.h
@@ -459,7 +459,8 @@ inline mapped_iterator<ItTy, FuncTy> map_iterator(ItTy I, FuncTy F) {
 
 template <class ContainerTy, class FuncTy>
 auto map_range(ContainerTy &&C, FuncTy F) {
-  return make_range(map_iterator(C.begin(), F), map_iterator(C.end(), F));
+  return make_range(map_iterator(std::begin(C), F),
+                    map_iterator(std::end(C), F));
 }
 
 /// A base type of mapped iterator, that is useful for building derived
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
index 7c282e346747e..da10ddcc58c64 100644
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -3084,6 +3084,16 @@ static int order_main(int argc, const char *argv[]) {
   return 0;
 }
 
+typedef int (*llvm_profdata_subcommand)(int, const char *[]);
+
+static std::tuple<StringRef, llvm_profdata_subcommand>
+    llvm_profdata_subcommands[] = {
+        {"merge", merge_main},
+        {"show", show_main},
+        {"order", order_main},
+        {"overlap", overlap_main},
+};
+
 int llvm_profdata_main(int argc, char **argvNonConst,
                        const llvm::ToolContext &) {
   const char **argv = const_cast<const char **>(argvNonConst);
@@ -3091,16 +3101,11 @@ int llvm_profdata_main(int argc, char **argvNonConst,
 
   StringRef ProgName(sys::path::filename(argv[0]));
   if (argc > 1) {
-    int (*func)(int, const char *[]) = nullptr;
 
-    if (strcmp(argv[1], "merge") == 0)
-      func = merge_main;
-    else if (strcmp(argv[1], "show") == 0)
-      func = show_main;
-    else if (strcmp(argv[1], "overlap") == 0)
-      func = overlap_main;
-    else if (strcmp(argv[1], "order") == 0)
-      func = order_main;
+    llvm_profdata_subcommand func = nullptr;
+    for (auto [subcmd_name, subcmd_action] : llvm_profdata_subcommands)
+      if (subcmd_name == argv[1])
+        func = subcmd_action;
 
     if (func) {
       std::string Invocation(ProgName.str() + " " + argv[1]);
@@ -3115,7 +3120,11 @@ int llvm_profdata_main(int argc, char **argvNonConst,
              << "USAGE: " << ProgName << " <command> [args...]\n"
              << "USAGE: " << ProgName << " <command> -help\n\n"
              << "See each individual command --help for more details.\n"
-             << "Available commands: merge, show, overlap\n";
+             << "Available commands: "
+             << join(map_range(llvm_profdata_subcommands,
+                               [](auto const &KV) { return std::get<0>(KV); }),
+                     ", ")
+             << "\n";
       return 0;
     }
 
@@ -3131,6 +3140,10 @@ int llvm_profdata_main(int argc, char **argvNonConst,
   else
     errs() << ProgName << ": Unknown command!\n";
 
-  errs() << "USAGE: " << ProgName << " <merge|show|overlap|order> [args...]\n";
+  errs() << "USAGE: " << ProgName << " <"
+         << join(map_range(llvm_profdata_subcommands,
+                           [](auto const &KV) { return std::get<0>(KV); }),
+                 "|")
+         << "> [args...]\n";
   return 1;
 }

From 5c5eff44ce8191df8e101d4caa16bd24475f5f07 Mon Sep 17 00:00:00 2001
From: Diana Picus <Diana-Magda.Picus@amd.com>
Date: Wed, 7 Jun 2023 13:52:36 +0200
Subject: [PATCH 100/130] [AMDGPU] Start documenting calling conventions. NFC

Add a section to AMDGPUUsage.rst about calling conventions and list the
ones from the CallingConv enum. Full descriptions can come later (help
appreciated).

Differential Revision: https://reviews.llvm.org/D151996
---
 llvm/docs/AMDGPUUsage.rst | 61 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 61 insertions(+)

diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 90f50e487d8ff..67e9780caf0b5 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -1064,6 +1064,67 @@ The AMDGPU backend supports the following LLVM IR attributes.
 
      ======================================= ==========================================================
 
+Calling Conventions
+-------------------
+
+The AMDGPU backend supports the following calling conventions:
+
+  .. table:: AMDGPU Calling Conventions
+     :name: amdgpu-cc
+
+     =============================== ==========================================================
+     Calling Convention              Description
+     =============================== ==========================================================
+     ``ccc``                         The C calling convention. Used by default.
+                                     See :ref:`amdgpu-amdhsa-function-call-convention-non-kernel-functions`
+                                     for more details.
+
+     ``fastcc``                      The fast calling convention. Mostly the same as the ``ccc``.
+
+     ``coldcc``                      The cold calling convention. Mostly the same as the ``ccc``.
+
+     ``amdgpu_cs``                   Used for Mesa/AMDPAL compute shaders.
+                                     ..TODO::
+                                     Describe.
+
+     ``amdgpu_es``                   Used for AMDPAL shader stage before geometry shader if geometry is in
+                                     use. So either the domain (= tessellation evaluation) shader if
+                                     tessellation is in use, or otherwise the vertex shader.
+                                     ..TODO::
+                                     Describe.
+
+     ``amdgpu_gfx``                  Used for AMD graphics targets. Functions with this calling convention
+                                     cannot be used as entry points.
+                                     ..TODO::
+                                     Describe.
+
+     ``amdgpu_gs``                   Used for Mesa/AMDPAL geometry shaders.
+                                     ..TODO::
+                                     Describe.
+
+     ``amdgpu_hs``                   Used for Mesa/AMDPAL hull shaders (= tessellation control shaders).
+                                     ..TODO::
+                                     Describe.
+
+     ``amdgpu_kernel``               See :ref:`amdgpu-amdhsa-function-call-convention-kernel-functions`
+
+     ``amdgpu_ls``                   Used for AMDPAL vertex shader if tessellation is in use.
+                                     ..TODO::
+                                     Describe.
+
+     ``amdgpu_ps``                   Used for Mesa/AMDPAL pixel shaders.
+                                     ..TODO::
+                                     Describe.
+
+     ``amdgpu_vs``                   Used for Mesa/AMDPAL last shader stage before rasterization (vertex
+                                     shader if tessellation and geometry are not in use, or otherwise
+                                     copy shader if one is needed).
+                                     ..TODO::
+                                     Describe.
+
+     =============================== ==========================================================
+
+
 .. _amdgpu-elf-code-object:
 
 ELF Code Object

From 041bfe40a771bbed776380dc9c875544647a748b Mon Sep 17 00:00:00 2001
From: Diana Picus <Diana-Magda.Picus@amd.com>
Date: Wed, 7 Jun 2023 14:42:02 +0200
Subject: [PATCH 101/130] [AMDGPU] Document amdgpu_cs_chain[_preserve] CCs. NFC
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Nicolai Hähnle <nicolai.haehnle@amd.com>

Differential Revision: https://reviews.llvm.org/D151997
---
 llvm/docs/AMDGPUUsage.rst | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 67e9780caf0b5..db3b3cda9f23c 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -1087,6 +1087,40 @@ The AMDGPU backend supports the following calling conventions:
                                      ..TODO::
                                      Describe.
 
+     ``amdgpu_cs_chain``             Similar to ``amdgpu_cs``, with differences described below.
+
+                                     Functions with this calling convention cannot be called directly. They must
+                                     instead be launched via the ``llvm.amdgcn.cs.chain`` intrinsic.
+
+                                     Arguments are passed in SGPRs, starting at s0, if they have the ``inreg``
+                                     attribute, and in VGPRs otherwise, starting at v8. Using more SGPRs or VGPRs
+                                     than available in the subtarget is not allowed.  On subtargets that use
+                                     a scratch buffer descriptor (as opposed to ``scratch_{load,store}_*`` instructions),
+                                     the scratch buffer descriptor is passed in s[48:51]. This limits the
+                                     SGPR / ``inreg`` arguments to the equivalent of 48 dwords; using more
+                                     than that is not allowed.
+
+                                     The return type must be void.
+                                     Varargs, sret, byval, byref, inalloca, preallocated are not supported.
+
+                                     Values in scalar registers as well as v0-v7 are not preserved. Values in
+                                     VGPRs starting at v8 are not preserved for the active lanes, but must be
+                                     saved by the callee for inactive lanes when using WWM.
+
+                                     Wave scratch is "empty" at function boundaries. There is no stack pointer input
+                                     or output value, but functions are free to use scratch starting from an initial
+                                     stack pointer. Calls to ``amdgpu_gfx`` functions are allowed and behave like they
+                                     do in ``amdgpu_cs`` functions.
+
+                                     All counters (``lgkmcnt``, ``vmcnt``, ``storecnt``, etc.) are presumed in an
+                                     unknown state at function entry.
+
+                                     A function may have multiple exits (e.g. one chain exit and one plain ``ret void``
+                                     for when the wave ends), but all ``llvm.amdgcn.cs.chain`` exits must be in
+                                     uniform control flow.
+
+     ``amdgpu_cs_chain_preserve``    Same as ``amdgpu_cs_chain``, but active lanes for VGPRs starting at v8 are preserved.
+
      ``amdgpu_es``                   Used for AMDPAL shader stage before geometry shader if geometry is in
                                      use. So either the domain (= tessellation evaluation) shader if
                                      tessellation is in use, or otherwise the vertex shader.

From 06ebed3e55759ab503c9bbcae5e47fcce9f26928 Mon Sep 17 00:00:00 2001
From: Nikita Popov <npopov@redhat.com>
Date: Tue, 20 Jun 2023 09:33:49 +0200
Subject: [PATCH 102/130] [SCEVNormalization] Short circuit case with no loops
 (NFC)

If there are no post-inc loops, normalization is a no-op. Don't
bother rewriting the SCEV in that case.
---
 llvm/lib/Analysis/ScalarEvolutionNormalization.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/llvm/lib/Analysis/ScalarEvolutionNormalization.cpp b/llvm/lib/Analysis/ScalarEvolutionNormalization.cpp
index 22dff5efec5c8..bc0fbffee61b0 100644
--- a/llvm/lib/Analysis/ScalarEvolutionNormalization.cpp
+++ b/llvm/lib/Analysis/ScalarEvolutionNormalization.cpp
@@ -97,6 +97,8 @@ NormalizeDenormalizeRewriter::visitAddRecExpr(const SCEVAddRecExpr *AR) {
 const SCEV *llvm::normalizeForPostIncUse(const SCEV *S,
                                          const PostIncLoopSet &Loops,
                                          ScalarEvolution &SE) {
+  if (Loops.empty())
+    return S;
   auto Pred = [&](const SCEVAddRecExpr *AR) {
     return Loops.count(AR->getLoop());
   };
@@ -111,6 +113,8 @@ const SCEV *llvm::normalizeForPostIncUseIf(const SCEV *S, NormalizePredTy Pred,
 const SCEV *llvm::denormalizeForPostIncUse(const SCEV *S,
                                            const PostIncLoopSet &Loops,
                                            ScalarEvolution &SE) {
+  if (Loops.empty())
+    return S;
   auto Pred = [&](const SCEVAddRecExpr *AR) {
     return Loops.count(AR->getLoop());
   };

From c63d2b2c714195d150ebec0ad4c712df36814c06 Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Tue, 20 Jun 2023 10:48:40 +0200
Subject: [PATCH 103/130] [mlir][transform] Add TransformRewriter

All `apply` functions now have a `TransformRewriter &` parameter. This rewriter should be used to modify the IR. It has a `TrackingListener` attached and updates the internal handle-payload mappings based on rewrites.

Implementations no longer need to create their own `TrackingListener` and `IRRewriter`. Error checking is integrated into `applyTransform`. Tracking listener errors are reported only for ops with the `ReportTrackingListenerFailuresOpTrait` trait attached, allowing for a gradual migration. Furthermore, errors can be silenced with an op attribute.

Additional API will be added to `TransformRewriter` in subsequent revisions. This revision just adds an "empty" `TransformRewriter` class and updates all `apply` implementations.

Differential Revision: https://reviews.llvm.org/D152427
---
 mlir/docs/Tutorials/transform/Ch1.md          |  13 +
 mlir/docs/Tutorials/transform/Ch2.md          |  51 ++--
 mlir/docs/Tutorials/transform/Ch3.md          |  39 +--
 .../transform/Ch2/lib/MyExtension.cpp         |  18 +-
 .../transform/Ch3/include/MyExtension.td      |   2 +
 .../transform/Ch3/lib/MyExtension.cpp         |  16 +-
 .../TransformOps/BufferizationTransformOps.td |   1 +
 .../GPU/TransformOps/GPUTransformOps.td       |   2 +
 .../Linalg/TransformOps/LinalgTransformOps.td | 111 ++++++---
 .../MemRef/TransformOps/MemRefTransformOps.td |   1 +
 .../SCF/TransformOps/SCFTransformOps.td       |   5 +
 .../Tensor/TransformOps/TensorTransformOps.td |   1 +
 .../Dialect/Transform/IR/MatchInterfaces.h    |   6 +-
 .../Dialect/Transform/IR/TransformDialect.td  |   7 +-
 .../Transform/IR/TransformInterfaces.h        |  60 ++++-
 .../Transform/IR/TransformInterfaces.td       |   9 +
 .../mlir/Dialect/Transform/IR/TransformOps.td |  16 +-
 .../TransformOps/AffineTransformOps.cpp       |   5 +-
 .../BufferizationTransformOps.cpp             |  18 +-
 .../GPU/TransformOps/GPUTransformOps.cpp      |  14 +-
 .../TransformOps/LinalgTransformOps.cpp       | 223 ++++++++----------
 .../TransformOps/MemRefTransformOps.cpp       |   6 +-
 .../SCF/TransformOps/SCFTransformOps.cpp      |  27 ++-
 .../TransformOps/TensorTransformOps.cpp       |   4 +-
 .../Dialect/Transform/IR/TransformDialect.cpp |   7 +
 .../Transform/IR/TransformInterfaces.cpp      |  91 ++++++-
 .../lib/Dialect/Transform/IR/TransformOps.cpp |  90 +++----
 .../PDLExtension/PDLExtensionOps.cpp          |   6 +-
 .../Transform/test-pattern-application.mlir   |   2 +-
 .../TestTransformDialectExtension.cpp         |  89 ++++---
 .../TestTransformDialectExtension.td          |  12 +-
 31 files changed, 602 insertions(+), 350 deletions(-)

diff --git a/mlir/docs/Tutorials/transform/Ch1.md b/mlir/docs/Tutorials/transform/Ch1.md
index 988117a995724..3db39312ebb13 100644
--- a/mlir/docs/Tutorials/transform/Ch1.md
+++ b/mlir/docs/Tutorials/transform/Ch1.md
@@ -362,3 +362,16 @@ test/Examples/transform/Ch1/invalidation-2.mlir:24:13: note: nested payload op
 Note that the “add” elementwise operation is indicated as payload ancestor because it was used to produce the tile loop, and the loop therefore has its location.
 
 Finally, we would like to replace the call to the outlined function with a call to the microkernel. Unfortunately, the Transform dialect doesn’t have support for this transformation (and cannot have if the call is rewritten to a custom, out-of-tree operation). Therefore, we need to define new transform operations. The next chapters will describe how this can be done.
+
+## Tracking IR Modifications
+
+The transform dialect automatically tracks all IR changes that are made as part
+of transform ops. (Implementations must use the provided rewriter to modify IR.)
+If a payload op is erased, it is automatically removed from all handles that it
+is currently associated with. If a payload op is replaced, the transform dialect
+tries to find the replacement op and updates all handles accordingly. If a
+multi-result op is replaced with values that are defined by multiple ops, or if
+an op is replaced with an op of a different type, an error is produced. This is
+because it is unclear whether the direct replacements actually represent the
+computation of the original op. There are ways to customize this behavior. More
+details can be found at the documentation of `transform::TrackingListener`.
diff --git a/mlir/docs/Tutorials/transform/Ch2.md b/mlir/docs/Tutorials/transform/Ch2.md
index e3d6cf738ffc1..04f79bf62a723 100644
--- a/mlir/docs/Tutorials/transform/Ch2.md
+++ b/mlir/docs/Tutorials/transform/Ch2.md
@@ -189,8 +189,13 @@ def ChangeCallTargetOp : Op<Transform_Dialect, "my.change_call_target",
 }
 ```
 
-To finalize the definition of the transform operation, we need to implement the interface methods. The `TransformOpInterface` currently requires only one method – `apply` – that performs the actual transformation. It is a good practice to limit the body of the method to manipulation of the Transform dialect constructs and have the actual transformation implemented as a standalone function so it can be used from other places in the code.
-
+To finalize the definition of the transform operation, we need to implement the
+interface methods. The `TransformOpInterface` currently requires only one method
+– `apply` – that performs the actual transformation. It is a good practice to
+limit the body of the method to manipulation of the Transform dialect constructs
+and have the actual transformation implemented as a standalone function so it
+can be used from other places in the code. Similar to rewrite patterns, all IR
+must be modified with the provided rewriter.
 
 ```c++
 // In MyExtension.cpp
@@ -198,30 +203,32 @@ To finalize the definition of the transform operation, we need to implement the
 // Implementation of our transform dialect operation.
 // This operation returns a tri-state result that can be one of:
 // - success when the transformation succeeded;
-// - definite failure when the transformation failed in such a way that following 
-//   transformations are impossible or undesirable, typically it could have left payload 
-//   IR in an invalid state; it is expected that a diagnostic is emitted immediately 
-//   before returning the definite error;
-// - silenceable failure when the transformation failed but following transformations 
-//   are still applicable, typically this means a precondition for the transformation is 
-//   not satisfied and the payload IR has not been modified.
-// The silenceable failure additionally carries a Diagnostic that can be emitted to the 
-// user.
-::mlir::DiagnosedSilenceableFailure ChangeCallTargetOp::apply(
-    // The list of payload IR entities that will be associated with the transform IR 
-    // values defined by this transform operation. In this case, it can remain empty as 
-    // there are no results.
+// - definite failure when the transformation failed in such a way that
+//   following transformations are impossible or undesirable, typically it could
+//   have left payload IR in an invalid state; it is expected that a diagnostic
+//   is emitted immediately before returning the definite error;
+// - silenceable failure when the transformation failed but following
+//   transformations are still applicable, typically this means a precondition
+//   for the transformation is not satisfied and the payload IR has not been
+//   modified. The silenceable failure additionally carries a Diagnostic that
+//   can be emitted to the user.
+::mlir::DiagnosedSilenceableFailure mlir::transform::ChangeCallTargetOp::apply(
+    // The rewriter that should be used when modifying IR.
+    ::mlir::transform::TransformRewriter &rewriter,
+    // The list of payload IR entities that will be associated with the
+    // transform IR values defined by this transform operation. In this case, it
+    // can remain empty as there are no results.
     ::mlir::transform::TransformResults &results,
-    // The transform application state. This object can be used to query the current 
-    // associations between transform IR values and payload IR entities. It can also 
-    // carry additional user-defined state.
+    // The transform application state. This object can be used to query the
+    // current associations between transform IR values and payload IR entities.
+    // It can also carry additional user-defined state.
     ::mlir::transform::TransformState &state) {
 
-  // First, we need to obtain the list of payload operations that are associated with 
+  // First, we need to obtain the list of payload operations that are associated with
   // the operand handle.
   auto payload = state.getPayloadOps(getCall());
-  
-  // Then, we iterate over the list of operands and call the actual IR-mutating 
+
+  // Then, we iterate over the list of operands and call the actual IR-mutating
   // function. We also check the preconditions here.
   for (Operation *payloadOp : payload) {
     auto call = dyn_cast<::mlir::func::CallOp>(payloadOp);
@@ -231,7 +238,7 @@ To finalize the definition of the transform operation, we need to implement the
       diag.attachNote(payloadOp->getLoc()) << "offending payload";
       return diag;
     }
-    
+
     updateCallee(call, getNewTarget());
   }
 
diff --git a/mlir/docs/Tutorials/transform/Ch3.md b/mlir/docs/Tutorials/transform/Ch3.md
index 64837a6f005f4..bf533c27be8cd 100644
--- a/mlir/docs/Tutorials/transform/Ch3.md
+++ b/mlir/docs/Tutorials/transform/Ch3.md
@@ -10,26 +10,26 @@ A transform operation that applies to each payload operation individually and re
 // Define the new operation. By convention, prefix its name with the name of the dialect extension, "my.". The full operation name will be further prefixed with "transform.".
 def ChangeCallTargetOp : Op<Transform_Dialect, "my.change_call_target",
     // Indicate that the operation implements the required TransformOpInterface and
-    // MemoryEffectsOpInterface. Use the TransformEach trait to provide the 
+    // MemoryEffectsOpInterface. Use the TransformEach trait to provide the
     // implementation for TransformOpInterface.
     [TransformOpInterface, TransformEachOpTrait,
      DeclareOpInterfaceMethods<MemoryEffectsOpInterface>]> {
-  // Provide a brief and a full description. It is recommended that the latter describes 
+  // Provide a brief and a full description. It is recommended that the latter describes
   // the effects on the operands and how the operation processes various failure modes.
   let summary = "Changes the callee of a call operation to the specified one";
   let description = [{
-    For each `func.call` payload operation associated with the handle, changes its 
+    For each `func.call` payload operation associated with the handle, changes its
     callee to be the symbol whose name is provided as an attribute to this operation.
 
-    Generates a silenceable failure if the operand is associated with payload operations 
+    Generates a silenceable failure if the operand is associated with payload operations
     that are not `func.call`.
     Only reads the operand.
   }];
 
-  // The arguments include the handle to the payload operations and the attribute that 
-  // specifies the new callee. The handle must implement TransformHandleTypeInterface.   
-  // We use a string attribute as the symbol may not exist in the transform IR so the 
-  // verification may fail. 
+  // The arguments include the handle to the payload operations and the attribute that
+  // specifies the new callee. The handle must implement TransformHandleTypeInterface.
+  // We use a string attribute as the symbol may not exist in the transform IR so the
+  // verification may fail.
   let arguments = (ins
     Transform_ConcreteOpType<"func.call">:$call,
     StrAttr:$new_target);
@@ -43,6 +43,7 @@ def ChangeCallTargetOp : Op<Transform_Dialect, "my.change_call_target",
   // Declare the function implementing the interface for a single payload operation.
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::func::CallOp call,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -54,7 +55,8 @@ Now, instead of defining the `apply` method with a loop, we can simply define a
 
 ```c++
 ::mlir::DiagnosedSilenceableFailure ChangeCallTargetOp::applyToOne(
-    ::mlir::func::CallOp call,,
+    ::mlir::transform::TransformRewriter &rewriter,
+    ::mlir::func::CallOp call,
     ::mlir::transform::ApplyToEachResultList &results,
     ::mlir::transform::TransformState &state) {
   // Call the actual transformation function.
@@ -150,14 +152,13 @@ This type is now directly available in the transform dialect and can be used in
 
 As an exercise, let us modify the rewriting operation to consume the operand. This would be necessary, for example, if the transformation were to rewrite the `func.call` operation into a custom operation `my.mm4`. Since the operand handle is now consumed, the operation can return a new handle to the newly produced payload operation. Otherwise, the ODS definition of the transform operation remains unchanged.
 
-
 ```tablegen
 // In MyExtension.td.
 
 // Define another transform operation.
 def CallToOp : Op<Transform_Dialect, "my.call_to_op",
      // Indicate that the operation implements the required TransformOpInterface and
-     // MemoryEffectsOpInterface. Use the TransformEach trait to provide the 
+     // MemoryEffectsOpInterface. Use the TransformEach trait to provide the
      // implementation for TransformOpInterface.
     [TransformOpInterface, TransformEachOpTrait,
      DeclareOpInterfaceMethods<MemoryEffectsOpInterface>]> {
@@ -166,7 +167,7 @@ def CallToOp : Op<Transform_Dialect, "my.call_to_op",
   // The argument is the handle to the payload operations.
   let arguments = (ins CallOpInterfaceHandle:$call);
 
-  // The result is the handle to the payload operations produced during the 
+  // The result is the handle to the payload operations produced during the
   // transformation.
   let results = (outs TransformHandleTypeInterface:$transformed);
 
@@ -176,6 +177,7 @@ def CallToOp : Op<Transform_Dialect, "my.call_to_op",
   // Declare the function implementing the interface for a single payload operation.
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::CallOpInterface call,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -189,23 +191,24 @@ Now let’s look at the implementation of interface methods.
 // In MyExtension.cpp.
 
 ::mlir::DiagnosedSilenceableFailure CallToOp::applyToOne(
+    ::mlir::transform::TransformRewriter &rewriter,
     ::mlir::CallOpInterface call,
     ::mlir::transform::ApplyToEachResultList &results,
     ::mlir::transform::TransformState &state) {
   // Call the actual rewrite.
   Operation *rewritten = rewriteToOp(call);
 
-  // Report an error if the rewriter produced a null pointer. Note that it may have 
+  // Report an error if the rewriter produced a null pointer. Note that it may have
   // irreversibly modified the payload IR, so we produce a definite failure.
   if (!rewritten) {
     return emitDefiniteError() << "failed to rewrite call to operation";
   }
 
-  // On success, push the resulting operation into the result list. The list is expected 
-  // to contain exactly one entity per result and per application. The handles will be 
+  // On success, push the resulting operation into the result list. The list is expected
+  // to contain exactly one entity per result and per application. The handles will be
   // associated with lists of the respective values produced by each application.
   results.push_back(rewritten);
-  
+
   // If everything is fine, return success.
   return DiagnosedSilenceableFailure::success();
 }
@@ -228,6 +231,7 @@ Note that since `applyToOne` always expects one payload entity to be associated
 
 ```c++
 ::mlir::DiagnosedSilenceableFailure SomeOtherOp::apply(
+    ::mlir::transform::TransformRewriter &rewriter,
     ::mlir::transform::TransformResults &results,
     ::mlir::transform::TransformState &state) {
   // ...
@@ -263,7 +267,7 @@ def CallToOp : Op<Transform_Dialect, "my.call_to_op",
   // The argument is the handle to the payload operations.
   let arguments = (ins CallOpInterfaceHandle:$call);
 
-  // The result is the handle to the payload operations produced during the 
+  // The result is the handle to the payload operations produced during the
   // transformation.
   let results = (outs TransformHandleTypeInterface:$transformed);
 
@@ -273,6 +277,7 @@ def CallToOp : Op<Transform_Dialect, "my.call_to_op",
   // Declare the function implementing the interface for a single payload operation.
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::CallOpInterface call,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
diff --git a/mlir/examples/transform/Ch2/lib/MyExtension.cpp b/mlir/examples/transform/Ch2/lib/MyExtension.cpp
index 5eda317db0a4e..af2890d3b5239 100644
--- a/mlir/examples/transform/Ch2/lib/MyExtension.cpp
+++ b/mlir/examples/transform/Ch2/lib/MyExtension.cpp
@@ -74,17 +74,17 @@ static void updateCallee(mlir::func::CallOp call, llvm::StringRef newTarget) {
 // This operation returns a tri-state result that can be one of:
 // - success when the transformation succeeded;
 // - definite failure when the transformation failed in such a way that
-// following
-//   transformations are impossible or undesirable, typically it could have left
-//   payload IR in an invalid state; it is expected that a diagnostic is emitted
-//   immediately before returning the definite error;
+//   following transformations are impossible or undesirable, typically it could
+//   have left payload IR in an invalid state; it is expected that a diagnostic
+//   is emitted immediately before returning the definite error;
 // - silenceable failure when the transformation failed but following
-// transformations
-//   are still applicable, typically this means a precondition for the
-//   transformation is not satisfied and the payload IR has not been modified.
-// The silenceable failure additionally carries a Diagnostic that can be emitted
-// to the user.
+//   transformations are still applicable, typically this means a precondition
+//   for the transformation is not satisfied and the payload IR has not been
+//   modified. The silenceable failure additionally carries a Diagnostic that
+//   can be emitted to the user.
 ::mlir::DiagnosedSilenceableFailure mlir::transform::ChangeCallTargetOp::apply(
+    // The rewriter that should be used when modifying IR.
+    ::mlir::transform::TransformRewriter &rewriter,
     // The list of payload IR entities that will be associated with the
     // transform IR values defined by this transform operation. In this case, it
     // can remain empty as there are no results.
diff --git a/mlir/examples/transform/Ch3/include/MyExtension.td b/mlir/examples/transform/Ch3/include/MyExtension.td
index 3c5695739fa2d..f444df18d69e8 100644
--- a/mlir/examples/transform/Ch3/include/MyExtension.td
+++ b/mlir/examples/transform/Ch3/include/MyExtension.td
@@ -60,6 +60,7 @@ def ChangeCallTargetOp : Op<Transform_Dialect, "my.change_call_target",
   // Declare the function implementing the interface for a single payload operation.
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::func::CallOp call,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -89,6 +90,7 @@ def CallToOp : Op<Transform_Dialect, "my.call_to_op",
   // Declare the function implementing the interface for a single payload operation.
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::CallOpInterface call,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
diff --git a/mlir/examples/transform/Ch3/lib/MyExtension.cpp b/mlir/examples/transform/Ch3/lib/MyExtension.cpp
index a4f9108df0e8c..dc0a8a0ab3033 100644
--- a/mlir/examples/transform/Ch3/lib/MyExtension.cpp
+++ b/mlir/examples/transform/Ch3/lib/MyExtension.cpp
@@ -113,6 +113,8 @@ static void updateCallee(mlir::func::CallOp call, llvm::StringRef newTarget) {
 // to the user.
 ::mlir::DiagnosedSilenceableFailure
 mlir::transform::ChangeCallTargetOp::applyToOne(
+    // The rewriter that should be used when modifying IR.
+    ::mlir::transform::TransformRewriter &rewriter,
     // The single payload operation to which the transformation is applied.
     ::mlir::func::CallOp call,
     // The payload IR entities that will be appended to lists associated with
@@ -146,27 +148,27 @@ void mlir::transform::ChangeCallTargetOp::getEffects(
 // CallToOp
 //===---------------------------------------------------------------------===//
 
-static mlir::Operation *replaceCallWithOp(mlir::CallOpInterface call) {
+static mlir::Operation *replaceCallWithOp(mlir::RewriterBase &rewriter,
+                                          mlir::CallOpInterface call) {
   // Construct an operation from an unregistered dialect. This is discouraged
   // and is only used here for brevity of the overall example.
   mlir::OperationState state(call.getLoc(), "my.mm4");
   state.types.assign(call->result_type_begin(), call->result_type_end());
   state.operands.assign(call->operand_begin(), call->operand_end());
 
-  mlir::OpBuilder builder(call);
-  mlir::Operation *replacement = builder.create(state);
-  call->replaceAllUsesWith(replacement->getResults());
-  call->erase();
+  mlir::Operation *replacement = rewriter.create(state);
+  rewriter.replaceOp(call, replacement->getResults());
   return replacement;
 }
 
 // See above for the signature description.
 mlir::DiagnosedSilenceableFailure mlir::transform::CallToOp::applyToOne(
-    mlir::CallOpInterface call, mlir::transform::ApplyToEachResultList &results,
+    mlir::transform::TransformRewriter &rewriter, mlir::CallOpInterface call,
+    mlir::transform::ApplyToEachResultList &results,
     mlir::transform::TransformState &state) {
 
   // Dispatch to the actual transformation.
-  Operation *replacement = replaceCallWithOp(call);
+  Operation *replacement = replaceCallWithOp(rewriter, call);
 
   // Associate the payload operation produced by the rewrite with the result
   // handle of this transform operation.
diff --git a/mlir/include/mlir/Dialect/Bufferization/TransformOps/BufferizationTransformOps.td b/mlir/include/mlir/Dialect/Bufferization/TransformOps/BufferizationTransformOps.td
index 49b5ef6f0fee8..807a63d36f393 100644
--- a/mlir/include/mlir/Dialect/Bufferization/TransformOps/BufferizationTransformOps.td
+++ b/mlir/include/mlir/Dialect/Bufferization/TransformOps/BufferizationTransformOps.td
@@ -152,6 +152,7 @@ def EmptyTensorToAllocTensorOp
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::tensor::EmptyOp target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
diff --git a/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.td b/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.td
index 42162315aebdf..4e47638698180 100644
--- a/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.td
+++ b/mlir/include/mlir/Dialect/GPU/TransformOps/GPUTransformOps.td
@@ -126,6 +126,7 @@ def MapNestedForallToThreads :
   }];
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::Operation *target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -189,6 +190,7 @@ def MapForallToBlocks :
   }];
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::Operation *target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
index 3e2cb78bf9d12..0157fc2b2c1dc 100644
--- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
+++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td
@@ -80,7 +80,8 @@ def ApplyTilingCanonicalizationPatternsOp : Op<Transform_Dialect,
 def BufferizeToAllocationOp : Op<Transform_Dialect,
     "structured.bufferize_to_allocation",
     [DeclareOpInterfaceMethods<TransformOpInterface>,
-     DeclareOpInterfaceMethods<MemoryEffectsOpInterface>]> {
+     DeclareOpInterfaceMethods<MemoryEffectsOpInterface>,
+     ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     This transform materializes an allocation for the targeted tensor value. It
     replaces all original uses of the target with the newly allocated buffer,
@@ -133,7 +134,8 @@ def DecomposeOp : Op<Transform_Dialect, "structured.decompose",
     [FunctionalStyleTransformOpTrait, 
      MemoryEffectsOpInterface,
      TransformOpInterface, 
-     TransformEachOpTrait]> {
+     TransformEachOpTrait,
+     ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Decomposes named complex operations, such as higher-dimensional
     (depthwise) convolutions, into combinations of lower-dimensional equivalents
@@ -155,6 +157,7 @@ def DecomposeOp : Op<Transform_Dialect, "structured.decompose",
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::linalg::LinalgOp target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -167,7 +170,8 @@ def DecomposeOp : Op<Transform_Dialect, "structured.decompose",
 
 def FuseOp : Op<Transform_Dialect, "structured.fuse",
     [FunctionalStyleTransformOpTrait, MemoryEffectsOpInterface,
-     DeclareOpInterfaceMethods<TransformOpInterface>]> {
+     DeclareOpInterfaceMethods<TransformOpInterface>,
+     ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Tiles the operations pointed to by the target handle and fuses their
     producers greedily using the options provided as attributes.
@@ -192,7 +196,8 @@ def FuseIntoContainingOp :
     Op<Transform_Dialect, "structured.fuse_into_containing_op",
       [DeclareOpInterfaceMethods<TransformOpInterface,
           ["allowsRepeatedHandleOperands"]>,
-       DeclareOpInterfaceMethods<MemoryEffectsOpInterface>]> {
+       DeclareOpInterfaceMethods<MemoryEffectsOpInterface>,
+       ReportTrackingListenerFailuresOpTrait]> {
   let summary = "Fuse a producer into a containing operation.";
 
   let description = [{
@@ -247,7 +252,8 @@ def FuseIntoContainingOp :
 
 def GeneralizeOp : Op<Transform_Dialect, "structured.generalize",
     [FunctionalStyleTransformOpTrait, MemoryEffectsOpInterface,
-     TransformOpInterface, TransformEachOpTrait]> {
+     TransformOpInterface, TransformEachOpTrait,
+     ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Transforms a named structured operation into the generic form with the
     explicit attached region.
@@ -270,6 +276,7 @@ def GeneralizeOp : Op<Transform_Dialect, "structured.generalize",
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::linalg::LinalgOp target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -282,7 +289,8 @@ def GeneralizeOp : Op<Transform_Dialect, "structured.generalize",
 
 def InterchangeOp : Op<Transform_Dialect, "structured.interchange",
     [FunctionalStyleTransformOpTrait, MemoryEffectsOpInterface,
-    TransformOpInterface, TransformEachOpTrait]> {
+    TransformOpInterface, TransformEachOpTrait,
+    ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Interchanges the iterators of the operations pointed to by the target handle
     using the iterator interchange attribute.
@@ -313,6 +321,7 @@ def InterchangeOp : Op<Transform_Dialect, "structured.interchange",
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::linalg::GenericOp target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -326,7 +335,8 @@ def LowerPackOp : Op<Transform_Dialect, "structured.lower_pack", [
                          FunctionalStyleTransformOpTrait,
                          MemoryEffectsOpInterface,
                          TransformEachOpTrait,
-                         TransformOpInterface]> {
+                         TransformOpInterface,
+                         ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Rewrite a tensor.pack into tensor.pad + tensor.expand_shape + linalg.transpose.
 
@@ -350,6 +360,7 @@ def LowerPackOp : Op<Transform_Dialect, "structured.lower_pack", [
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::tensor::PackOp target,
         ::mlir::transform::ApplyToEachResultList &transformResults,
         ::mlir::transform::TransformState &state);
@@ -363,7 +374,8 @@ def LowerUnPackOp : Op<Transform_Dialect, "structured.lower_unpack", [
                          FunctionalStyleTransformOpTrait,
                          MemoryEffectsOpInterface,
                          TransformEachOpTrait,
-                         TransformOpInterface]> {
+                         TransformOpInterface,
+                         ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Lower a tensor.unpack into empty + linalg.transpose + tensor.collapse_shape + 
     tensor.extract_slice.
@@ -389,6 +401,7 @@ def LowerUnPackOp : Op<Transform_Dialect, "structured.lower_unpack", [
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::tensor::UnPackOp target,
         ::mlir::transform::ApplyToEachResultList &transformResults,
         ::mlir::transform::TransformState &state);
@@ -461,7 +474,8 @@ def MatchOp : Op<Transform_Dialect, "structured.match",
 
 def MultiTileSizesOp : Op<Transform_Dialect, "structured.multitile_sizes",
     [DeclareOpInterfaceMethods<MemoryEffectsOpInterface>,
-     TransformOpInterface, TransformEachOpTrait]> {
+     TransformOpInterface, TransformEachOpTrait,
+     ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Emits the IR computing the tile sizes `s1` and `s2` such that:
 
@@ -531,6 +545,7 @@ def MultiTileSizesOp : Op<Transform_Dialect, "structured.multitile_sizes",
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::linalg::LinalgOp target,
         ::mlir::transform::ApplyToEachResultList &results,
         TransformState &state);
@@ -543,7 +558,8 @@ def MultiTileSizesOp : Op<Transform_Dialect, "structured.multitile_sizes",
 
 def PackOp : Op<Transform_Dialect, "structured.pack", [
                 DeclareOpInterfaceMethods<TransformOpInterface>,
-                DeclareOpInterfaceMethods<MemoryEffectsOpInterface>]> {
+                DeclareOpInterfaceMethods<MemoryEffectsOpInterface>,
+                ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Pack a LinalgOp by applying a data tiling transformation on the op and
     packing the operands according to the `packed_sizes` specification.
@@ -632,7 +648,8 @@ def PackOp : Op<Transform_Dialect, "structured.pack", [
 //===----------------------------------------------------------------------===//
 def PackGreedilyOp : Op<Transform_Dialect, "structured.pack_greedily", [
                         DeclareOpInterfaceMethods<TransformOpInterface>,
-                        DeclareOpInterfaceMethods<MemoryEffectsOpInterface>]> {
+                        DeclareOpInterfaceMethods<MemoryEffectsOpInterface>,
+                        ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Target a Linalg op and rewrite it into packed LinalgOp form by trying to
     infer whether a known suboperation is embedded
@@ -740,7 +757,8 @@ def PackGreedilyOp : Op<Transform_Dialect, "structured.pack_greedily", [
 def PackTransposeOp : Op<Transform_Dialect, "structured.pack_transpose", [
                          FunctionalStyleTransformOpTrait,
                          MemoryEffectsOpInterface,
-                         DeclareOpInterfaceMethods<TransformOpInterface>]> {
+                         DeclareOpInterfaceMethods<TransformOpInterface>,
+                         ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Apply a transposition to a single `tensor.pack` (resp. `tensor.unpack`) and 
     update the `linalg.generic` op that consumes (resp. produces) the operation.
@@ -803,7 +821,8 @@ def PackTransposeOp : Op<Transform_Dialect, "structured.pack_transpose", [
 
 def PadOp : Op<Transform_Dialect, "structured.pad",
     [FunctionalStyleTransformOpTrait, MemoryEffectsOpInterface,
-     TransformOpInterface, TransformEachOpTrait]> {
+     TransformOpInterface, TransformEachOpTrait,
+     ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Pads the operations pointed to by the target handle using the options
     provides as operation attributes.
@@ -837,6 +856,7 @@ def PadOp : Op<Transform_Dialect, "structured.pad",
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::linalg::LinalgOp target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -851,7 +871,8 @@ def HoistPadBuildPackingLoopNestOp :
     Op<Transform_Dialect,
        "structured.hoist_pad.build_packing_loop_nest",
     [DeclareOpInterfaceMethods<MemoryEffectsOpInterface>,
-     DeclareOpInterfaceMethods<TransformOpInterface>]> {
+     DeclareOpInterfaceMethods<TransformOpInterface>,
+     ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Helper transform used to hoist a tensor.pad target operation. This operation
     creates the packing loop nest required by the hoist_pad operation and makes
@@ -931,6 +952,7 @@ def HoistPadOp : Op<Transform_Dialect, "structured.hoist_pad",
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::tensor::PadOp,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -944,7 +966,8 @@ def HoistPadOp : Op<Transform_Dialect, "structured.hoist_pad",
 
 def PromoteOp : Op<Transform_Dialect, "structured.promote",
     [FunctionalStyleTransformOpTrait, MemoryEffectsOpInterface,
-    TransformOpInterface, TransformEachOpTrait]> {
+    TransformOpInterface, TransformEachOpTrait,
+    ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Promotes the specified operands of the target into a separate memory buffer.
 
@@ -978,6 +1001,7 @@ def PromoteOp : Op<Transform_Dialect, "structured.promote",
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::linalg::LinalgOp target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -990,7 +1014,8 @@ def PromoteOp : Op<Transform_Dialect, "structured.promote",
 
 def ReplaceOp : Op<Transform_Dialect, "structured.replace",
     [IsolatedFromAbove, DeclareOpInterfaceMethods<TransformOpInterface>,
-     DeclareOpInterfaceMethods<MemoryEffectsOpInterface>] # GraphRegionNoTerminator.traits> {
+     DeclareOpInterfaceMethods<MemoryEffectsOpInterface>,
+     ReportTrackingListenerFailuresOpTrait] # GraphRegionNoTerminator.traits> {
   let description = [{
     Replace all `target` payload ops with the single op that is contained in
     this op's region. All targets must have zero arguments and must be isolated
@@ -1018,7 +1043,8 @@ def ReplaceOp : Op<Transform_Dialect, "structured.replace",
 
 def ScalarizeOp : Op<Transform_Dialect, "structured.scalarize",
     [FunctionalStyleTransformOpTrait, MemoryEffectsOpInterface,
-     TransformOpInterface, TransformEachOpTrait]> {
+     TransformOpInterface, TransformEachOpTrait,
+     ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Indicates that ops of a specific kind in the given function should be
     scalarized (i.e. their dynamic dimensions tiled by 1).
@@ -1051,6 +1077,7 @@ def ScalarizeOp : Op<Transform_Dialect, "structured.scalarize",
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::linalg::LinalgOp target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -1066,7 +1093,8 @@ def RewriteInDestinationPassingStyleOp : Op<
     [FunctionalStyleTransformOpTrait, 
      MemoryEffectsOpInterface,
      TransformOpInterface, 
-     TransformEachOpTrait]> {
+     TransformEachOpTrait,
+     ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Rewrite a supported tensor operation that is not in destination-passing style
     into a form that is in destination-passing style.
@@ -1098,6 +1126,7 @@ def RewriteInDestinationPassingStyleOp : Op<
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::Operation *target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -1110,7 +1139,8 @@ def RewriteInDestinationPassingStyleOp : Op<
 
 def SplitOp : Op<Transform_Dialect, "structured.split",
     [DeclareOpInterfaceMethods<MemoryEffectsOpInterface>,
-     DeclareOpInterfaceMethods<TransformOpInterface>]> {
+     DeclareOpInterfaceMethods<TransformOpInterface>,
+     ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Indicates that the given `target` op should be split into two complementary
     parts, which combined cover the entire iteration domain of the original op.
@@ -1147,7 +1177,8 @@ def SplitOp : Op<Transform_Dialect, "structured.split",
 
 def SplitReductionOp : Op<Transform_Dialect, "structured.split_reduction",
        [FunctionalStyleTransformOpTrait, MemoryEffectsOpInterface,
-        TransformEachOpTrait, TransformOpInterface]> {
+        TransformEachOpTrait, TransformOpInterface,
+        ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Indicates that the given `target` op should be transformed with the
     `splitReduction` transformation and split factor provided as attribute.
@@ -1309,6 +1340,7 @@ def SplitReductionOp : Op<Transform_Dialect, "structured.split_reduction",
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::linalg::LinalgOp target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -1321,7 +1353,8 @@ def SplitReductionOp : Op<Transform_Dialect, "structured.split_reduction",
 
 def TileReductionUsingScfOp : Op<Transform_Dialect, "structured.tile_reduction_using_scf",
        [FunctionalStyleTransformOpTrait, MemoryEffectsOpInterface,
-        TransformEachOpTrait, TransformOpInterface]> {
+        TransformEachOpTrait, TransformOpInterface,
+        ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Indicates that the given `target` op should be transformed with the
     `tileReduction` transformation with the tile size provided as attribute.
@@ -1413,6 +1446,7 @@ def TileReductionUsingScfOp : Op<Transform_Dialect, "structured.tile_reduction_u
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::linalg::LinalgOp target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -1426,7 +1460,8 @@ def TileReductionUsingScfOp : Op<Transform_Dialect, "structured.tile_reduction_u
 def TileReductionUsingForallOp :
   Op<Transform_Dialect, "structured.tile_reduction_using_forall",
        [FunctionalStyleTransformOpTrait, MemoryEffectsOpInterface,
-        TransformEachOpTrait, TransformOpInterface]> {
+        TransformEachOpTrait, TransformOpInterface,
+        ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Tile a PartialReductionOpInterface op to a tiled `scf.forall` doing
     partial reduction.
@@ -1522,6 +1557,7 @@ def TileReductionUsingForallOp :
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::linalg::LinalgOp target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -1535,7 +1571,8 @@ def TileReductionUsingForallOp :
 
 def TileOp : Op<Transform_Dialect, "structured.tile",
        [DeclareOpInterfaceMethods<TransformOpInterface>,
-        DeclareOpInterfaceMethods<MemoryEffectsOpInterface>]> {
+        DeclareOpInterfaceMethods<MemoryEffectsOpInterface>,
+        ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Indicates that the given `target` op should be tiled with the given sizes.
     This transform generates a loop nest with a smaller ("tiled") target
@@ -1616,7 +1653,7 @@ def TileToForallOp :
     Op<Transform_Dialect, "structured.tile_to_forall_op",
       [AttrSizedOperandSegments,
        DeclareOpInterfaceMethods<MemoryEffectsOpInterface>,
-       TransformOpInterface]> {
+       TransformOpInterface, ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Tile a TilingInterface op to a tiled `scf.forall`.
 
@@ -1726,6 +1763,7 @@ def TileToForallOp :
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure apply(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::transform::TransformResults &transformResults,
         ::mlir::transform::TransformState &state);
 
@@ -1740,7 +1778,8 @@ def TileToForallOp :
 
 def TileToScfForOp : Op<Transform_Dialect, "structured.tile_to_scf_for",
        [DeclareOpInterfaceMethods<TransformOpInterface>,
-        DeclareOpInterfaceMethods<MemoryEffectsOpInterface>]> {
+        DeclareOpInterfaceMethods<MemoryEffectsOpInterface>,
+        ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Indicates that the given `target` op should be tiled with the given sizes.
     This transform generates a loop nest with a smaller ("tiled") target
@@ -1807,7 +1846,8 @@ def TileToScfForOp : Op<Transform_Dialect, "structured.tile_to_scf_for",
 
 def VectorizeOp : Op<Transform_Dialect, "structured.vectorize",
     [FunctionalStyleTransformOpTrait, MemoryEffectsOpInterface,
-     TransformEachOpTrait, TransformOpInterface]> {
+     TransformEachOpTrait, TransformOpInterface,
+     ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Indicates that the given `target` op all the ops it contains should be
     vectorized with the configuration specified by the attributes of this op.
@@ -1862,6 +1902,7 @@ def VectorizeOp : Op<Transform_Dialect, "structured.vectorize",
   ];
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::Operation *target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -1870,7 +1911,7 @@ def VectorizeOp : Op<Transform_Dialect, "structured.vectorize",
 
 def MaskedVectorizeOp : Op<Transform_Dialect, "structured.masked_vectorize",
     [DeclareOpInterfaceMethods<MemoryEffectsOpInterface>,
-     TransformOpInterface]> {
+     TransformOpInterface, ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Vectorize the target ops, which must be Linalg ops, with masked vectors
     of the specified size.
@@ -1914,6 +1955,7 @@ def MaskedVectorizeOp : Op<Transform_Dialect, "structured.masked_vectorize",
   let extraClassDeclaration = [{
     // TODO: applyToOne.
     ::mlir::DiagnosedSilenceableFailure apply(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::transform::TransformResults &transformResults,
         ::mlir::transform::TransformState &state);
 
@@ -1928,7 +1970,8 @@ def MaskedVectorizeOp : Op<Transform_Dialect, "structured.masked_vectorize",
 def HoistRedundantVectorTransfersOp :
   Op<Transform_Dialect, "structured.hoist_redundant_vector_transfers",
     [FunctionalStyleTransformOpTrait, MemoryEffectsOpInterface,
-     TransformEachOpTrait, TransformOpInterface]> {
+     TransformEachOpTrait, TransformOpInterface,
+     ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Hoist vector.transfer_read / vector.transfer_write pairs out of immediately
     enclosing scf::ForOp iteratively, if the following conditions are true:
@@ -1958,7 +2001,8 @@ def HoistRedundantVectorTransfersOp :
   ];
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
-        ::mlir::func::FuncOp target,
+         ::mlir::transform::TransformRewriter &rewriter,
+         ::mlir::func::FuncOp target,
          ::mlir::transform::ApplyToEachResultList &results,
          ::mlir::transform::TransformState &state);
    }];
@@ -1973,7 +2017,8 @@ def ConvertConv2DToImg2ColOp : Op<Transform_Dialect,
     [FunctionalStyleTransformOpTrait,
      MemoryEffectsOpInterface,
      TransformOpInterface,
-     TransformEachOpTrait]> {
+     TransformEachOpTrait,
+     ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Convert linalg.conv_2d_xxx into linalg.generic (for img2col packing)
     and linalg.matmul.
@@ -2034,6 +2079,7 @@ def ConvertConv2DToImg2ColOp : Op<Transform_Dialect,
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::linalg::LinalgOp target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -2048,7 +2094,8 @@ def HoistRedundantTensorSubsetsOp :
   Op<Transform_Dialect, "structured.hoist_redundant_tensor_subsets",
     [DeclareOpInterfaceMethods<MemoryEffectsOpInterface>,
      TransformEachOpTrait,
-     TransformOpInterface]> {
+     TransformOpInterface,
+     ReportTrackingListenerFailuresOpTrait]> {
   let description = [{
     Hoists supported tensor subset extract/insert operation pairs out of 
     immediately enclosing loop iteratively, if the following conditions
@@ -2082,6 +2129,7 @@ def HoistRedundantTensorSubsetsOp :
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::Operation *target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -2120,6 +2168,7 @@ def InsertSliceToCopyOp :
   ];
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::Operation *target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
diff --git a/mlir/include/mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.td b/mlir/include/mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.td
index 35a1d840f2d69..86a3586bcc58b 100644
--- a/mlir/include/mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.td
+++ b/mlir/include/mlir/Dialect/MemRef/TransformOps/MemRefTransformOps.td
@@ -158,6 +158,7 @@ def MemRefMakeLoopIndependentOp
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::Operation *target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
diff --git a/mlir/include/mlir/Dialect/SCF/TransformOps/SCFTransformOps.td b/mlir/include/mlir/Dialect/SCF/TransformOps/SCFTransformOps.td
index d128c0e4f2796..b9d2dd1ee4705 100644
--- a/mlir/include/mlir/Dialect/SCF/TransformOps/SCFTransformOps.td
+++ b/mlir/include/mlir/Dialect/SCF/TransformOps/SCFTransformOps.td
@@ -126,6 +126,7 @@ def LoopPeelOp : Op<Transform_Dialect, "loop.peel",
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::scf::ForOp target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -168,6 +169,7 @@ def LoopPipelineOp : Op<Transform_Dialect, "loop.pipeline",
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::scf::ForOp target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -201,6 +203,7 @@ def LoopUnrollOp : Op<Transform_Dialect, "loop.unroll",
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::Operation *target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -228,6 +231,7 @@ def LoopCoalesceOp : Op<Transform_Dialect, "loop.coalesce", [
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::Operation *target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -269,6 +273,7 @@ def TakeAssumedBranchOp : Op<Transform_Dialect, "scf.take_assumed_branch", [
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::scf::IfOp ifOp,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
diff --git a/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.td b/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.td
index f23e310497812..5d9ced8960385 100644
--- a/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.td
+++ b/mlir/include/mlir/Dialect/Tensor/TransformOps/TensorTransformOps.td
@@ -138,6 +138,7 @@ def MakeLoopIndependentOp
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::Operation *target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
diff --git a/mlir/include/mlir/Dialect/Transform/IR/MatchInterfaces.h b/mlir/include/mlir/Dialect/Transform/IR/MatchInterfaces.h
index c0d7545622e69..1a6afc58fef27 100644
--- a/mlir/include/mlir/Dialect/Transform/IR/MatchInterfaces.h
+++ b/mlir/include/mlir/Dialect/Transform/IR/MatchInterfaces.h
@@ -50,7 +50,8 @@ class SingleOpMatcherOpTrait
     return success();
   }
 
-  DiagnosedSilenceableFailure apply(TransformResults &results,
+  DiagnosedSilenceableFailure apply(TransformRewriter &rewriter,
+                                    TransformResults &results,
                                     TransformState &state) {
     Value operandHandle = cast<OpTy>(this->getOperation()).getOperandHandle();
     auto payload = state.getPayloadOps(operandHandle);
@@ -90,7 +91,8 @@ class SingleValueMatcherOpTrait
     return success();
   }
 
-  DiagnosedSilenceableFailure apply(TransformResults &results,
+  DiagnosedSilenceableFailure apply(TransformRewriter &rewriter,
+                                    TransformResults &results,
                                     TransformState &state) {
     Value operandHandle = cast<OpTy>(this->getOperation()).getOperandHandle();
     ValueRange payload = state.getPayloadValues(operandHandle);
diff --git a/mlir/include/mlir/Dialect/Transform/IR/TransformDialect.td b/mlir/include/mlir/Dialect/Transform/IR/TransformDialect.td
index 0539187256dde..62775801d4ac0 100644
--- a/mlir/include/mlir/Dialect/Transform/IR/TransformDialect.td
+++ b/mlir/include/mlir/Dialect/Transform/IR/TransformDialect.td
@@ -27,11 +27,16 @@ def Transform_Dialect : Dialect {
       constexpr const static ::llvm::StringLiteral
           kWithNamedSequenceAttrName = "transform.with_named_sequence";
 
-      /// Names of the attribute attachable to an operation so it can be
+      /// Name of the attribute attachable to an operation so it can be
       /// identified as root by the default interpreter pass.
       constexpr const static ::llvm::StringLiteral
           kTargetTagAttrName = "transform.target_tag";
 
+      /// Name of the attribute attachable to an operation, indicating that
+      /// TrackingListener failures should be silenced.
+      constexpr const static ::llvm::StringLiteral
+          kSilenceTrackingFailuresAttrName = "transform.silence_tracking_failures";
+
       /// Names of the attributes indicating whether an argument of an external
       /// transform dialect symbol is consumed or only read.
       constexpr const static ::llvm::StringLiteral
diff --git a/mlir/include/mlir/Dialect/Transform/IR/TransformInterfaces.h b/mlir/include/mlir/Dialect/Transform/IR/TransformInterfaces.h
index 1afac0aba48dc..5a22ae51d21dd 100644
--- a/mlir/include/mlir/Dialect/Transform/IR/TransformInterfaces.h
+++ b/mlir/include/mlir/Dialect/Transform/IR/TransformInterfaces.h
@@ -18,11 +18,11 @@
 #include "mlir/Support/LogicalResult.h"
 
 namespace mlir {
-
 namespace transform {
 
 class TransformOpInterface;
 class TransformResults;
+class TransformRewriter;
 class TransformState;
 
 using Param = Attribute;
@@ -856,8 +856,7 @@ class TrackingListener : public RewriterBase::Listener,
                          public TransformState::Extension {
 public:
   /// Create a new TrackingListener for usage in the specified transform op.
-  explicit TrackingListener(TransformState &state, TransformOpInterface op)
-      : TransformState::Extension(state), transformOp(op) {}
+  TrackingListener(TransformState &state, TransformOpInterface op);
 
 protected:
   /// Return a replacement payload op for the given op, which is going to be
@@ -937,6 +936,9 @@ class TrackingListener : public RewriterBase::Listener,
 
   /// The transform op in which this TrackingListener is used.
   TransformOpInterface transformOp;
+
+  /// The handles that are consumed by the transform op.
+  DenseSet<Value> consumedHandles;
 };
 
 /// A specialized listener that keeps track of cases in which no replacement
@@ -968,6 +970,28 @@ class ErrorCheckingTrackingListener : public TrackingListener {
   int64_t errorCounter = 0;
 };
 
+/// This is a special rewriter to be used in transform op implementations,
+/// providing additional helper functions to update the transform state, etc.
+// TODO: Helper functions will be added in a subsequent change.
+class TransformRewriter : public RewriterBase {
+protected:
+  friend class TransformState;
+
+  /// Create a new TransformRewriter.
+  explicit TransformRewriter(MLIRContext *ctx,
+                             ErrorCheckingTrackingListener *listener);
+
+public:
+  /// Return "true" if the tracking listener had failures.
+  bool hasTrackingFailures() const;
+
+  /// Silence all tracking failures that have been encountered so far.
+  void silenceTrackingFailure();
+
+private:
+  ErrorCheckingTrackingListener *const listener;
+};
+
 /// This trait is supposed to be attached to Transform dialect operations that
 /// can be standalone top-level transforms. Such operations typically contain
 /// other Transform dialect operations that can be executed following some
@@ -1064,7 +1088,8 @@ class TransformEachOpTrait
   ///   5. If any `applyToOne` return silenceableFailure, the transformation is
   ///      considered silenceable.
   ///   6. Otherwise the transformation is considered successful.
-  DiagnosedSilenceableFailure apply(TransformResults &transformResults,
+  DiagnosedSilenceableFailure apply(transform::TransformRewriter &rewriter,
+                                    TransformResults &transformResults,
                                     TransformState &state);
 
   /// Checks that the op matches the expectations of this trait.
@@ -1213,6 +1238,15 @@ class ParamProducerTransformOpTrait
   }
 };
 
+/// `TrackingListener` failures are reported only for ops that have this trait.
+/// The purpose of this trait is to give users more time to update their custom
+/// transform ops to use the provided `TransformRewriter` for all IR
+/// modifications. This trait will eventually be removed, and failures will be
+/// reported for all transform ops.
+template <typename OpTy>
+class ReportTrackingListenerFailuresOpTrait
+    : public OpTrait::TraitBase<OpTy, ReportTrackingListenerFailuresOpTrait> {};
+
 /// A single result of applying a transform op with `ApplyEachOpTrait` to a
 /// single payload operation.
 using ApplyToEachResult = MappedValue;
@@ -1307,14 +1341,14 @@ void setApplyToOneResults(Operation *transformOp,
 ///   `targets` contains operations of the same class and a silenceable failure
 ///   is reported if it does not.
 template <typename TransformOpTy, typename Range>
-DiagnosedSilenceableFailure
-applyTransformToEach(TransformOpTy transformOp, Range &&targets,
-                     SmallVectorImpl<ApplyToEachResultList> &results,
-                     TransformState &state) {
+DiagnosedSilenceableFailure applyTransformToEach(
+    TransformOpTy transformOp, TransformRewriter &rewriter, Range &&targets,
+    SmallVectorImpl<ApplyToEachResultList> &results, TransformState &state) {
   using OpTy = typename llvm::function_traits<
-      decltype(&TransformOpTy::applyToOne)>::template arg_t<0>;
+      decltype(&TransformOpTy::applyToOne)>::template arg_t<1>;
   static_assert(std::is_convertible<OpTy, Operation *>::value,
                 "expected transform function to take an operation");
+  OpBuilder::InsertionGuard g(rewriter);
 
   SmallVector<Diagnostic> silenceableStack;
   unsigned expectedNumResults = transformOp->getNumResults();
@@ -1331,8 +1365,9 @@ applyTransformToEach(TransformOpTy transformOp, Range &&targets,
     ApplyToEachResultList partialResults;
     partialResults.reserve(expectedNumResults);
     Location specificOpLoc = specificOp->getLoc();
+    rewriter.setInsertionPoint(specificOp);
     DiagnosedSilenceableFailure res =
-        transformOp.applyToOne(specificOp, partialResults, state);
+        transformOp.applyToOne(rewriter, specificOp, partialResults, state);
     if (res.isDefiniteFailure())
       return DiagnosedSilenceableFailure::definiteFailure();
 
@@ -1367,7 +1402,8 @@ LogicalResult checkNestedConsumption(Location loc,
 template <typename OpTy>
 mlir::DiagnosedSilenceableFailure
 mlir::transform::TransformEachOpTrait<OpTy>::apply(
-    TransformResults &transformResults, TransformState &state) {
+    TransformRewriter &rewriter, TransformResults &transformResults,
+    TransformState &state) {
   Value handle = this->getOperation()->getOperand(0);
   auto targets = state.getPayloadOps(handle);
 
@@ -1403,7 +1439,7 @@ mlir::transform::TransformEachOpTrait<OpTy>::apply(
   // corresponding results entry.
   SmallVector<ApplyToEachResultList, 1> results;
   DiagnosedSilenceableFailure result = detail::applyTransformToEach(
-      cast<OpTy>(this->getOperation()), targets, results, state);
+      cast<OpTy>(this->getOperation()), rewriter, targets, results, state);
 
   // Step 3. Propagate the definite failure if any and bail out.
   if (result.isDefiniteFailure())
diff --git a/mlir/include/mlir/Dialect/Transform/IR/TransformInterfaces.td b/mlir/include/mlir/Dialect/Transform/IR/TransformInterfaces.td
index 07f8ebcf8d6d9..eaab05766fc45 100644
--- a/mlir/include/mlir/Dialect/Transform/IR/TransformInterfaces.td
+++ b/mlir/include/mlir/Dialect/Transform/IR/TransformInterfaces.td
@@ -41,10 +41,15 @@ def TransformOpInterface : OpInterface<"TransformOpInterface"> {
         transformation represented by the current op is targeted. Returns a
         special status object indicating whether the transformation succeeded
         or failed, and, if it failed, whether the failure is recoverable or not.
+
+        IR must be created, modified and deleted with the provided rewriter.
+        implementations are responsible for setting the insertion point of the
+        rewriter to the desired location.
       }],
       /*returnType=*/"::mlir::DiagnosedSilenceableFailure",
       /*name=*/"apply",
       /*arguments=*/(ins
+          "::mlir::transform::TransformRewriter &":$rewriter,
           "::mlir::transform::TransformResults &":$transformResults,
           "::mlir::transform::TransformState &":$state
     )>,
@@ -194,6 +199,10 @@ def ParamProducerTransformOpTrait : NativeOpTrait<"ParamProducerTransformOpTrait
   let cppNamespace = "::mlir::transform";
 }
 
+def ReportTrackingListenerFailuresOpTrait : NativeOpTrait<"ReportTrackingListenerFailuresOpTrait"> {
+  let cppNamespace = "::mlir::transform";
+}
+
 def FindPayloadReplacementOpInterface
     : OpInterface<"FindPayloadReplacementOpInterface"> {
   let description = [{
diff --git a/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td b/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td
index 6f9aaa6f379d7..a5ed64ee262b3 100644
--- a/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td
+++ b/mlir/include/mlir/Dialect/Transform/IR/TransformOps.td
@@ -129,7 +129,8 @@ def AnnotateOp : TransformDialectOp<"annotate",
 
 def ApplyPatternsOp : TransformDialectOp<"apply_patterns",
     [TransformOpInterface, TransformEachOpTrait,
-     DeclareOpInterfaceMethods<MemoryEffectsOpInterface>]
+     DeclareOpInterfaceMethods<MemoryEffectsOpInterface>,
+     ReportTrackingListenerFailuresOpTrait]
         # GraphRegionNoTerminator.traits> {
   let summary = "Greedily applies patterns to the body of the targeted op";
   let description = [{
@@ -149,8 +150,8 @@ def ApplyPatternsOp : TransformDialectOp<"apply_patterns",
     considered "payload op replacements". Furthermore, only if the replacement
     values are defined by the same op and that op has the same type as the
     original op, the mapping is updated. Otherwise, this transform fails
-    silently unless `fail_on_payload_replacement_not_found` is set to "false".
-    More details can be found at the documentation site of `TrackingListener`.
+    silently. More details can be found at the documentation site of
+    `TrackingListener`.
 
     This transform also fails silently if the pattern application did not
     converge within the default number of iterations/rewrites of the greedy
@@ -158,8 +159,7 @@ def ApplyPatternsOp : TransformDialectOp<"apply_patterns",
   }];
 
   let arguments = (ins
-    TransformHandleTypeInterface:$target,
-    DefaultValuedAttr<BoolAttr, "true">:$fail_on_payload_replacement_not_found);
+    TransformHandleTypeInterface:$target);
   let results = (outs);
   let regions = (region MaxSizedRegion<1>:$region);
 
@@ -171,12 +171,12 @@ def ApplyPatternsOp : TransformDialectOp<"apply_patterns",
     OpBuilder<(ins
         "Value":$target,
         CArg<"function_ref<void(OpBuilder &, Location)>", "nullptr">:
-            $bodyBuilder,
-        CArg<"bool", "true">:$failOnPayloadReplacementNotFound)>,
+            $bodyBuilder)>,
   ];
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+      ::mlir::transform::TransformRewriter &rewriter,
       ::mlir::Operation *target,
       ::mlir::transform::ApplyToEachResultList &results,
       ::mlir::transform::TransformState &state);
@@ -222,6 +222,7 @@ def ApplyRegisteredPassOp : TransformDialectOp<"apply_registered_pass",
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+      ::mlir::transform::TransformRewriter &rewriter, 
       ::mlir::Operation *target,
       ::mlir::transform::ApplyToEachResultList &results,
       ::mlir::transform::TransformState &state);
@@ -238,6 +239,7 @@ def CastOp : TransformDialectOp<"cast",
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+      ::mlir::transform::TransformRewriter &rewriter,
       ::mlir::Operation *target,
       ::mlir::transform::ApplyToEachResultList &results,
       ::mlir::transform::TransformState &state);
diff --git a/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp b/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp
index 5a7f092e80bce..0b4570c65c515 100644
--- a/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp
+++ b/mlir/lib/Dialect/Affine/TransformOps/AffineTransformOps.cpp
@@ -63,7 +63,8 @@ struct SimplifyAffineMinMaxOp : public OpRewritePattern<OpTy> {
 } // namespace
 
 DiagnosedSilenceableFailure
-SimplifyBoundedAffineOpsOp::apply(TransformResults &results,
+SimplifyBoundedAffineOpsOp::apply(transform::TransformRewriter &rewriter,
+                                  TransformResults &results,
                                   TransformState &state) {
   // Get constraints for bounded values.
   SmallVector<int64_t> lbs;
@@ -127,6 +128,8 @@ SimplifyBoundedAffineOpsOp::apply(TransformResults &results,
                   SimplifyAffineMinMaxOp<AffineMaxOp>>(getContext(), cstr);
   FrozenRewritePatternSet frozenPatterns(std::move(patterns));
   GreedyRewriteConfig config;
+  config.listener =
+      static_cast<RewriterBase::Listener *>(rewriter.getListener());
   config.strictMode = GreedyRewriteStrictness::ExistingAndNewOps;
   // Apply the simplification pattern to a fixpoint.
   if (failed(applyOpPatternsAndFold(targets, frozenPatterns, config))) {
diff --git a/mlir/lib/Dialect/Bufferization/TransformOps/BufferizationTransformOps.cpp b/mlir/lib/Dialect/Bufferization/TransformOps/BufferizationTransformOps.cpp
index 9d45442add5b8..9c23ad6bfd902 100644
--- a/mlir/lib/Dialect/Bufferization/TransformOps/BufferizationTransformOps.cpp
+++ b/mlir/lib/Dialect/Bufferization/TransformOps/BufferizationTransformOps.cpp
@@ -26,7 +26,8 @@ using namespace mlir::transform;
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure
-transform::OneShotBufferizeOp::apply(TransformResults &transformResults,
+transform::OneShotBufferizeOp::apply(transform::TransformRewriter &rewriter,
+                                     TransformResults &transformResults,
                                      TransformState &state) {
   OneShotBufferizationOptions options;
   options.allowReturnAllocs = getAllowReturnAllocs();
@@ -71,10 +72,9 @@ void transform::EliminateEmptyTensorsOp::getEffects(
   modifiesPayload(effects);
 }
 
-DiagnosedSilenceableFailure
-transform::EliminateEmptyTensorsOp::apply(TransformResults &transformResults,
-                                          TransformState &state) {
-  IRRewriter rewriter(getContext());
+DiagnosedSilenceableFailure transform::EliminateEmptyTensorsOp::apply(
+    transform::TransformRewriter &rewriter, TransformResults &transformResults,
+    TransformState &state) {
   OneShotBufferizationOptions options;
   options.allowReturnAllocs = true;
 
@@ -95,11 +95,9 @@ transform::EliminateEmptyTensorsOp::apply(TransformResults &transformResults,
 // EmptyTensorToAllocTensorOp
 //===----------------------------------------------------------------------===//
 
-DiagnosedSilenceableFailure
-EmptyTensorToAllocTensorOp::applyToOne(tensor::EmptyOp target,
-                                       ApplyToEachResultList &results,
-                                       transform::TransformState &state) {
-  IRRewriter rewriter(target->getContext());
+DiagnosedSilenceableFailure EmptyTensorToAllocTensorOp::applyToOne(
+    transform::TransformRewriter &rewriter, tensor::EmptyOp target,
+    ApplyToEachResultList &results, transform::TransformState &state) {
   rewriter.setInsertionPoint(target);
   auto alloc = rewriter.replaceOpWithNewOp<bufferization::AllocTensorOp>(
       target, target.getType(), target.getDynamicSizes());
diff --git a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
index e9027d103cb6b..28b645e0ebc68 100644
--- a/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
+++ b/mlir/lib/Dialect/GPU/TransformOps/GPUTransformOps.cpp
@@ -398,7 +398,7 @@ createGpuLaunch(RewriterBase &rewriter, Location loc,
 
 /// Alter kernel configuration of the given kernel.
 static DiagnosedSilenceableFailure
-alterGpuLaunch(IRRewriter &rewriter, LaunchOp gpuLaunch,
+alterGpuLaunch(RewriterBase &rewriter, LaunchOp gpuLaunch,
                TransformOpInterface transformOp,
                std::optional<int64_t> gridDimX = std::nullopt,
                std::optional<int64_t> gridDimY = std::nullopt,
@@ -661,12 +661,10 @@ mlir::transform::gpu::findTopLevelForallOp(Operation *target,
   return DiagnosedSilenceableFailure::success();
 }
 
-DiagnosedSilenceableFailure
-transform::MapForallToBlocks::applyToOne(Operation *target,
-                                         ApplyToEachResultList &results,
-                                         transform::TransformState &state) {
+DiagnosedSilenceableFailure transform::MapForallToBlocks::applyToOne(
+    transform::TransformRewriter &rewriter, Operation *target,
+    ApplyToEachResultList &results, transform::TransformState &state) {
   LaunchOp gpuLaunch = dyn_cast<LaunchOp>(target);
-  IRRewriter rewriter(getContext());
   auto transformOp = cast<TransformOpInterface>(getOperation());
 
   if (!getGenerateGpuLaunch() && !gpuLaunch) {
@@ -856,7 +854,8 @@ DiagnosedSilenceableFailure mlir::transform::gpu::mapNestedForallToThreadsImpl(
 }
 
 DiagnosedSilenceableFailure transform::MapNestedForallToThreads::applyToOne(
-    Operation *target, ApplyToEachResultList &results, TransformState &state) {
+    transform::TransformRewriter &rewriter, Operation *target,
+    ApplyToEachResultList &results, TransformState &state) {
   LaunchOp gpuLaunch = dyn_cast<LaunchOp>(target);
   auto transformOp = cast<TransformOpInterface>(getOperation());
 
@@ -877,7 +876,6 @@ DiagnosedSilenceableFailure transform::MapNestedForallToThreads::applyToOne(
 
   // Set the GPU launch configuration for the block dims early, this is not
   // subject to IR inspection.
-  IRRewriter rewriter(getContext());
   diag = alterGpuLaunch(rewriter, gpuLaunch, transformOp, std::nullopt,
                         std::nullopt, std::nullopt, blockDims[0], blockDims[1],
                         blockDims[2]);
diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
index 4a9fc8e51954c..042476390c280 100644
--- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
+++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp
@@ -169,13 +169,11 @@ void transform::ApplyTilingCanonicalizationPatternsOp::populatePatterns(
 // BufferizeToAllocationOp
 //===----------------------------------------------------------------------===//
 
-DiagnosedSilenceableFailure
-transform::BufferizeToAllocationOp::apply(transform::TransformResults &results,
-                                          transform::TransformState &state) {
+DiagnosedSilenceableFailure transform::BufferizeToAllocationOp::apply(
+    transform::TransformRewriter &rewriter,
+    transform::TransformResults &results, transform::TransformState &state) {
   Attribute memorySpace =
       getMemorySpace().has_value() ? getMemorySpace().value() : Attribute();
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(getContext(), &listener);
   auto transformed = llvm::to_vector(
       llvm::map_range(state.getPayloadValues(getTarget()), [&](Value v) {
         return linalg::bufferizeToAllocation(rewriter, v, memorySpace);
@@ -196,7 +194,8 @@ void transform::BufferizeToAllocationOp::getEffects(
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure
-transform::DecomposeOp::applyToOne(LinalgOp target,
+transform::DecomposeOp::applyToOne(transform::TransformRewriter &rewriter,
+                                   LinalgOp target,
                                    transform::ApplyToEachResultList &results,
                                    transform::TransformState &state) {
 #define DOWNSCALE(trans)                                                       \
@@ -286,7 +285,8 @@ static LogicalResult applyTilingToAll(
 }
 
 DiagnosedSilenceableFailure
-transform::FuseOp::apply(mlir::transform::TransformResults &transformResults,
+transform::FuseOp::apply(transform::TransformRewriter &rewriter,
+                         mlir::transform::TransformResults &transformResults,
                          mlir::transform::TransformState &state) {
   SmallVector<int64_t> tileSizes = extractFromI64ArrayAttr(getTileSizes());
   SmallVector<int64_t> tileInterchange =
@@ -297,8 +297,6 @@ transform::FuseOp::apply(mlir::transform::TransformResults &transformResults,
   tilingOptions = tilingOptions.setTileSizes(tileSizes);
   scf::SCFTileAndFuseOptions tileAndFuseOptions;
   tileAndFuseOptions.tilingOptions = tilingOptions;
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(getContext(), &listener);
   LogicalResult result = applyTilingToAll(
       rewriter, getOperation(), state.getPayloadOps(getTarget()),
       tileSizes.size() - llvm::count(tileSizes, 0), transformResults,
@@ -721,7 +719,8 @@ static void forciblyReplaceReferencedPayloadOperation(TransformState &state,
 }
 
 DiagnosedSilenceableFailure
-transform::FuseIntoContainingOp::apply(transform::TransformResults &results,
+transform::FuseIntoContainingOp::apply(transform::TransformRewriter &rewriter,
+                                       transform::TransformResults &results,
                                        transform::TransformState &state) {
   SmallVector<Operation *> fusedOps;
   auto producerOps = state.getPayloadOps(getProducerOp());
@@ -764,8 +763,6 @@ transform::FuseIntoContainingOp::apply(transform::TransformResults &results,
     return failure();
   };
 
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(getContext(), &listener);
   while (!remainingProducers.empty()) {
     auto nextProducer = getNextProducer();
     if (failed(nextProducer)) {
@@ -842,7 +839,8 @@ void transform::FuseIntoContainingOp::getEffects(
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure
-transform::GeneralizeOp::applyToOne(LinalgOp target,
+transform::GeneralizeOp::applyToOne(transform::TransformRewriter &rewriter,
+                                    LinalgOp target,
                                     transform::ApplyToEachResultList &results,
                                     transform::TransformState &state) {
   // Exit early if no transformation is needed.
@@ -850,8 +848,6 @@ transform::GeneralizeOp::applyToOne(LinalgOp target,
     results.push_back(target);
     return DiagnosedSilenceableFailure::success();
   }
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(getContext(), &listener);
   rewriter.setInsertionPoint(target);
   FailureOr<LinalgOp> generic = generalizeNamedOp(rewriter, target);
   if (succeeded(generic)) {
@@ -866,7 +862,8 @@ transform::GeneralizeOp::applyToOne(LinalgOp target,
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure
-transform::InterchangeOp::applyToOne(GenericOp target,
+transform::InterchangeOp::applyToOne(transform::TransformRewriter &rewriter,
+                                     GenericOp target,
                                      transform::ApplyToEachResultList &results,
                                      transform::TransformState &state) {
   ArrayRef<int64_t> interchangeVector = getIteratorInterchange();
@@ -875,8 +872,6 @@ transform::InterchangeOp::applyToOne(GenericOp target,
     results.push_back(target);
     return DiagnosedSilenceableFailure::success();
   }
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(getContext(), &listener);
   FailureOr<GenericOp> res =
       interchangeGenericOp(rewriter, target,
                            SmallVector<unsigned>(interchangeVector.begin(),
@@ -904,10 +899,9 @@ LogicalResult transform::InterchangeOp::verify() {
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure transform::LowerPackOp::applyToOne(
-    tensor::PackOp target, transform::ApplyToEachResultList &transformResults,
+    transform::TransformRewriter &rewriter, tensor::PackOp target,
+    transform::ApplyToEachResultList &transformResults,
     transform::TransformState &state) {
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(getContext(), &listener);
   rewriter.setInsertionPoint(target);
   FailureOr<LowerPackResult> res = lowerPack(rewriter, target);
   if (failed(res)) {
@@ -925,10 +919,9 @@ DiagnosedSilenceableFailure transform::LowerPackOp::applyToOne(
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure transform::LowerUnPackOp::applyToOne(
-    tensor::UnPackOp target, transform::ApplyToEachResultList &transformResults,
+    transform::TransformRewriter &rewriter, tensor::UnPackOp target,
+    transform::ApplyToEachResultList &transformResults,
     transform::TransformState &state) {
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(getContext(), &listener);
   rewriter.setInsertionPoint(target);
   FailureOr<LowerUnPackOpResult> res = lowerUnPack(rewriter, target);
   if (failed(res)) {
@@ -964,7 +957,8 @@ void transform::MatchOp::build(OpBuilder &builder, OperationState &result,
 }
 
 DiagnosedSilenceableFailure
-transform::MatchOp::apply(transform::TransformResults &results,
+transform::MatchOp::apply(transform::TransformRewriter &rewriter,
+                          transform::TransformResults &results,
                           transform::TransformState &state) {
   llvm::StringSet<> strs;
   if (getOps().has_value())
@@ -1053,8 +1047,8 @@ static ParseResult parseMultitileSizesTypes(OpAsmParser &parser,
 }
 
 DiagnosedSilenceableFailure transform::MultiTileSizesOp::applyToOne(
-    LinalgOp target, transform::ApplyToEachResultList &results,
-    TransformState &state) {
+    transform::TransformRewriter &rewriter, LinalgOp target,
+    transform::ApplyToEachResultList &results, TransformState &state) {
   if (isa<TransformParamTypeInterface>(getLowSize().getType())) {
     if (target.hasDynamicShape()) {
       auto diag = emitSilenceableError()
@@ -1155,7 +1149,8 @@ SmallVector<OpFoldResult> transform::PackOp::getMixedPackedSizes() {
 }
 
 DiagnosedSilenceableFailure
-transform::PackOp::apply(transform::TransformResults &transformResults,
+transform::PackOp::apply(transform::TransformRewriter &rewriter,
+                         transform::TransformResults &transformResults,
                          transform::TransformState &state) {
   auto targetOps = state.getPayloadOps(getTarget());
   // If nothing to pack, propagate success.
@@ -1184,8 +1179,6 @@ transform::PackOp::apply(transform::TransformResults &transformResults,
   DiagnosedSilenceableFailure status = unpackSingleIndexResultPayloadOperations(
       state, *this, packedSizes, getMixedPackedSizes());
 
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(getContext(), &listener);
   rewriter.setInsertionPoint(linalgOp);
   FailureOr<PackResult> maybeResult = pack(rewriter, linalgOp, packedSizes);
   if (failed(maybeResult))
@@ -1364,11 +1357,10 @@ packMatmulGreedily(RewriterBase &rewriter, LinalgOp linalgOp,
 }
 
 DiagnosedSilenceableFailure
-PackGreedilyOp::apply(transform::TransformResults &transformResults,
+PackGreedilyOp::apply(transform::TransformRewriter &rewriter,
+                      transform::TransformResults &transformResults,
                       transform::TransformState &state) {
   SmallVector<Operation *> results;
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(getContext(), &listener);
   for (Operation *op : state.getPayloadOps(getTarget())) {
     auto linalgOp = dyn_cast<LinalgOp>(op);
     if (!linalgOp)
@@ -1464,7 +1456,8 @@ bool isValidPackingPermutation(
 }
 
 DiagnosedSilenceableFailure
-transform::PackTransposeOp::apply(transform::TransformResults &transformResults,
+transform::PackTransposeOp::apply(transform::TransformRewriter &rewriter,
+                                  transform::TransformResults &transformResults,
                                   transform::TransformState &state) {
   auto packOrUnpackOps = state.getPayloadOps(getTargetPackOrUnPackOp());
   auto linalgOps = state.getPayloadOps(getTargetLinalgOp());
@@ -1542,8 +1535,6 @@ transform::PackTransposeOp::apply(transform::TransformResults &transformResults,
   assert(packOp && linalgOp && "unexpected null op");
 
   // Step 3. Actually transpose the ops.
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(getContext(), &listener);
   FailureOr<PackTransposeResult> res = packTranspose(
       rewriter, packOp, linalgOp, unPackOp, getOuterPerm(), getInnerPerm());
   // Preconditions have been checked, it is an error to fail here.
@@ -1568,7 +1559,8 @@ transform::PackTransposeOp::apply(transform::TransformResults &transformResults,
 //===---------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure
-transform::PadOp::applyToOne(LinalgOp target,
+transform::PadOp::applyToOne(transform::TransformRewriter &rewriter,
+                             LinalgOp target,
                              transform::ApplyToEachResultList &results,
                              transform::TransformState &state) {
   // Convert the integer packing flags to booleans.
@@ -1616,8 +1608,6 @@ transform::PadOp::applyToOne(LinalgOp target,
     transposePaddings.push_back(
         extractFromI64ArrayAttr(cast<ArrayAttr>(transposeVector)));
 
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(getContext(), &listener);
   LinalgOp paddedOp;
   SmallVector<int64_t> paddingDimensions =
       extractFromI64ArrayAttr(getPaddingDimensions());
@@ -1684,6 +1674,7 @@ LogicalResult transform::PadOp::verify() {
 //===---------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure transform::HoistPadBuildPackingLoopNestOp::apply(
+    transform::TransformRewriter &rewriter,
     transform::TransformResults &transformResults,
     transform::TransformState &state) {
   auto targetOps = state.getPayloadOps(getTarget());
@@ -1700,8 +1691,6 @@ DiagnosedSilenceableFailure transform::HoistPadBuildPackingLoopNestOp::apply(
   if (!padOp || !loopOp)
     return emitDefiniteFailure() << "requires exactly 2 non-null handles";
 
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(getContext(), &listener);
   FailureOr<linalg::detail::PackingResult> result =
       linalg::detail::buildPackingLoopNest(rewriter, padOp, loopOp,
                                            getTranspose());
@@ -1740,13 +1729,12 @@ void transform::HoistPadBuildPackingLoopNestOp::getEffects(
 }
 
 DiagnosedSilenceableFailure
-transform::HoistPadOp::applyToOne(tensor::PadOp target,
+transform::HoistPadOp::applyToOne(transform::TransformRewriter &rewriter,
+                                  tensor::PadOp target,
                                   transform::ApplyToEachResultList &results,
                                   transform::TransformState &state) {
   tensor::PadOp hoistedPadOp;
   SmallVector<GenericOp> transposeOps;
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(getContext(), &listener);
   FailureOr<Value> result =
       hoistPaddingOnTensors(rewriter, target, getNumLoops(), getTranspose(),
                             hoistedPadOp, transposeOps);
@@ -1779,7 +1767,8 @@ LogicalResult transform::HoistPadOp::verify() {
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure
-transform::PromoteOp::applyToOne(LinalgOp target,
+transform::PromoteOp::applyToOne(transform::TransformRewriter &rewriter,
+                                 LinalgOp target,
                                  transform::ApplyToEachResultList &results,
                                  transform::TransformState &state) {
   LinalgPromotionOptions promotionOptions;
@@ -1829,8 +1818,6 @@ transform::PromoteOp::applyToOne(LinalgOp target,
   if (failed(promoteSubviewsPrecondition(target, promotionOptions)))
     return emitDefaultDefiniteFailure(target);
 
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(getContext(), &listener);
   rewriter.setInsertionPoint(target);
   FailureOr<LinalgOp> res = promoteSubViews(rewriter, target, promotionOptions);
   if (failed(res))
@@ -1844,7 +1831,8 @@ transform::PromoteOp::applyToOne(LinalgOp target,
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure
-transform::ReplaceOp::apply(TransformResults &transformResults,
+transform::ReplaceOp::apply(transform::TransformRewriter &rewriter,
+                            TransformResults &transformResults,
                             TransformState &state) {
   auto payload = state.getPayloadOps(getTarget());
 
@@ -1859,8 +1847,6 @@ transform::ReplaceOp::apply(TransformResults &transformResults,
   }
 
   // Clone and replace.
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(getContext(), &listener);
   Operation *pattern = &getBodyRegion().front().front();
   SmallVector<Operation *> replacements;
   for (Operation *target : payload) {
@@ -1904,7 +1890,8 @@ LogicalResult transform::ReplaceOp::verify() {
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure
-transform::ScalarizeOp::applyToOne(LinalgOp target,
+transform::ScalarizeOp::applyToOne(transform::TransformRewriter &rewriter,
+                                   LinalgOp target,
                                    transform::ApplyToEachResultList &results,
                                    transform::TransformState &state) {
   scf::SCFTilingOptions tilingOptions;
@@ -1916,8 +1903,6 @@ transform::ScalarizeOp::applyToOne(LinalgOp target,
     AffineMap map = target.getShapesToLoopsMap();
     if (!map)
       return tileSizes;
-    TrackingListener listener(state, *this);
-    IRRewriter rewriter(getContext(), &listener);
     SmallVector<OpFoldResult> shapeSizes =
         affine::makeComposedFoldedMultiResultAffineApply(rewriter, loc, map,
                                                          allShapeSizes);
@@ -1931,8 +1916,6 @@ transform::ScalarizeOp::applyToOne(LinalgOp target,
     return tileSizes;
   });
   SmallVector<int64_t> emptyTileSizes;
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(getContext(), &listener);
   rewriter.setInsertionPoint(target);
   FailureOr<scf::SCFTilingResult> maybeTilingResult = tileUsingSCFForOp(
       rewriter, cast<TilingInterface>(target.getOperation()), tilingOptions);
@@ -1956,11 +1939,10 @@ transform::ScalarizeOp::applyToOne(LinalgOp target,
 
 DiagnosedSilenceableFailure
 transform::RewriteInDestinationPassingStyleOp::applyToOne(
-    Operation *target, transform::ApplyToEachResultList &results,
+    transform::TransformRewriter &rewriter, Operation *target,
+    transform::ApplyToEachResultList &results,
     transform::TransformState &state) {
   SmallVector<Operation *> res;
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(getContext(), &listener);
   rewriter.setInsertionPoint(target);
   FailureOr<Operation *> maybeResult =
       TypeSwitch<Operation *, FailureOr<Operation *>>(target)
@@ -1978,13 +1960,12 @@ transform::RewriteInDestinationPassingStyleOp::applyToOne(
 // SplitOp
 //===----------------------------------------------------------------------===//
 
-DiagnosedSilenceableFailure SplitOp::apply(TransformResults &results,
-                                           TransformState &state) {
+DiagnosedSilenceableFailure
+SplitOp::apply(transform::TransformRewriter &rewriter,
+               TransformResults &results, TransformState &state) {
   // Collect the dynamic split points if provided.
   SmallVector<Operation *> payload =
       llvm::to_vector(state.getPayloadOps(getTarget()));
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(getContext(), &listener);
   SmallVector<OpFoldResult> splitPoints;
   splitPoints.reserve(payload.size());
   if (getDynamicSplitPoint()) {
@@ -2184,15 +2165,14 @@ void transform::SplitReductionOp::build(
 }
 
 DiagnosedSilenceableFailure transform::SplitReductionOp::applyToOne(
-    LinalgOp target, transform::ApplyToEachResultList &results,
+    transform::TransformRewriter &rewriter, LinalgOp target,
+    transform::ApplyToEachResultList &results,
     transform::TransformState &state) {
   ControlSplitReductionFn splitFn = [&](LinalgOp) {
     return linalg::SplitReductionOptions{int64_t(getSplitFactor()),
                                          unsigned(getInsertSplitDimension()),
                                          bool(getInnerParallel())};
   };
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(getContext(), &listener);
   rewriter.setInsertionPoint(target);
   FailureOr<SplitReductionResult> splitResult =
       (getUseScalingAlgorithm())
@@ -2230,10 +2210,9 @@ void transform::TileReductionUsingScfOp::build(
 }
 
 DiagnosedSilenceableFailure transform::TileReductionUsingScfOp::applyToOne(
-    LinalgOp target, transform::ApplyToEachResultList &results,
+    transform::TransformRewriter &rewriter, LinalgOp target,
+    transform::ApplyToEachResultList &results,
     transform::TransformState &state) {
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(getContext(), &listener);
   rewriter.setInsertionPoint(target);
   FailureOr<scf::SCFReductionTilingResult> result = scf::tileReductionUsingScf(
       rewriter, cast<PartialReductionOpInterface>(target.getOperation()),
@@ -2274,10 +2253,9 @@ void transform::TileReductionUsingForallOp::build(
 }
 
 DiagnosedSilenceableFailure transform::TileReductionUsingForallOp::applyToOne(
-    LinalgOp target, transform::ApplyToEachResultList &results,
+    transform::TransformRewriter &rewriter, LinalgOp target,
+    transform::ApplyToEachResultList &results,
     transform::TransformState &state) {
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(getContext(), &listener);
   rewriter.setInsertionPoint(target);
   SmallVector<OpFoldResult> numThreads =
       getAsOpFoldResult(rewriter.getI64ArrayAttr(getNumThreads()));
@@ -2363,7 +2341,8 @@ void transform::TileOp::build(OpBuilder &builder, OperationState &result,
 }
 
 DiagnosedSilenceableFailure
-transform::TileOp::apply(TransformResults &transformResults,
+transform::TileOp::apply(transform::TransformRewriter &rewriter,
+                         TransformResults &transformResults,
                          TransformState &state) {
   ArrayRef<int64_t> tileSizes = getStaticSizes();
 
@@ -2478,8 +2457,6 @@ transform::TileOp::apply(TransformResults &transformResults,
     }
 
     tilingOptions.setInterchange(getInterchange());
-    TrackingListener listener(state, *this);
-    IRRewriter rewriter(getContext(), &listener);
     FailureOr<scf::SCFTilingResult> maybeTilingResult =
         tileUsingSCFForOp(rewriter, tilingInterface, tilingOptions);
     if (failed(maybeTilingResult))
@@ -2720,45 +2697,44 @@ DiagnosedSilenceableFailure transform::tileToForallOpImpl(
 }
 
 DiagnosedSilenceableFailure
-transform::TileToForallOp::apply(transform::TransformResults &transformResults,
+transform::TileToForallOp::apply(transform::TransformRewriter &rewriter,
+                                 transform::TransformResults &transformResults,
                                  transform::TransformState &state) {
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(getContext(), &listener);
-  auto transformOp = cast<TransformOpInterface>(getOperation());
-
-  // Result payload ops.
-  SmallVector<Operation *> tileOps;
-  SmallVector<Operation *> tiledOps;
-
-  // Unpack handles.
-  SmallVector<OpFoldResult> mixedNumThreads;
-  DiagnosedSilenceableFailure status =
-      getPackedNumThreads()
-          ? unpackSingleIndexResultPayloadOperations(
-                state, transformOp, mixedNumThreads, getPackedNumThreads())
-          : unpackSingleIndexResultPayloadOperations(
-                state, transformOp, mixedNumThreads, getMixedNumThreads());
-  if (!status.succeeded())
-    return status;
-  SmallVector<OpFoldResult> mixedTileSizes;
-  status = getPackedTileSizes()
-               ? unpackSingleIndexResultPayloadOperations(
-                     state, transformOp, mixedTileSizes, getPackedTileSizes())
-               : unpackSingleIndexResultPayloadOperations(
-                     state, transformOp, mixedTileSizes, getMixedTileSizes());
-  if (!status.succeeded())
-    return status;
-
-  for (Operation *target : state.getPayloadOps(getTarget())) {
-    linalg::ForallTilingResult tilingResult;
-    DiagnosedSilenceableFailure diag = tileToForallOpImpl(
-        rewriter, state, transformOp, target, mixedNumThreads, mixedTileSizes,
-        getMapping(), tilingResult);
-    if (!diag.succeeded())
+    auto transformOp = cast<TransformOpInterface>(getOperation());
+
+    // Result payload ops.
+    SmallVector<Operation *> tileOps;
+    SmallVector<Operation *> tiledOps;
+
+    // Unpack handles.
+    SmallVector<OpFoldResult> mixedNumThreads;
+    DiagnosedSilenceableFailure status =
+        getPackedNumThreads()
+            ? unpackSingleIndexResultPayloadOperations(
+                  state, transformOp, mixedNumThreads, getPackedNumThreads())
+            : unpackSingleIndexResultPayloadOperations(
+                  state, transformOp, mixedNumThreads, getMixedNumThreads());
+    if (!status.succeeded())
+      return status;
+    SmallVector<OpFoldResult> mixedTileSizes;
+    status = getPackedTileSizes()
+                 ? unpackSingleIndexResultPayloadOperations(
+                       state, transformOp, mixedTileSizes, getPackedTileSizes())
+                 : unpackSingleIndexResultPayloadOperations(
+                       state, transformOp, mixedTileSizes, getMixedTileSizes());
+    if (!status.succeeded())
+      return status;
+
+    for (Operation *target : state.getPayloadOps(getTarget())) {
+      linalg::ForallTilingResult tilingResult;
+      DiagnosedSilenceableFailure diag = tileToForallOpImpl(
+          rewriter, state, transformOp, target, mixedNumThreads, mixedTileSizes,
+          getMapping(), tilingResult);
+      if (!diag.succeeded())
       return diag;
     tileOps.push_back(tilingResult.tileOp);
     tiledOps.push_back(tilingResult.tiledOp);
-  }
+    }
 
   transformResults.set(cast<OpResult>(getForallOp()), tileOps);
   transformResults.set(cast<OpResult>(getTiledOp()), tiledOps);
@@ -2833,7 +2809,8 @@ void transform::TileToScfForOp::build(OpBuilder &builder,
 }
 
 DiagnosedSilenceableFailure
-transform::TileToScfForOp::apply(TransformResults &transformResults,
+transform::TileToScfForOp::apply(transform::TransformRewriter &rewriter,
+                                 TransformResults &transformResults,
                                  TransformState &state) {
   ArrayRef<int64_t> tileSizes = getStaticSizes();
 
@@ -2902,8 +2879,6 @@ transform::TileToScfForOp::apply(TransformResults &transformResults,
     }
 
     tilingOptions.setInterchange(getInterchange());
-    TrackingListener listener(state, *this);
-    IRRewriter rewriter(getContext(), &listener);
     FailureOr<scf::SCFTilingResult> tilingResult =
         tileUsingSCFForOp(rewriter, tilingInterfaceOp, tilingOptions);
     if (failed(tilingResult))
@@ -3047,7 +3022,8 @@ struct VectorizationPattern : public RewritePattern {
 } // namespace
 
 DiagnosedSilenceableFailure
-transform::VectorizeOp::applyToOne(Operation *target,
+transform::VectorizeOp::applyToOne(transform::TransformRewriter &rewriter,
+                                   Operation *target,
                                    transform::ApplyToEachResultList &results,
                                    transform::TransformState &state) {
   if (!target->hasTrait<OpTrait::IsIsolatedFromAbove>()) {
@@ -3094,10 +3070,9 @@ transform::VectorizeOp::applyToOne(Operation *target,
 // MaskedVectorizeOp
 //===----------------------------------------------------------------------===//
 DiagnosedSilenceableFailure transform::MaskedVectorizeOp::apply(
+    transform::TransformRewriter &rewriter,
     mlir::transform::TransformResults &transformResults,
     mlir::transform::TransformState &state) {
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(getContext(), &listener);
   auto targets = state.getPayloadOps(getTarget());
   if (std::empty(targets))
     return DiagnosedSilenceableFailure::success();
@@ -3175,7 +3150,8 @@ SmallVector<OpFoldResult> MaskedVectorizeOp::getMixedVectorSizes() {
 
 DiagnosedSilenceableFailure
 transform::HoistRedundantVectorTransfersOp::applyToOne(
-    func::FuncOp target, transform::ApplyToEachResultList &results,
+    transform::TransformRewriter &rewriter, func::FuncOp target,
+    transform::ApplyToEachResultList &results,
     transform::TransformState &state) {
   // WARNING: This hoisting does not model parallelism and is generally
   // incorrect when used on distributed loops with memref semantics!
@@ -3190,10 +3166,9 @@ transform::HoistRedundantVectorTransfersOp::applyToOne(
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure transform::ConvertConv2DToImg2ColOp::applyToOne(
-    linalg::LinalgOp target, transform::ApplyToEachResultList &results,
+    transform::TransformRewriter &rewriter, linalg::LinalgOp target,
+    transform::ApplyToEachResultList &results,
     transform::TransformState &state) {
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(getContext(), &listener);
   rewriter.setInsertionPoint(target);
   auto maybeTransformed =
       TypeSwitch<Operation *, FailureOr<std::pair<Operation *, Operation *>>>(
@@ -3225,10 +3200,9 @@ DiagnosedSilenceableFailure transform::ConvertConv2DToImg2ColOp::applyToOne(
 
 DiagnosedSilenceableFailure
 transform::HoistRedundantTensorSubsetsOp::applyToOne(
-    Operation *target, transform::ApplyToEachResultList &results,
+    transform::TransformRewriter &rewriter, Operation *target,
+    transform::ApplyToEachResultList &results,
     transform::TransformState &state) {
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(getContext(), &listener);
   auto forOp = dyn_cast<scf::ForOp>(target);
   if (forOp) {
     linalg::hoistRedundantSubsetExtractInsert(rewriter, forOp);
@@ -3291,11 +3265,10 @@ DiagnosedSilenceableFailure doit(RewriterBase &rewriter, OpTy target,
 }
 
 DiagnosedSilenceableFailure transform::InsertSliceToCopyOp::applyToOne(
-    Operation *targetOp, transform::ApplyToEachResultList &results,
+    transform::TransformRewriter &rewriter, Operation *targetOp,
+    transform::ApplyToEachResultList &results,
     transform::TransformState &state) {
 
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(targetOp->getContext(), &listener);
   rewriter.setInsertionPoint(targetOp);
   if (auto target = dyn_cast<tensor::InsertSliceOp>(targetOp))
     return doit(rewriter, target, results, state);
diff --git a/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp b/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
index 0f84fe498d911..a46f2a37cb1d2 100644
--- a/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
+++ b/mlir/lib/Dialect/MemRef/TransformOps/MemRefTransformOps.cpp
@@ -60,10 +60,10 @@ void transform::ApplyResolveRankedShapedTypeResultDimsPatternsOp::
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure transform::MemRefMultiBufferOp::apply(
+    transform::TransformRewriter &rewriter,
     transform::TransformResults &transformResults,
     transform::TransformState &state) {
   SmallVector<Operation *> results;
-  IRRewriter rewriter(getContext());
   for (Operation *op : state.getPayloadOps(getTarget())) {
     bool canApplyMultiBuffer = true;
     auto target = cast<memref::AllocOp>(op);
@@ -105,7 +105,8 @@ DiagnosedSilenceableFailure transform::MemRefMultiBufferOp::apply(
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure transform::MemRefMakeLoopIndependentOp::applyToOne(
-    Operation *target, transform::ApplyToEachResultList &results,
+    transform::TransformRewriter &rewriter, Operation *target,
+    transform::ApplyToEachResultList &results,
     transform::TransformState &state) {
   // Gather IVs.
   SmallVector<Value> ivs;
@@ -123,7 +124,6 @@ DiagnosedSilenceableFailure transform::MemRefMakeLoopIndependentOp::applyToOne(
   }
 
   // Rewrite IR.
-  IRRewriter rewriter(target->getContext());
   FailureOr<Value> replacement = failure();
   if (auto allocaOp = dyn_cast<memref::AllocaOp>(target)) {
     replacement = memref::replaceWithIndependentOp(rewriter, allocaOp, ivs);
diff --git a/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp b/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
index edd1d328d57bc..43c7d7eb5ddfb 100644
--- a/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
+++ b/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp
@@ -35,7 +35,8 @@ void transform::ApplyForLoopCanonicalizationPatternsOp::populatePatterns(
 // GetParentForOp
 //===----------------------------------------------------------------------===//
 DiagnosedSilenceableFailure
-transform::GetParentForOp::apply(transform::TransformResults &results,
+transform::GetParentForOp::apply(transform::TransformRewriter &rewriter,
+                                 transform::TransformResults &results,
                                  transform::TransformState &state) {
   SetVector<Operation *> parents;
   for (Operation *target : state.getPayloadOps(getTarget())) {
@@ -92,7 +93,8 @@ static scf::ExecuteRegionOp wrapInExecuteRegion(RewriterBase &b,
 }
 
 DiagnosedSilenceableFailure
-transform::LoopOutlineOp::apply(transform::TransformResults &results,
+transform::LoopOutlineOp::apply(transform::TransformRewriter &rewriter,
+                                transform::TransformResults &results,
                                 transform::TransformState &state) {
   SmallVector<Operation *> functions;
   SmallVector<Operation *> calls;
@@ -100,7 +102,6 @@ transform::LoopOutlineOp::apply(transform::TransformResults &results,
   for (Operation *target : state.getPayloadOps(getTarget())) {
     Location location = target->getLoc();
     Operation *symbolTableOp = SymbolTable::getNearestSymbolTable(target);
-    IRRewriter rewriter(getContext());
     scf::ExecuteRegionOp exec = wrapInExecuteRegion(rewriter, target);
     if (!exec) {
       DiagnosedSilenceableFailure diag = emitSilenceableError()
@@ -135,11 +136,11 @@ transform::LoopOutlineOp::apply(transform::TransformResults &results,
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure
-transform::LoopPeelOp::applyToOne(scf::ForOp target,
+transform::LoopPeelOp::applyToOne(transform::TransformRewriter &rewriter,
+                                  scf::ForOp target,
                                   transform::ApplyToEachResultList &results,
                                   transform::TransformState &state) {
   scf::ForOp result;
-  IRRewriter rewriter(target->getContext());
   // This helper returns failure when peeling does not occur (i.e. when the IR
   // is not modified). This is not a failure for the op as the postcondition:
   //    "the loop trip count is divisible by the step"
@@ -192,7 +193,8 @@ loopScheduling(scf::ForOp forOp,
 }
 
 DiagnosedSilenceableFailure
-transform::LoopPipelineOp::applyToOne(scf::ForOp target,
+transform::LoopPipelineOp::applyToOne(transform::TransformRewriter &rewriter,
+                                      scf::ForOp target,
                                       transform::ApplyToEachResultList &results,
                                       transform::TransformState &state) {
   scf::PipeliningOption options;
@@ -203,7 +205,6 @@ transform::LoopPipelineOp::applyToOne(scf::ForOp target,
                        getReadLatency());
       };
   scf::ForLoopPipeliningPattern pattern(options, target->getContext());
-  IRRewriter rewriter(getContext());
   rewriter.setInsertionPoint(target);
   FailureOr<scf::ForOp> patternResult =
       scf::pipelineForLoop(rewriter, target, options);
@@ -219,7 +220,8 @@ transform::LoopPipelineOp::applyToOne(scf::ForOp target,
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure
-transform::LoopUnrollOp::applyToOne(Operation *op,
+transform::LoopUnrollOp::applyToOne(transform::TransformRewriter &rewriter,
+                                    Operation *op,
                                     transform::ApplyToEachResultList &results,
                                     transform::TransformState &state) {
   LogicalResult result(failure());
@@ -241,7 +243,8 @@ transform::LoopUnrollOp::applyToOne(Operation *op,
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure
-transform::LoopCoalesceOp::applyToOne(Operation *op,
+transform::LoopCoalesceOp::applyToOne(transform::TransformRewriter &rewriter,
+                                      Operation *op,
                                       transform::ApplyToEachResultList &results,
                                       transform::TransformState &state) {
   LogicalResult result(failure());
@@ -276,12 +279,10 @@ static void replaceOpWithRegion(RewriterBase &rewriter, Operation *op,
 }
 
 DiagnosedSilenceableFailure transform::TakeAssumedBranchOp::applyToOne(
-    scf::IfOp ifOp, transform::ApplyToEachResultList &results,
+    transform::TransformRewriter &rewriter, scf::IfOp ifOp,
+    transform::ApplyToEachResultList &results,
     transform::TransformState &state) {
-  TrackingListener listener(state, *this);
-  IRRewriter rewriter(ifOp->getContext(), &listener);
   rewriter.setInsertionPoint(ifOp);
-
   Region &region =
       getTakeElseBranch() ? ifOp.getElseRegion() : ifOp.getThenRegion();
   if (!llvm::hasSingleElement(region)) {
diff --git a/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp b/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
index b4b822976d99a..6ee4bfad23bbf 100644
--- a/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
+++ b/mlir/lib/Dialect/Tensor/TransformOps/TensorTransformOps.cpp
@@ -123,7 +123,8 @@ void transform::ApplyRewriteTensorOpsAsConstantPatternsOp::populatePatterns(
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure transform::MakeLoopIndependentOp::applyToOne(
-    Operation *target, transform::ApplyToEachResultList &results,
+    transform::TransformRewriter &rewriter, Operation *target,
+    transform::ApplyToEachResultList &results,
     transform::TransformState &state) {
   // Gather IVs.
   SmallVector<Value> ivs;
@@ -141,7 +142,6 @@ DiagnosedSilenceableFailure transform::MakeLoopIndependentOp::applyToOne(
   }
 
   // Rewrite IR.
-  IRRewriter rewriter(target->getContext());
   FailureOr<Value> replacement = failure();
   if (auto padOp = dyn_cast<tensor::PadOp>(target)) {
     replacement = tensor::buildIndependentOp(rewriter, padOp, ivs);
diff --git a/mlir/lib/Dialect/Transform/IR/TransformDialect.cpp b/mlir/lib/Dialect/Transform/IR/TransformDialect.cpp
index d4c4327392c28..d7205ec02690e 100644
--- a/mlir/lib/Dialect/Transform/IR/TransformDialect.cpp
+++ b/mlir/lib/Dialect/Transform/IR/TransformDialect.cpp
@@ -157,6 +157,13 @@ LogicalResult transform::TransformDialect::verifyOperationAttribute(
     }
     return success();
   }
+  if (attribute.getName().getValue() == kSilenceTrackingFailuresAttrName) {
+    if (!llvm::isa<UnitAttr>(attribute.getValue())) {
+      return op->emitError()
+             << attribute.getName() << " must be a unit attribute";
+    }
+    return success();
+  }
   return emitError(op->getLoc())
          << "unknown attribute: " << attribute.getName();
 }
diff --git a/mlir/lib/Dialect/Transform/IR/TransformInterfaces.cpp b/mlir/lib/Dialect/Transform/IR/TransformInterfaces.cpp
index 679a8601ccda1..32637b3d4442f 100644
--- a/mlir/lib/Dialect/Transform/IR/TransformInterfaces.cpp
+++ b/mlir/lib/Dialect/Transform/IR/TransformInterfaces.cpp
@@ -8,6 +8,7 @@
 
 #include "mlir/Dialect/Transform/IR/TransformInterfaces.h"
 
+#include "mlir/Dialect/Transform/IR/TransformDialect.h"
 #include "mlir/Dialect/Transform/IR/TransformTypes.h"
 #include "mlir/IR/Diagnostics.h"
 #include "mlir/IR/Operation.h"
@@ -896,12 +897,41 @@ transform::TransformState::applyTransform(TransformOpInterface transform) {
     return diag;
   }
 
+  // Prepare rewriter and listener.
+  transform::ErrorCheckingTrackingListener trackingListener(*this, transform);
+  transform::TransformRewriter rewriter(transform->getContext(),
+                                        &trackingListener);
+
   // Compute the result but do not short-circuit the silenceable failure case as
   // we still want the handles to propagate properly so the "suppress" mode can
   // proceed on a best effort basis.
   transform::TransformResults results(transform->getNumResults());
-  DiagnosedSilenceableFailure result(transform.apply(results, *this));
+  DiagnosedSilenceableFailure result(transform.apply(rewriter, results, *this));
   compactOpHandles();
+
+  // Error handling: fail if transform or listener failed.
+  DiagnosedSilenceableFailure trackingFailure =
+      trackingListener.checkAndResetError();
+  if (!transform->hasTrait<ReportTrackingListenerFailuresOpTrait>() ||
+      transform->hasAttr(
+          transform::TransformDialect::kSilenceTrackingFailuresAttrName)) {
+    // Only report failures for ReportTrackingListenerFailuresOpTrait ops. Also
+    // do not report failures if the above mentioned attribute is set.
+    if (trackingFailure.isSilenceableFailure())
+      (void)trackingFailure.silence();
+    trackingFailure = DiagnosedSilenceableFailure::success();
+  }
+  if (!trackingFailure.succeeded()) {
+    if (result.succeeded()) {
+      result = std::move(trackingFailure);
+    } else {
+      // Transform op errors have precedence, report those first.
+      if (result.isSilenceableFailure())
+        result.attachNote() << "tracking listener also failed: "
+                            << trackingFailure.getMessage();
+      (void)trackingFailure.silence();
+    }
+  }
   if (result.isDefiniteFailure())
     return result;
 
@@ -1161,6 +1191,16 @@ bool transform::TransformResults::isSet(unsigned resultNumber) const {
 // TrackingListener
 //===----------------------------------------------------------------------===//
 
+transform::TrackingListener::TrackingListener(TransformState &state,
+                                              TransformOpInterface op)
+    : TransformState::Extension(state), transformOp(op) {
+  if (op) {
+    for (OpOperand *opOperand : transformOp.getConsumedHandleOpOperands()) {
+      consumedHandles.insert(opOperand->get());
+    }
+  }
+}
+
 Operation *transform::TrackingListener::getCommonDefiningOp(ValueRange values) {
   Operation *defOp = nullptr;
   for (Value v : values) {
@@ -1267,15 +1307,34 @@ void transform::TrackingListener::notifyOperationReplaced(
     // Op is not tracked.
     return;
   }
+
+  // Helper function to check if the current transform op consumes any handle
+  // that is mapped to `op`.
+  //
+  // Note: If a handle was consumed, there shouldn't be any alive users, so it
+  // is not really necessary to check for consumed handles. However, in case
+  // there are indeed alive handles that were consumed (which is undefined
+  // behavior) and a replacement op could not be found, we want to fail with a
+  // nicer error message: "op uses a handle invalidated..." instead of "could
+  // not find replacement op". This nicer error is produced later.
+  auto handleWasConsumed = [&] {
+    return llvm::any_of(opHandles,
+                        [&](Value h) { return consumedHandles.contains(h); });
+  };
+
+  // Helper function to check if the handle is alive.
   auto hasAliveUser = [&]() {
-    for (Value v : opHandles)
+    for (Value v : opHandles) {
       for (Operation *user : v.getUsers())
-        if (!happensBefore(user, transformOp))
+        if (user != transformOp && !happensBefore(user, transformOp))
           return true;
+    }
     return false;
   };
-  if (!hasAliveUser()) {
-    // The op is tracked but the corresponding handles are dead.
+
+  if (!hasAliveUser() || handleWasConsumed()) {
+    // The op is tracked but the corresponding handles are dead or were
+    // consumed. Drop the op form the mapping.
     (void)replacePayloadOp(op, nullptr);
     return;
   }
@@ -1326,6 +1385,28 @@ void transform::ErrorCheckingTrackingListener::notifyPayloadReplacementNotFound(
   ++errorCounter;
 }
 
+//===----------------------------------------------------------------------===//
+// TransformRewriter
+//===----------------------------------------------------------------------===//
+
+transform::TransformRewriter::TransformRewriter(
+    MLIRContext *ctx, ErrorCheckingTrackingListener *listener)
+    : RewriterBase(ctx), listener(listener) {
+  setListener(listener);
+}
+
+bool transform::TransformRewriter::hasTrackingFailures() const {
+  return listener->failed();
+}
+
+/// Silence all tracking failures that have been encountered so far.
+void transform::TransformRewriter::silenceTrackingFailure() {
+  if (hasTrackingFailures()) {
+    DiagnosedSilenceableFailure status = listener->checkAndResetError();
+    (void)status.silence();
+  }
+}
+
 //===----------------------------------------------------------------------===//
 // Utilities for TransformEachOpTrait.
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
index 3049930b0ced1..72cf6630c3f80 100644
--- a/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
+++ b/mlir/lib/Dialect/Transform/IR/TransformOps.cpp
@@ -1,4 +1,4 @@
-//===- TransformDialect.cpp - Transform dialect operations ----------------===//
+//===- TransformOps.cpp - Transform dialect operations --------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "mlir/Dialect/Transform/IR/TransformOps.h"
+
 #include "mlir/Dialect/Transform/IR/MatchInterfaces.h"
 #include "mlir/Dialect/Transform/IR/TransformAttrs.h"
 #include "mlir/Dialect/Transform/IR/TransformDialect.h"
@@ -115,7 +116,8 @@ static void forwardEmptyOperands(Block *block, transform::TransformState &state,
 }
 
 DiagnosedSilenceableFailure
-transform::AlternativesOp::apply(transform::TransformResults &results,
+transform::AlternativesOp::apply(transform::TransformRewriter &rewriter,
+                                 transform::TransformResults &results,
                                  transform::TransformState &state) {
   SmallVector<Operation *> originals;
   if (Value scopeHandle = getScope())
@@ -222,7 +224,8 @@ LogicalResult transform::AlternativesOp::verify() {
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure
-transform::AnnotateOp::apply(transform::TransformResults &results,
+transform::AnnotateOp::apply(transform::TransformRewriter &rewriter,
+                             transform::TransformResults &results,
                              transform::TransformState &state) {
   SmallVector<Operation *> targets =
       llvm::to_vector(state.getPayloadOps(getTarget()));
@@ -258,10 +261,9 @@ void transform::AnnotateOp::getEffects(
 // ApplyPatternsOp
 //===----------------------------------------------------------------------===//
 
-DiagnosedSilenceableFailure
-transform::ApplyPatternsOp::applyToOne(Operation *target,
-                                       ApplyToEachResultList &results,
-                                       transform::TransformState &state) {
+DiagnosedSilenceableFailure transform::ApplyPatternsOp::applyToOne(
+    transform::TransformRewriter &rewriter, Operation *target,
+    ApplyToEachResultList &results, transform::TransformState &state) {
   // Make sure that this transform is not applied to itself. Modifying the
   // transform IR while it is being interpreted is generally dangerous. Even
   // more so for the ApplyPatternsOp because the GreedyPatternRewriteDriver
@@ -282,9 +284,9 @@ transform::ApplyPatternsOp::applyToOne(Operation *target,
   }
 
   // Configure the GreedyPatternRewriteDriver.
-  ErrorCheckingTrackingListener listener(state, *this);
   GreedyRewriteConfig config;
-  config.listener = &listener;
+  config.listener =
+      static_cast<RewriterBase::Listener *>(rewriter.getListener());
 
   LogicalResult result = failure();
   if (target->hasTrait<OpTrait::IsIsolatedFromAbove>()) {
@@ -312,14 +314,6 @@ transform::ApplyPatternsOp::applyToOne(Operation *target,
            << "greedy pattern application failed";
   }
 
-  // Check listener state for tracking errors.
-  if (listener.failed()) {
-    DiagnosedSilenceableFailure status = listener.checkAndResetError();
-    if (getFailOnPayloadReplacementNotFound())
-      return status;
-    (void)status.silence();
-  }
-
   return DiagnosedSilenceableFailure::success();
 }
 
@@ -346,12 +340,8 @@ void transform::ApplyPatternsOp::getEffects(
 
 void transform::ApplyPatternsOp::build(
     OpBuilder &builder, OperationState &result, Value target,
-    function_ref<void(OpBuilder &, Location)> bodyBuilder,
-    bool failOnPayloadReplacementNotFound) {
+    function_ref<void(OpBuilder &, Location)> bodyBuilder) {
   result.addOperands(target);
-  result.getOrAddProperties<Properties>()
-      .fail_on_payload_replacement_not_found =
-      builder.getBoolAttr(failOnPayloadReplacementNotFound);
 
   OpBuilder::InsertionGuard g(builder);
   Region *region = result.addRegion();
@@ -377,10 +367,9 @@ void transform::ApplyCanonicalizationPatternsOp::populatePatterns(
 // ApplyRegisteredPassOp
 //===----------------------------------------------------------------------===//
 
-DiagnosedSilenceableFailure
-transform::ApplyRegisteredPassOp::applyToOne(Operation *target,
-                                             ApplyToEachResultList &results,
-                                             transform::TransformState &state) {
+DiagnosedSilenceableFailure transform::ApplyRegisteredPassOp::applyToOne(
+    transform::TransformRewriter &rewriter, Operation *target,
+    ApplyToEachResultList &results, transform::TransformState &state) {
   // Make sure that this transform is not applied to itself. Modifying the
   // transform IR while it is being interpreted is generally dangerous. Even
   // more so when applying passes because they may perform a wide range of IR
@@ -420,7 +409,8 @@ transform::ApplyRegisteredPassOp::applyToOne(Operation *target,
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure
-transform::CastOp::applyToOne(Operation *target, ApplyToEachResultList &results,
+transform::CastOp::applyToOne(transform::TransformRewriter &rewriter,
+                              Operation *target, ApplyToEachResultList &results,
                               transform::TransformState &state) {
   results.push_back(target);
   return DiagnosedSilenceableFailure::success();
@@ -482,7 +472,8 @@ matchBlock(Block &block, Operation *op, transform::TransformState &state,
 }
 
 DiagnosedSilenceableFailure
-transform::ForeachMatchOp::apply(transform::TransformResults &results,
+transform::ForeachMatchOp::apply(transform::TransformRewriter &rewriter,
+                                 transform::TransformResults &results,
                                  transform::TransformState &state) {
   SmallVector<std::pair<FunctionOpInterface, FunctionOpInterface>>
       matchActionPairs;
@@ -780,7 +771,8 @@ LogicalResult transform::ForeachMatchOp::verifySymbolUses(
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure
-transform::ForeachOp::apply(transform::TransformResults &results,
+transform::ForeachOp::apply(transform::TransformRewriter &rewriter,
+                            transform::TransformResults &results,
                             transform::TransformState &state) {
   SmallVector<SmallVector<Operation *>> resultOps(getNumResults(), {});
 
@@ -869,6 +861,7 @@ LogicalResult transform::ForeachOp::verify() {
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure transform::GetClosestIsolatedParentOp::apply(
+    transform::TransformRewriter &rewriter,
     transform::TransformResults &results, transform::TransformState &state) {
   SetVector<Operation *> parents;
   for (Operation *target : state.getPayloadOps(getTarget())) {
@@ -892,7 +885,8 @@ DiagnosedSilenceableFailure transform::GetClosestIsolatedParentOp::apply(
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure
-transform::GetConsumersOfResult::apply(transform::TransformResults &results,
+transform::GetConsumersOfResult::apply(transform::TransformRewriter &rewriter,
+                                       transform::TransformResults &results,
                                        transform::TransformState &state) {
   int64_t resultNumber = getResultNumber();
   auto payloadOps = state.getPayloadOps(getTarget());
@@ -917,7 +911,8 @@ transform::GetConsumersOfResult::apply(transform::TransformResults &results,
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure
-transform::GetDefiningOp::apply(transform::TransformResults &results,
+transform::GetDefiningOp::apply(transform::TransformRewriter &rewriter,
+                                transform::TransformResults &results,
                                 transform::TransformState &state) {
   SmallVector<Operation *> definingOps;
   for (Value v : state.getPayloadValues(getTarget())) {
@@ -938,7 +933,8 @@ transform::GetDefiningOp::apply(transform::TransformResults &results,
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure
-transform::GetProducerOfOperand::apply(transform::TransformResults &results,
+transform::GetProducerOfOperand::apply(transform::TransformRewriter &rewriter,
+                                       transform::TransformResults &results,
                                        transform::TransformState &state) {
   int64_t operandNumber = getOperandNumber();
   SmallVector<Operation *> producers;
@@ -966,7 +962,8 @@ transform::GetProducerOfOperand::apply(transform::TransformResults &results,
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure
-transform::GetResultOp::apply(transform::TransformResults &results,
+transform::GetResultOp::apply(transform::TransformRewriter &rewriter,
+                              transform::TransformResults &results,
                               transform::TransformState &state) {
   int64_t resultNumber = getResultNumber();
   SmallVector<Value> opResults;
@@ -1017,7 +1014,8 @@ applySequenceBlock(Block &block, transform::FailurePropagationMode mode,
 }
 
 DiagnosedSilenceableFailure
-transform::IncludeOp::apply(transform::TransformResults &results,
+transform::IncludeOp::apply(transform::TransformRewriter &rewriter,
+                            transform::TransformResults &results,
                             transform::TransformState &state) {
   auto callee = SymbolTable::lookupNearestSymbolFrom<NamedSequenceOp>(
       getOperation(), getTarget());
@@ -1155,7 +1153,8 @@ DiagnosedSilenceableFailure transform::MatchOperationNameOp::matchOperation(
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure
-transform::MatchParamCmpIOp::apply(transform::TransformResults &results,
+transform::MatchParamCmpIOp::apply(transform::TransformRewriter &rewriter,
+                                   transform::TransformResults &results,
                                    transform::TransformState &state) {
   auto signedAPIntAsString = [&](APInt value) {
     std::string str;
@@ -1241,7 +1240,8 @@ void transform::MatchParamCmpIOp::getEffects(
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure
-transform::ParamConstantOp::apply(transform::TransformResults &results,
+transform::ParamConstantOp::apply(transform::TransformRewriter &rewriter,
+                                  transform::TransformResults &results,
                                   transform::TransformState &state) {
   results.setParams(cast<OpResult>(getParam()), {getValue()});
   return DiagnosedSilenceableFailure::success();
@@ -1252,7 +1252,8 @@ transform::ParamConstantOp::apply(transform::TransformResults &results,
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure
-transform::MergeHandlesOp::apply(transform::TransformResults &results,
+transform::MergeHandlesOp::apply(transform::TransformRewriter &rewriter,
+                                 transform::TransformResults &results,
                                  transform::TransformState &state) {
   SmallVector<Operation *> operations;
   for (Value operand : getHandles())
@@ -1295,7 +1296,8 @@ OpFoldResult transform::MergeHandlesOp::fold(FoldAdaptor adaptor) {
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure
-transform::NamedSequenceOp::apply(transform::TransformResults &results,
+transform::NamedSequenceOp::apply(transform::TransformRewriter &rewriter,
+                                  transform::TransformResults &results,
                                   transform::TransformState &state) {
   // Nothing to do here.
   return DiagnosedSilenceableFailure::success();
@@ -1461,7 +1463,8 @@ void transform::SplitHandleOp::build(OpBuilder &builder, OperationState &result,
 }
 
 DiagnosedSilenceableFailure
-transform::SplitHandleOp::apply(transform::TransformResults &results,
+transform::SplitHandleOp::apply(transform::TransformRewriter &rewriter,
+                                transform::TransformResults &results,
                                 transform::TransformState &state) {
   int64_t numPayloadOps = llvm::range_size(state.getPayloadOps(getHandle()));
   auto produceNumOpsError = [&]() {
@@ -1522,7 +1525,8 @@ LogicalResult transform::SplitHandleOp::verify() {
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure
-transform::ReplicateOp::apply(transform::TransformResults &results,
+transform::ReplicateOp::apply(transform::TransformRewriter &rewriter,
+                              transform::TransformResults &results,
                               transform::TransformState &state) {
   unsigned numRepetitions = llvm::range_size(state.getPayloadOps(getPattern()));
   for (const auto &en : llvm::enumerate(getHandles())) {
@@ -1562,7 +1566,8 @@ void transform::ReplicateOp::getEffects(
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure
-transform::SequenceOp::apply(transform::TransformResults &results,
+transform::SequenceOp::apply(transform::TransformRewriter &rewriter,
+                             transform::TransformResults &results,
                              transform::TransformState &state) {
   // Map the entry block argument to the list of operations.
   auto scope = state.make_region_scope(*getBodyBlock()->getParent());
@@ -1852,7 +1857,8 @@ void transform::PrintOp::build(OpBuilder &builder, OperationState &result,
 }
 
 DiagnosedSilenceableFailure
-transform::PrintOp::apply(transform::TransformResults &results,
+transform::PrintOp::apply(transform::TransformRewriter &rewriter,
+                          transform::TransformResults &results,
                           transform::TransformState &state) {
   llvm::outs() << "[[[ IR printer: ";
   if (getName().has_value())
diff --git a/mlir/lib/Dialect/Transform/PDLExtension/PDLExtensionOps.cpp b/mlir/lib/Dialect/Transform/PDLExtension/PDLExtensionOps.cpp
index 5126d79adb189..5585afc1caaff 100644
--- a/mlir/lib/Dialect/Transform/PDLExtension/PDLExtensionOps.cpp
+++ b/mlir/lib/Dialect/Transform/PDLExtension/PDLExtensionOps.cpp
@@ -138,7 +138,8 @@ transform::PDLMatchHooks::getPDLConstraintHooks() const {
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure
-transform::PDLMatchOp::apply(transform::TransformResults &results,
+transform::PDLMatchOp::apply(transform::TransformRewriter &rewriter,
+                             transform::TransformResults &results,
                              transform::TransformState &state) {
   auto *extension = state.getExtension<PatternApplicatorExtension>();
   assert(extension &&
@@ -167,7 +168,8 @@ void transform::PDLMatchOp::getEffects(
 //===----------------------------------------------------------------------===//
 
 DiagnosedSilenceableFailure
-transform::WithPDLPatternsOp::apply(transform::TransformResults &results,
+transform::WithPDLPatternsOp::apply(transform::TransformRewriter &rewriter,
+                                    transform::TransformResults &results,
                                     transform::TransformState &state) {
   TransformOpInterface transformOp = nullptr;
   for (Operation &nested : getBody().front()) {
diff --git a/mlir/test/Dialect/Transform/test-pattern-application.mlir b/mlir/test/Dialect/Transform/test-pattern-application.mlir
index 5d417f0ce6572..062ec6e12e35e 100644
--- a/mlir/test/Dialect/Transform/test-pattern-application.mlir
+++ b/mlir/test/Dialect/Transform/test-pattern-application.mlir
@@ -88,7 +88,7 @@ transform.sequence failures(propagate) {
   %1 = transform.structured.match ops{["test.foo"]} in %arg1 : (!transform.any_op) -> !transform.any_op
   transform.apply_patterns to %0 {
     transform.apply_patterns.transform.test_patterns
-  } {fail_on_payload_replacement_not_found = false} : !transform.any_op
+  } {transform.silence_tracking_failures} : !transform.any_op
   transform.annotate %1 "annotated" : !transform.any_op
 }
 
diff --git a/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp b/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp
index 835cbb3ae2d5e..8f53b9c927f0f 100644
--- a/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp
+++ b/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp
@@ -45,7 +45,8 @@ class TestTransformOp
     return llvm::StringLiteral("transform.test_transform_op");
   }
 
-  DiagnosedSilenceableFailure apply(transform::TransformResults &results,
+  DiagnosedSilenceableFailure apply(transform::TransformRewriter &rewriter,
+                                    transform::TransformResults &results,
                                     transform::TransformState &state) {
     InFlightDiagnostic remark = emitRemark() << "applying transformation";
     if (Attribute message = getMessage())
@@ -98,7 +99,8 @@ class TestTransformUnrestrictedOpNoInterface
         "transform.test_transform_unrestricted_op_no_interface");
   }
 
-  DiagnosedSilenceableFailure apply(transform::TransformResults &results,
+  DiagnosedSilenceableFailure apply(transform::TransformRewriter &rewriter,
+                                    transform::TransformResults &results,
                                     transform::TransformState &state) {
     return DiagnosedSilenceableFailure::success();
   }
@@ -110,6 +112,7 @@ class TestTransformUnrestrictedOpNoInterface
 
 DiagnosedSilenceableFailure
 mlir::test::TestProduceSelfHandleOrForwardOperandOp::apply(
+    transform::TransformRewriter &rewriter,
     transform::TransformResults &results, transform::TransformState &state) {
   if (getOperation()->getNumOperands() != 0) {
     results.set(cast<OpResult>(getResult()),
@@ -129,6 +132,7 @@ void mlir::test::TestProduceSelfHandleOrForwardOperandOp::getEffects(
 
 DiagnosedSilenceableFailure
 mlir::test::TestProduceValueHandleToSelfOperand::apply(
+    transform::TransformRewriter &rewriter,
     transform::TransformResults &results, transform::TransformState &state) {
   results.setValues(llvm::cast<OpResult>(getOut()), getIn());
   return DiagnosedSilenceableFailure::success();
@@ -143,7 +147,8 @@ void mlir::test::TestProduceValueHandleToSelfOperand::getEffects(
 
 DiagnosedSilenceableFailure
 mlir::test::TestProduceValueHandleToResult::applyToOne(
-    Operation *target, transform::ApplyToEachResultList &results,
+    transform::TransformRewriter &rewriter, Operation *target,
+    transform::ApplyToEachResultList &results,
     transform::TransformState &state) {
   if (target->getNumResults() <= getNumber())
     return emitSilenceableError() << "payload has no result #" << getNumber();
@@ -160,7 +165,8 @@ void mlir::test::TestProduceValueHandleToResult::getEffects(
 
 DiagnosedSilenceableFailure
 mlir::test::TestProduceValueHandleToArgumentOfParentBlock::applyToOne(
-    Operation *target, transform::ApplyToEachResultList &results,
+    transform::TransformRewriter &rewriter, Operation *target,
+    transform::ApplyToEachResultList &results,
     transform::TransformState &state) {
   if (!target->getBlock())
     return emitSilenceableError() << "payload has no parent block";
@@ -183,7 +189,8 @@ bool mlir::test::TestConsumeOperand::allowsRepeatedHandleOperands() {
 }
 
 DiagnosedSilenceableFailure
-mlir::test::TestConsumeOperand::apply(transform::TransformResults &results,
+mlir::test::TestConsumeOperand::apply(transform::TransformRewriter &rewriter,
+                                      transform::TransformResults &results,
                                       transform::TransformState &state) {
   return DiagnosedSilenceableFailure::success();
 }
@@ -197,6 +204,7 @@ void mlir::test::TestConsumeOperand::getEffects(
 }
 
 DiagnosedSilenceableFailure mlir::test::TestConsumeOperandOfOpKindOrFail::apply(
+    transform::TransformRewriter &rewriter,
     transform::TransformResults &results, transform::TransformState &state) {
   auto payload = state.getPayloadOps(getOperand());
   assert(llvm::hasSingleElement(payload) && "expected a single target op");
@@ -237,6 +245,7 @@ void mlir::test::TestSucceedIfOperandOfOpKind::getEffects(
 }
 
 DiagnosedSilenceableFailure mlir::test::TestPrintRemarkAtOperandOp::apply(
+    transform::TransformRewriter &rewriter,
     transform::TransformResults &results, transform::TransformState &state) {
   auto payload = state.getPayloadOps(getOperand());
   for (Operation *op : payload)
@@ -252,6 +261,7 @@ void mlir::test::TestPrintRemarkAtOperandOp::getEffects(
 }
 
 DiagnosedSilenceableFailure mlir::test::TestPrintRemarkAtOperandValue::apply(
+    transform::TransformRewriter &rewriter,
     transform::TransformResults &results, transform::TransformState &state) {
   ArrayRef<Value> values = state.getPayloadValues(getIn());
   for (Value value : values) {
@@ -277,15 +287,16 @@ void mlir::test::TestPrintRemarkAtOperandValue::getEffects(
   transform::onlyReadsPayload(effects);
 }
 
-DiagnosedSilenceableFailure
-mlir::test::TestAddTestExtensionOp::apply(transform::TransformResults &results,
-                                          transform::TransformState &state) {
+DiagnosedSilenceableFailure mlir::test::TestAddTestExtensionOp::apply(
+    transform::TransformRewriter &rewriter,
+    transform::TransformResults &results, transform::TransformState &state) {
   state.addExtension<TestTransformStateExtension>(getMessageAttr());
   return DiagnosedSilenceableFailure::success();
 }
 
 DiagnosedSilenceableFailure
 mlir::test::TestCheckIfTestExtensionPresentOp::apply(
+    transform::TransformRewriter &rewriter,
     transform::TransformResults &results, transform::TransformState &state) {
   auto *extension = state.getExtension<TestTransformStateExtension>();
   if (!extension) {
@@ -316,6 +327,7 @@ void mlir::test::TestCheckIfTestExtensionPresentOp::getEffects(
 }
 
 DiagnosedSilenceableFailure mlir::test::TestRemapOperandPayloadToSelfOp::apply(
+    transform::TransformRewriter &rewriter,
     transform::TransformResults &results, transform::TransformState &state) {
   auto *extension = state.getExtension<TestTransformStateExtension>();
   if (!extension)
@@ -337,14 +349,15 @@ void mlir::test::TestRemapOperandPayloadToSelfOp::getEffects(
 }
 
 DiagnosedSilenceableFailure mlir::test::TestRemoveTestExtensionOp::apply(
+    transform::TransformRewriter &rewriter,
     transform::TransformResults &results, transform::TransformState &state) {
   state.removeExtension<TestTransformStateExtension>();
   return DiagnosedSilenceableFailure::success();
 }
 
-DiagnosedSilenceableFailure
-mlir::test::TestReversePayloadOpsOp::apply(transform::TransformResults &results,
-                                           transform::TransformState &state) {
+DiagnosedSilenceableFailure mlir::test::TestReversePayloadOpsOp::apply(
+    transform::TransformRewriter &rewriter,
+    transform::TransformResults &results, transform::TransformState &state) {
   auto payloadOps = state.getPayloadOps(getTarget());
   auto reversedOps = llvm::to_vector(llvm::reverse(payloadOps));
   results.set(llvm::cast<OpResult>(getResult()), reversedOps);
@@ -352,6 +365,7 @@ mlir::test::TestReversePayloadOpsOp::apply(transform::TransformResults &results,
 }
 
 DiagnosedSilenceableFailure mlir::test::TestTransformOpWithRegions::apply(
+    transform::TransformRewriter &rewriter,
     transform::TransformResults &results, transform::TransformState &state) {
   return DiagnosedSilenceableFailure::success();
 }
@@ -361,6 +375,7 @@ void mlir::test::TestTransformOpWithRegions::getEffects(
 
 DiagnosedSilenceableFailure
 mlir::test::TestBranchingTransformOpTerminator::apply(
+    transform::TransformRewriter &rewriter,
     transform::TransformResults &results, transform::TransformState &state) {
   return DiagnosedSilenceableFailure::success();
 }
@@ -369,6 +384,7 @@ void mlir::test::TestBranchingTransformOpTerminator::getEffects(
     SmallVectorImpl<MemoryEffects::EffectInstance> &effects) {}
 
 DiagnosedSilenceableFailure mlir::test::TestEmitRemarkAndEraseOperandOp::apply(
+    transform::TransformRewriter &rewriter,
     transform::TransformResults &results, transform::TransformState &state) {
   emitRemark() << getRemark();
   for (Operation *op : state.getPayloadOps(getTarget()))
@@ -386,7 +402,8 @@ void mlir::test::TestEmitRemarkAndEraseOperandOp::getEffects(
 }
 
 DiagnosedSilenceableFailure mlir::test::TestWrongNumberOfResultsOp::applyToOne(
-    Operation *target, transform::ApplyToEachResultList &results,
+    transform::TransformRewriter &rewriter, Operation *target,
+    transform::ApplyToEachResultList &results,
     transform::TransformState &state) {
   OperationState opState(target->getLoc(), "foo");
   results.push_back(OpBuilder(target).create(opState));
@@ -395,7 +412,8 @@ DiagnosedSilenceableFailure mlir::test::TestWrongNumberOfResultsOp::applyToOne(
 
 DiagnosedSilenceableFailure
 mlir::test::TestWrongNumberOfMultiResultsOp::applyToOne(
-    Operation *target, transform::ApplyToEachResultList &results,
+    transform::TransformRewriter &rewriter, Operation *target,
+    transform::ApplyToEachResultList &results,
     transform::TransformState &state) {
   static int count = 0;
   if (count++ == 0) {
@@ -407,7 +425,8 @@ mlir::test::TestWrongNumberOfMultiResultsOp::applyToOne(
 
 DiagnosedSilenceableFailure
 mlir::test::TestCorrectNumberOfMultiResultsOp::applyToOne(
-    Operation *target, transform::ApplyToEachResultList &results,
+    transform::TransformRewriter &rewriter, Operation *target,
+    transform::ApplyToEachResultList &results,
     transform::TransformState &state) {
   OperationState opState(target->getLoc(), "foo");
   results.push_back(OpBuilder(target).create(opState));
@@ -417,7 +436,8 @@ mlir::test::TestCorrectNumberOfMultiResultsOp::applyToOne(
 
 DiagnosedSilenceableFailure
 mlir::test::TestMixedNullAndNonNullResultsOp::applyToOne(
-    Operation *target, transform::ApplyToEachResultList &results,
+    transform::TransformRewriter &rewriter, Operation *target,
+    transform::ApplyToEachResultList &results,
     transform::TransformState &state) {
   OperationState opState(target->getLoc(), "foo");
   results.push_back(nullptr);
@@ -427,7 +447,8 @@ mlir::test::TestMixedNullAndNonNullResultsOp::applyToOne(
 
 DiagnosedSilenceableFailure
 mlir::test::TestMixedSuccessAndSilenceableOp::applyToOne(
-    Operation *target, transform::ApplyToEachResultList &results,
+    transform::TransformRewriter &rewriter, Operation *target,
+    transform::ApplyToEachResultList &results,
     transform::TransformState &state) {
   if (target->hasAttr("target_me"))
     return DiagnosedSilenceableFailure::success();
@@ -436,6 +457,7 @@ mlir::test::TestMixedSuccessAndSilenceableOp::applyToOne(
 
 DiagnosedSilenceableFailure
 mlir::test::TestPrintNumberOfAssociatedPayloadIROps::apply(
+    transform::TransformRewriter &rewriter,
     transform::TransformResults &results, transform::TransformState &state) {
   if (!getHandle())
     emitRemark() << 0;
@@ -449,7 +471,8 @@ void mlir::test::TestPrintNumberOfAssociatedPayloadIROps::getEffects(
 }
 
 DiagnosedSilenceableFailure
-mlir::test::TestCopyPayloadOp::apply(transform::TransformResults &results,
+mlir::test::TestCopyPayloadOp::apply(transform::TransformRewriter &rewriter,
+                                     transform::TransformResults &results,
                                      transform::TransformState &state) {
   results.set(llvm::cast<OpResult>(getCopy()),
               state.getPayloadOps(getHandle()));
@@ -498,6 +521,7 @@ void mlir::test::TestReportNumberOfTrackedHandlesNestedUnder::getEffects(
 
 DiagnosedSilenceableFailure
 mlir::test::TestReportNumberOfTrackedHandlesNestedUnder::apply(
+    transform::TransformRewriter &rewriter,
     transform::TransformResults &results, transform::TransformState &state) {
   int64_t count = 0;
   for (Operation *op : state.getPayloadOps(getTarget())) {
@@ -520,7 +544,8 @@ void mlir::test::TestPrintParamOp::getEffects(
 }
 
 DiagnosedSilenceableFailure
-mlir::test::TestPrintParamOp::apply(transform::TransformResults &results,
+mlir::test::TestPrintParamOp::apply(transform::TransformRewriter &rewriter,
+                                    transform::TransformResults &results,
                                     transform::TransformState &state) {
   std::string str;
   llvm::raw_string_ostream os(str);
@@ -537,7 +562,8 @@ mlir::test::TestPrintParamOp::apply(transform::TransformResults &results,
 }
 
 DiagnosedSilenceableFailure
-mlir::test::TestAddToParamOp::apply(transform::TransformResults &results,
+mlir::test::TestAddToParamOp::apply(transform::TransformRewriter &rewriter,
+                                    transform::TransformResults &results,
                                     transform::TransformState &state) {
   SmallVector<uint32_t> values(/*Size=*/1, /*Value=*/0);
   if (Value param = getParam()) {
@@ -559,6 +585,7 @@ mlir::test::TestAddToParamOp::apply(transform::TransformResults &results,
 
 DiagnosedSilenceableFailure
 mlir::test::TestProduceParamWithNumberOfTestOps::apply(
+    transform::TransformRewriter &rewriter,
     transform::TransformResults &results, transform::TransformState &state) {
   Builder builder(getContext());
   SmallVector<Attribute> result = llvm::to_vector(
@@ -577,6 +604,7 @@ mlir::test::TestProduceParamWithNumberOfTestOps::apply(
 
 DiagnosedSilenceableFailure
 mlir::test::TestProduceIntegerParamWithTypeOp::apply(
+    transform::TransformRewriter &rewriter,
     transform::TransformResults &results, transform::TransformState &state) {
   Attribute zero = IntegerAttr::get(getType(), 0);
   results.setParams(llvm::cast<OpResult>(getResult()), zero);
@@ -599,7 +627,8 @@ void mlir::test::TestProduceTransformParamOrForwardOperandOp::getEffects(
 
 DiagnosedSilenceableFailure
 mlir::test::TestProduceTransformParamOrForwardOperandOp::applyToOne(
-    Operation *target, ::transform::ApplyToEachResultList &results,
+    transform::TransformRewriter &rewriter, Operation *target,
+    ::transform::ApplyToEachResultList &results,
     ::transform::TransformState &state) {
   Builder builder(getContext());
   if (getFirstResultIsParam()) {
@@ -625,6 +654,7 @@ void mlir::test::TestProduceNullPayloadOp::getEffects(
 }
 
 DiagnosedSilenceableFailure mlir::test::TestProduceNullPayloadOp::apply(
+    transform::TransformRewriter &rewriter,
     transform::TransformResults &results, transform::TransformState &state) {
   SmallVector<Operation *, 1> null({nullptr});
   results.set(llvm::cast<OpResult>(getOut()), null);
@@ -632,6 +662,7 @@ DiagnosedSilenceableFailure mlir::test::TestProduceNullPayloadOp::apply(
 }
 
 DiagnosedSilenceableFailure mlir::test::TestProduceEmptyPayloadOp::apply(
+    transform::TransformRewriter &rewriter,
     transform::TransformResults &results, transform::TransformState &state) {
   results.set(cast<OpResult>(getOut()), {});
   return DiagnosedSilenceableFailure::success();
@@ -642,9 +673,9 @@ void mlir::test::TestProduceNullParamOp::getEffects(
   transform::producesHandle(getOut(), effects);
 }
 
-DiagnosedSilenceableFailure
-mlir::test::TestProduceNullParamOp::apply(transform::TransformResults &results,
-                                          transform::TransformState &state) {
+DiagnosedSilenceableFailure mlir::test::TestProduceNullParamOp::apply(
+    transform::TransformRewriter &rewriter,
+    transform::TransformResults &results, transform::TransformState &state) {
   results.setParams(llvm::cast<OpResult>(getOut()), Attribute());
   return DiagnosedSilenceableFailure::success();
 }
@@ -654,9 +685,9 @@ void mlir::test::TestProduceNullValueOp::getEffects(
   transform::producesHandle(getOut(), effects);
 }
 
-DiagnosedSilenceableFailure
-mlir::test::TestProduceNullValueOp::apply(transform::TransformResults &results,
-                                          transform::TransformState &state) {
+DiagnosedSilenceableFailure mlir::test::TestProduceNullValueOp::apply(
+    transform::TransformRewriter &rewriter,
+    transform::TransformResults &results, transform::TransformState &state) {
   results.setValues(llvm::cast<OpResult>(getOut()), Value());
   return DiagnosedSilenceableFailure::success();
 }
@@ -676,6 +707,7 @@ void mlir::test::TestRequiredMemoryEffectsOp::getEffects(
 }
 
 DiagnosedSilenceableFailure mlir::test::TestRequiredMemoryEffectsOp::apply(
+    transform::TransformRewriter &rewriter,
     transform::TransformResults &results, transform::TransformState &state) {
   results.set(llvm::cast<OpResult>(getOut()), state.getPayloadOps(getIn()));
   return DiagnosedSilenceableFailure::success();
@@ -694,10 +726,9 @@ void mlir::test::TestDummyPayloadOp::getEffects(
 }
 
 DiagnosedSilenceableFailure
-mlir::test::TestTrackedRewriteOp::apply(transform::TransformResults &results,
+mlir::test::TestTrackedRewriteOp::apply(transform::TransformRewriter &rewriter,
+                                        transform::TransformResults &results,
                                         transform::TransformState &state) {
-  transform::ErrorCheckingTrackingListener listener(state, *this);
-  IRRewriter rewriter(getContext(), &listener);
   int64_t numIterations = 0;
 
   // `getPayloadOps` returns an iterator that skips ops that are erased in the
diff --git a/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.td b/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.td
index 76111653dd922..fda619053867f 100644
--- a/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.td
+++ b/mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.td
@@ -72,6 +72,7 @@ def TestProduceValueHandleToResult
   let cppNamespace = "::mlir::test";
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::Operation *target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -89,6 +90,7 @@ def TestProduceValueHandleToArgumentOfParentBlock
   let cppNamespace = "::mlir::test";
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::Operation *target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -116,6 +118,7 @@ def TestConsumeOperandEach : Op<Transform_Dialect, "test_consume_operand_each",
   let cppNamespace = "::mlir::test";
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::Operation *target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state) {
@@ -260,6 +263,7 @@ def TestWrongNumberOfResultsOp
   let cppNamespace = "::mlir::test";
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::Operation * target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -277,6 +281,7 @@ def TestWrongNumberOfMultiResultsOp
   let cppNamespace = "::mlir::test";
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::Operation * target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -295,6 +300,7 @@ def TestCorrectNumberOfMultiResultsOp
   let cppNamespace = "::mlir::test";
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::Operation * target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -313,6 +319,7 @@ def TestMixedNullAndNonNullResultsOp
   let cppNamespace = "::mlir::test";
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::Operation * target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -329,6 +336,7 @@ def TestMixedSuccessAndSilenceableOp
   let cppNamespace = "::mlir::test";
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::Operation * target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -428,6 +436,7 @@ def TestProduceTransformParamOrForwardOperandOp
 
   let extraClassDeclaration = [{
     ::mlir::DiagnosedSilenceableFailure applyToOne(
+        ::mlir::transform::TransformRewriter &rewriter,
         ::mlir::Operation *target,
         ::mlir::transform::ApplyToEachResultList &results,
         ::mlir::transform::TransformState &state);
@@ -498,7 +507,8 @@ def TestDummyPayloadOp
   let cppNamespace = "::mlir::test";
 
   let extraClassDeclaration = [{
-    DiagnosedSilenceableFailure apply(transform::TransformResults &results,
+    DiagnosedSilenceableFailure apply(transform::TransformRewriter &rewriter,
+                                      transform::TransformResults &results,
                                       transform::TransformState &state) {
       llvm_unreachable("op should not be used as a transform");
       return DiagnosedSilenceableFailure::definiteFailure();

From fc06262c1c365777e71207b6a5de281cba927c96 Mon Sep 17 00:00:00 2001
From: Francesco Petrogalli <francesco.petrogalli@apple.com>
Date: Tue, 20 Jun 2023 11:13:39 +0200
Subject: [PATCH 104/130] [llc][MISched] Add `-misched-detail-resource-booking`
 to llc.

The option `-misched-detail-resource-booking` prints the following
information every time the method
`SchedBoundary::getNextResourceCycle` is invoked:

1. counters of the resources that have already been booked;

2. the values returned by `getNextResourceCycle`, which is the next
available cycle in which a resource can be booked.

The method is useful to debug low-level checks inside the machine
scheduler that make decisions based on the values returned by
`getNextResourceCycle`.

Reviewed By: andreadb

Differential Revision: https://reviews.llvm.org/D153116
---
 llvm/lib/CodeGen/MachineScheduler.cpp         |   18 +-
 .../misched-detail-resource-booking-01.mir    | 1618 +++++++++++++++++
 .../misched-detail-resource-booking-02.mir    |  479 +++++
 3 files changed, 2114 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/AArch64/misched-detail-resource-booking-01.mir
 create mode 100644 llvm/test/CodeGen/AArch64/misched-detail-resource-booking-02.mir

diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index 28b17f76c99e2..3adcf706f8388 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -98,11 +98,15 @@ cl::opt<bool> PrintDAGs("misched-print-dags", cl::Hidden,
 cl::opt<bool> MISchedDumpReservedCycles(
     "misched-dump-reserved-cycles", cl::Hidden, cl::init(false),
     cl::desc("Dump resource usage at schedule boundary."));
+cl::opt<bool> MischedDetailResourceBooking(
+    "misched-detail-resource-booking", cl::Hidden, cl::init(false),
+    cl::desc("Show details of invoking getNextResoufceCycle."));
 #else
 const bool ViewMISchedDAGs = false;
 const bool PrintDAGs = false;
 #ifdef LLVM_ENABLE_DUMP
 const bool MISchedDumpReservedCycles = false;
+const bool MischedDetailResourceBooking = false;
 #endif // LLVM_ENABLE_DUMP
 #endif // NDEBUG
 
@@ -2314,7 +2318,11 @@ unsigned SchedBoundary::getNextResourceCycleByInstance(unsigned InstanceIdx,
 std::pair<unsigned, unsigned>
 SchedBoundary::getNextResourceCycle(const MCSchedClassDesc *SC, unsigned PIdx,
                                     unsigned Cycles, unsigned StartAtCycle) {
-
+  if (MischedDetailResourceBooking) {
+    LLVM_DEBUG(dbgs() << "  Resource booking (@" << CurrCycle << "c): \n");
+    LLVM_DEBUG(dumpReservedCycles());
+    LLVM_DEBUG(dbgs() << "  getNextResourceCycle (@" << CurrCycle << "c): \n");
+  }
   unsigned MinNextUnreserved = InvalidCycle;
   unsigned InstanceIdx = 0;
   unsigned StartIndex = ReservedCyclesIndex[PIdx];
@@ -2355,11 +2363,19 @@ SchedBoundary::getNextResourceCycle(const MCSchedClassDesc *SC, unsigned PIdx,
        ++I) {
     unsigned NextUnreserved =
         getNextResourceCycleByInstance(I, Cycles, StartAtCycle);
+    if (MischedDetailResourceBooking)
+      LLVM_DEBUG(dbgs() << "    Instance " << I - StartIndex << " available @"
+                        << NextUnreserved << "c\n");
     if (MinNextUnreserved > NextUnreserved) {
       InstanceIdx = I;
       MinNextUnreserved = NextUnreserved;
     }
   }
+  if (MischedDetailResourceBooking)
+    LLVM_DEBUG(dbgs() << "    selecting " << SchedModel->getResourceName(PIdx)
+                      << "[" << InstanceIdx - StartIndex << "]"
+                      << " available @" << MinNextUnreserved << "c"
+                      << "\n");
   return std::make_pair(MinNextUnreserved, InstanceIdx);
 }
 
diff --git a/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-01.mir b/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-01.mir
new file mode 100644
index 0000000000000..a012fc5608a3f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-01.mir
@@ -0,0 +1,1618 @@
+# RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+neon  %s -o - 2>&1 \
+# RUN:   -misched-dump-reserved-cycles=true \
+# RUN:   -run-pass=machine-scheduler -debug-only=machine-scheduler \
+# RUN:   -misched-bottomup=true -sched-print-cycles=true \
+# RUN:   -misched-detail-resource-booking=true \
+# RUN:   -misched-dump-schedule-trace=true -misched-dump-schedule-trace-col-header-width=21 \
+# RUN: | FileCheck  %s
+
+--- |
+  ; ModuleID = '../llvm-project/llvm/test/CodeGen/AArch64/aarch64-smull.failing.ll'
+  source_filename = "../llvm-project/llvm/test/CodeGen/AArch64/aarch64-smull.failing.ll"
+  target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+  target triple = "aarch64-none-linux-gnu"
+  
+  define <8 x i32> @umull_and_v8i32(<8 x i16> %src1, <8 x i32> %src2) #0 {
+  entry:
+    %in1 = zext <8 x i16> %src1 to <8 x i32>
+    %in2 = and <8 x i32> %src2, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
+    %out = mul nsw <8 x i32> %in1, %in2
+    ret <8 x i32> %out
+  }
+  
+  attributes #0 = { "target-features"="+neon" }
+
+...
+---
+name:            umull_and_v8i32
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr128 }
+  - { id: 1, class: fpr128 }
+  - { id: 2, class: fpr128 }
+  - { id: 3, class: fpr128 }
+  - { id: 4, class: fpr64 }
+  - { id: 5, class: fpr64 }
+  - { id: 6, class: fpr128 }
+  - { id: 7, class: fpr128 }
+  - { id: 8, class: fpr128 }
+  - { id: 9, class: fpr64 }
+  - { id: 10, class: fpr128 }
+  - { id: 11, class: fpr64 }
+  - { id: 12, class: fpr128 }
+liveins:
+  - { reg: '$q0', virtual-reg: '%0' }
+  - { reg: '$q1', virtual-reg: '%1' }
+  - { reg: '$q2', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+  maxCallFrameSize: 0
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    liveins: $q0, $q1, $q2
+  
+    %2:fpr128 = COPY $q2
+    %1:fpr128 = COPY $q1
+    %0:fpr128 = COPY $q0
+    %3:fpr128 = EXTv16i8 %0, %0, 8
+    %6:fpr128 = MOVIv2d_ns 17
+    %7:fpr128 = ANDv16i8 %2, %6
+    %8:fpr128 = ANDv16i8 %1, %6
+    %9:fpr64 = XTNv4i16 %8
+    %10:fpr128 = UMULLv4i16_v4i32 %0.dsub, %9
+    %11:fpr64 = XTNv4i16 %7
+    %12:fpr128 = UMULLv4i16_v4i32 %3.dsub, %11
+    $q0 = COPY %10
+    $q1 = COPY %12
+    RET_ReallyLR implicit $q0, implicit $q1
+
+...
+
+# CHECK-LABEL: Before MISched:
+# CHECK-NEXT: # Machine code for function umull_and_v8i32: IsSSA, NoPHIs, TracksLiveness
+# CHECK-NEXT: Function Live Ins: $q0 in %0, $q1 in %1, $q2 in %2
+# CHECK-EMPTY: 
+# CHECK-NEXT: bb.0.entry:
+# CHECK-NEXT:   liveins: $q0, $q1, $q2
+# CHECK-NEXT:   %2:fpr128 = COPY $q2
+# CHECK-NEXT:   %1:fpr128 = COPY $q1
+# CHECK-NEXT:   %0:fpr128 = COPY $q0
+# CHECK-NEXT:   %3:fpr128 = EXTv16i8 %0:fpr128, %0:fpr128, 8
+# CHECK-NEXT:   %6:fpr128 = MOVIv2d_ns 17
+# CHECK-NEXT:   %7:fpr128 = ANDv16i8 %2:fpr128, %6:fpr128
+# CHECK-NEXT:   %8:fpr128 = ANDv16i8 %1:fpr128, %6:fpr128
+# CHECK-NEXT:   %9:fpr64 = XTNv4i16 %8:fpr128
+# CHECK-NEXT:   %10:fpr128 = UMULLv4i16_v4i32 %0.dsub:fpr128, %9:fpr64
+# CHECK-NEXT:   %11:fpr64 = XTNv4i16 %7:fpr128
+# CHECK-NEXT:   %12:fpr128 = UMULLv4i16_v4i32 %3.dsub:fpr128, %11:fpr64
+# CHECK-NEXT:   $q0 = COPY %10:fpr128
+# CHECK-NEXT:   $q1 = COPY %12:fpr128
+# CHECK-NEXT:   RET_ReallyLR implicit $q0, implicit $q1
+# CHECK-EMPTY: 
+# CHECK-NEXT: # End machine code for function umull_and_v8i32.
+# CHECK-EMPTY: 
+# CHECK-NEXT: ********** MI Scheduling **********
+# CHECK-NEXT: umull_and_v8i32:%bb.0 entry
+# CHECK-NEXT:   From: %2:fpr128 = COPY $q2
+# CHECK-NEXT:     To: RET_ReallyLR implicit $q0, implicit $q1
+# CHECK-NEXT:  RegionInstrs: 13
+# CHECK-NEXT: ScheduleDAGMILive::schedule starting
+# CHECK-NEXT: GenericScheduler RegionPolicy:  ShouldTrackPressure=0 OnlyTopDown=0 OnlyBottomUp=1
+# CHECK-NEXT: Disabled scoreboard hazard recognizer
+# CHECK-NEXT: Disabled scoreboard hazard recognizer
+# CHECK-NEXT: SU(0) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %2:fpr128 = COPY $q2
+# CHECK-NEXT:   # preds left       : 0
+# CHECK-NEXT:   # succs left       : 1
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 3
+# CHECK-NEXT:   Depth              : 0
+# CHECK-NEXT:   Height             : 13
+# CHECK-NEXT:   Successors:
+# CHECK-NEXT:     SU(5): Data Latency=3 Reg=%2
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(1) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %1:fpr128 = COPY $q1
+# CHECK-NEXT:   # preds left       : 0
+# CHECK-NEXT:   # succs left       : 2
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 3
+# CHECK-NEXT:   Depth              : 0
+# CHECK-NEXT:   Height             : 13
+# CHECK-NEXT:   Successors:
+# CHECK-NEXT:     SU(6): Data Latency=3 Reg=%1
+# CHECK-NEXT:     SU(12): Anti Latency=0
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(2) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %0:fpr128 = COPY $q0
+# CHECK-NEXT:   # preds left       : 0
+# CHECK-NEXT:   # succs left       : 3
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 3
+# CHECK-NEXT:   Depth              : 0
+# CHECK-NEXT:   Height             : 14
+# CHECK-NEXT:   Successors:
+# CHECK-NEXT:     SU(8): Data Latency=3 Reg=%0
+# CHECK-NEXT:     SU(3): Data Latency=3 Reg=%0
+# CHECK-NEXT:     SU(11): Anti Latency=0
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(3) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %3:fpr128 = EXTv16i8 %0:fpr128, %0:fpr128, 8
+# CHECK-NEXT:   # preds left       : 1
+# CHECK-NEXT:   # succs left       : 1
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 4
+# CHECK-NEXT:   Depth              : 3
+# CHECK-NEXT:   Height             : 11
+# CHECK-NEXT:   Predecessors:
+# CHECK-NEXT:     SU(2): Data Latency=3 Reg=%0
+# CHECK-NEXT:   Successors:
+# CHECK-NEXT:     SU(10): Data Latency=4 Reg=%3
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(4) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %6:fpr128 = MOVIv2d_ns 17
+# CHECK-NEXT:   # preds left       : 0
+# CHECK-NEXT:   # succs left       : 2
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 4
+# CHECK-NEXT:   Depth              : 0
+# CHECK-NEXT:   Height             : 14
+# CHECK-NEXT:   Successors:
+# CHECK-NEXT:     SU(6): Data Latency=4 Reg=%6
+# CHECK-NEXT:     SU(5): Data Latency=4 Reg=%6
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(5) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %7:fpr128 = ANDv16i8 %2:fpr128, %6:fpr128
+# CHECK-NEXT:   # preds left       : 2
+# CHECK-NEXT:   # succs left       : 1
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 1
+# CHECK-NEXT:   Depth              : 4
+# CHECK-NEXT:   Height             : 10
+# CHECK-NEXT:   Predecessors:
+# CHECK-NEXT:     SU(4): Data Latency=4 Reg=%6
+# CHECK-NEXT:     SU(0): Data Latency=3 Reg=%2
+# CHECK-NEXT:   Successors:
+# CHECK-NEXT:     SU(9): Data Latency=1 Reg=%7
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(6) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %8:fpr128 = ANDv16i8 %1:fpr128, %6:fpr128
+# CHECK-NEXT:   # preds left       : 2
+# CHECK-NEXT:   # succs left       : 1
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 1
+# CHECK-NEXT:   Depth              : 4
+# CHECK-NEXT:   Height             : 10
+# CHECK-NEXT:   Predecessors:
+# CHECK-NEXT:     SU(4): Data Latency=4 Reg=%6
+# CHECK-NEXT:     SU(1): Data Latency=3 Reg=%1
+# CHECK-NEXT:   Successors:
+# CHECK-NEXT:     SU(7): Data Latency=1 Reg=%8
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(7) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %9:fpr64 = XTNv4i16 %8:fpr128
+# CHECK-NEXT:   # preds left       : 1
+# CHECK-NEXT:   # succs left       : 1
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 2
+# CHECK-NEXT:   Depth              : 5
+# CHECK-NEXT:   Height             : 9
+# CHECK-NEXT:   Predecessors:
+# CHECK-NEXT:     SU(6): Data Latency=1 Reg=%8
+# CHECK-NEXT:   Successors:
+# CHECK-NEXT:     SU(8): Data Latency=2 Reg=%9
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(8) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %10:fpr128 = UMULLv4i16_v4i32 %0.dsub:fpr128, %9:fpr64
+# CHECK-NEXT:   # preds left       : 2
+# CHECK-NEXT:   # succs left       : 1
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 4
+# CHECK-NEXT:   Depth              : 7
+# CHECK-NEXT:   Height             : 7
+# CHECK-NEXT:   Predecessors:
+# CHECK-NEXT:     SU(7): Data Latency=2 Reg=%9
+# CHECK-NEXT:     SU(2): Data Latency=3 Reg=%0
+# CHECK-NEXT:   Successors:
+# CHECK-NEXT:     SU(11): Data Latency=4 Reg=%10
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(9) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %11:fpr64 = XTNv4i16 %7:fpr128
+# CHECK-NEXT:   # preds left       : 1
+# CHECK-NEXT:   # succs left       : 1
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 2
+# CHECK-NEXT:   Depth              : 5
+# CHECK-NEXT:   Height             : 9
+# CHECK-NEXT:   Predecessors:
+# CHECK-NEXT:     SU(5): Data Latency=1 Reg=%7
+# CHECK-NEXT:   Successors:
+# CHECK-NEXT:     SU(10): Data Latency=2 Reg=%11
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(10) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %12:fpr128 = UMULLv4i16_v4i32 %3.dsub:fpr128, %11:fpr64
+# CHECK-NEXT:   # preds left       : 2
+# CHECK-NEXT:   # succs left       : 1
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 4
+# CHECK-NEXT:   Depth              : 7
+# CHECK-NEXT:   Height             : 7
+# CHECK-NEXT:   Predecessors:
+# CHECK-NEXT:     SU(9): Data Latency=2 Reg=%11
+# CHECK-NEXT:     SU(3): Data Latency=4 Reg=%3
+# CHECK-NEXT:   Successors:
+# CHECK-NEXT:     SU(12): Data Latency=4 Reg=%12
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(11) [TopReadyCycle = 0, BottomReadyCycle = 0]:   $q0 = COPY %10:fpr128
+# CHECK-NEXT:   # preds left       : 2
+# CHECK-NEXT:   # succs left       : 1
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 3
+# CHECK-NEXT:   Depth              : 11
+# CHECK-NEXT:   Height             : 3
+# CHECK-NEXT:   Predecessors:
+# CHECK-NEXT:     SU(8): Data Latency=4 Reg=%10
+# CHECK-NEXT:     SU(2): Anti Latency=0
+# CHECK-NEXT:   Successors:
+# CHECK-NEXT:     ExitSU: Ord  Latency=3 Artificial
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(12) [TopReadyCycle = 0, BottomReadyCycle = 0]:   $q1 = COPY %12:fpr128
+# CHECK-NEXT:   # preds left       : 2
+# CHECK-NEXT:   # succs left       : 1
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 3
+# CHECK-NEXT:   Depth              : 11
+# CHECK-NEXT:   Height             : 3
+# CHECK-NEXT:   Predecessors:
+# CHECK-NEXT:     SU(10): Data Latency=4 Reg=%12
+# CHECK-NEXT:     SU(1): Anti Latency=0
+# CHECK-NEXT:   Successors:
+# CHECK-NEXT:     ExitSU: Ord  Latency=3 Artificial
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: ExitSU [TopReadyCycle = 0, BottomReadyCycle = 0]:   RET_ReallyLR implicit $q0, implicit $q1
+# CHECK-NEXT:   # preds left       : 2
+# CHECK-NEXT:   # succs left       : 0
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 0
+# CHECK-NEXT:   Depth              : 14
+# CHECK-NEXT:   Height             : 0
+# CHECK-NEXT:   Predecessors:
+# CHECK-NEXT:     SU(12): Ord  Latency=3 Artificial
+# CHECK-NEXT:     SU(11): Ord  Latency=3 Artificial
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @0c
+# CHECK-NEXT: Critical Path(GS-RR ): 14
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT: Cycle: 3 BotQ.A
+# CHECK-NEXT:   Resource booking (@3c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@3c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@3c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@3c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT: Queue BotQ.P: 
+# CHECK-NEXT: Queue BotQ.A: 12 11 
+# CHECK-NEXT:   Cand SU(12) ORDER                              
+# CHECK-NEXT: Pick Bot ORDER     
+# CHECK-NEXT: Scheduling SU(12) $q1 = COPY %12:fpr128
+# CHECK-NEXT:   Ready @3c
+# CHECK-NEXT:   CortexA55UnitALU +1x1u
+# CHECK-NEXT:   Resource booking (@3c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@3c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@3c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@3c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   BotQ.A TopLatency SU(12) 11c
+# CHECK-NEXT:   BotQ.A BotLatency SU(12) 3c
+# CHECK-NEXT: BotQ.A @3c
+# CHECK-NEXT:   Retired: 1
+# CHECK-NEXT:   Executed: 3c
+# CHECK-NEXT:   Critical: 0c, 1 MOps
+# CHECK-NEXT:   ExpectedLatency: 3c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT:   Resource booking (@3c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@3c): 
+# CHECK-NEXT:     Instance 0 available @4c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @0c
+# CHECK-NEXT: Queue BotQ.P: 10 
+# CHECK-NEXT: Queue BotQ.A: 11 
+# CHECK-NEXT: Scheduling SU(11) $q0 = COPY %10:fpr128
+# CHECK-NEXT:   Ready @3c
+# CHECK-NEXT:   CortexA55UnitALU +1x1u
+# CHECK-NEXT:   Resource booking (@3c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@3c): 
+# CHECK-NEXT:     Instance 0 available @4c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @0c
+# CHECK-NEXT:   Resource booking (@3c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@3c): 
+# CHECK-NEXT:     Instance 0 available @3c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @0c
+# CHECK-NEXT:   *** Max MOps 2 at cycle 3
+# CHECK-NEXT: Cycle: 4 BotQ.A
+# CHECK-NEXT: BotQ.A @4c
+# CHECK-NEXT:   Retired: 2
+# CHECK-NEXT:   Executed: 4c
+# CHECK-NEXT:   Critical: 1c, 2 MOps
+# CHECK-NEXT:   ExpectedLatency: 3c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT: Cycle: 7 BotQ.A
+# CHECK-NEXT:   Resource booking (@7c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@7c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@7c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@7c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @0c
+# CHECK-NEXT: Queue BotQ.P: 
+# CHECK-NEXT: Queue BotQ.A: 10 8 
+# CHECK-NEXT:   Cand SU(10) ORDER                              
+# CHECK-NEXT: Pick Bot ORDER     
+# CHECK-NEXT: Scheduling SU(10) %12:fpr128 = UMULLv4i16_v4i32 %3.dsub:fpr128, %11:fpr64
+# CHECK-NEXT:   Ready @7c
+# CHECK-NEXT:   CortexA55UnitFPALU +2x1u
+# CHECK-NEXT:   Resource booking (@7c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@7c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@7c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@7c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @0c
+# CHECK-NEXT:   BotQ.A BotLatency SU(10) 7c
+# CHECK-NEXT:   Bump cycle to begin group
+# CHECK-NEXT: Cycle: 8 BotQ.A
+# CHECK-NEXT: BotQ.A @8c
+# CHECK-NEXT:   Retired: 3
+# CHECK-NEXT:   Executed: 8c
+# CHECK-NEXT:   Critical: 1c, 3 MOps
+# CHECK-NEXT:   ExpectedLatency: 7c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT:   Resource booking (@8c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@8c): 
+# CHECK-NEXT:     Instance 0 available @9c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @0c
+# CHECK-NEXT: Queue BotQ.P: 9 3 
+# CHECK-NEXT: Queue BotQ.A: 8 
+# CHECK-NEXT: Scheduling SU(8) %10:fpr128 = UMULLv4i16_v4i32 %0.dsub:fpr128, %9:fpr64
+# CHECK-NEXT:   Ready @8c
+# CHECK-NEXT:   CortexA55UnitFPALU +2x1u
+# CHECK-NEXT:   Resource booking (@8c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@8c): 
+# CHECK-NEXT:     Instance 0 available @9c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @0c
+# CHECK-NEXT:   Resource booking (@8c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@8c): 
+# CHECK-NEXT:     Instance 0 available @7c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @0c
+# CHECK-NEXT:   BotQ.A TopLatency SU(8) 7c
+# CHECK-NEXT:   Bump cycle to begin group
+# CHECK-NEXT: Cycle: 9 BotQ.A
+# CHECK-NEXT: BotQ.A @9c
+# CHECK-NEXT:   Retired: 4
+# CHECK-NEXT:   Executed: 9c
+# CHECK-NEXT:   Critical: 2c, 4 MOps
+# CHECK-NEXT:   ExpectedLatency: 7c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT:   Resource booking (@9c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@9c): 
+# CHECK-NEXT:     Instance 0 available @8c
+# CHECK-NEXT:     Instance 1 available @9c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @8c
+# CHECK-NEXT:   Resource booking (@9c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@9c): 
+# CHECK-NEXT:     Instance 0 available @8c
+# CHECK-NEXT:     Instance 1 available @9c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @8c
+# CHECK-NEXT: Queue BotQ.P: 7 3 
+# CHECK-NEXT: Queue BotQ.A: 9 
+# CHECK-NEXT: Scheduling SU(9) %11:fpr64 = XTNv4i16 %7:fpr128
+# CHECK-NEXT:   Ready @9c
+# CHECK-NEXT:   CortexA55UnitFPALU +1x1u
+# CHECK-NEXT:   Resource booking (@9c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@9c): 
+# CHECK-NEXT:     Instance 0 available @8c
+# CHECK-NEXT:     Instance 1 available @9c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @8c
+# CHECK-NEXT:   Resource booking (@9c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@9c): 
+# CHECK-NEXT:     Instance 0 available @7c
+# CHECK-NEXT:     Instance 1 available @8c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @7c
+# CHECK-NEXT:   BotQ.A BotLatency SU(9) 9c
+# CHECK-NEXT: BotQ.A @9c
+# CHECK-NEXT:   Retired: 5
+# CHECK-NEXT:   Executed: 9c
+# CHECK-NEXT:   Critical: 2c, 5 MOps
+# CHECK-NEXT:   ExpectedLatency: 9c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT: Cycle: 10 BotQ.A
+# CHECK-NEXT:   Resource booking (@10c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@10c): 
+# CHECK-NEXT:     Instance 0 available @10c
+# CHECK-NEXT:     Instance 1 available @9c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @9c
+# CHECK-NEXT:   Resource booking (@10c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@10c): 
+# CHECK-NEXT:     Instance 0 available @11c
+# CHECK-NEXT:     Instance 1 available @10c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @10c
+# CHECK-NEXT: Queue BotQ.P: 3 
+# CHECK-NEXT: Queue BotQ.A: 7 5 
+# CHECK-NEXT:   Cand SU(7) ORDER                              
+# CHECK-NEXT: Pick Bot ORDER     
+# CHECK-NEXT: Scheduling SU(7) %9:fpr64 = XTNv4i16 %8:fpr128
+# CHECK-NEXT:   Ready @10c
+# CHECK-NEXT:   CortexA55UnitFPALU +1x1u
+# CHECK-NEXT:   Resource booking (@10c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@10c): 
+# CHECK-NEXT:     Instance 0 available @10c
+# CHECK-NEXT:     Instance 1 available @9c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @9c
+# CHECK-NEXT:   Resource booking (@10c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@10c): 
+# CHECK-NEXT:     Instance 0 available @9c
+# CHECK-NEXT:     Instance 1 available @8c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @8c
+# CHECK-NEXT: BotQ.A @10c
+# CHECK-NEXT:   Retired: 6
+# CHECK-NEXT:   Executed: 10c
+# CHECK-NEXT:   Critical: 3c, 6 MOps
+# CHECK-NEXT:   ExpectedLatency: 9c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT:   Resource booking (@10c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@10c): 
+# CHECK-NEXT:     Instance 0 available @11c
+# CHECK-NEXT:     Instance 1 available @12c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @11c
+# CHECK-NEXT:   SU(5) CortexA55UnitFPALU[0]=11c
+# CHECK-NEXT: Cycle: 11 BotQ.A
+# CHECK-NEXT:   Resource booking (@11c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@11c): 
+# CHECK-NEXT:     Instance 0 available @11c
+# CHECK-NEXT:     Instance 1 available @12c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @11c
+# CHECK-NEXT:   Resource booking (@11c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@11c): 
+# CHECK-NEXT:     Instance 0 available @11c
+# CHECK-NEXT:     Instance 1 available @12c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @11c
+# CHECK-NEXT:   Resource booking (@11c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@11c): 
+# CHECK-NEXT:     Instance 0 available @11c
+# CHECK-NEXT:     Instance 1 available @12c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @11c
+# CHECK-NEXT: Queue BotQ.P: 
+# CHECK-NEXT: Queue BotQ.A: 3 5 6 
+# CHECK-NEXT:   Cand SU(3) ORDER                              
+# CHECK-NEXT:   Cand SU(5) ORDER                              
+# CHECK-NEXT:   Cand SU(6) ORDER                              
+# CHECK-NEXT: Pick Bot ORDER     
+# CHECK-NEXT: Scheduling SU(6) %8:fpr128 = ANDv16i8 %1:fpr128, %6:fpr128
+# CHECK-NEXT:   Ready @11c
+# CHECK-NEXT:   CortexA55UnitFPALU +2x1u
+# CHECK-NEXT:   *** Critical resource CortexA55UnitFPALU: 4c
+# CHECK-NEXT:   Resource booking (@11c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@11c): 
+# CHECK-NEXT:     Instance 0 available @11c
+# CHECK-NEXT:     Instance 1 available @12c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @11c
+# CHECK-NEXT:   Resource booking (@11c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@11c): 
+# CHECK-NEXT:     Instance 0 available @9c
+# CHECK-NEXT:     Instance 1 available @10c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @9c
+# CHECK-NEXT:   BotQ.A BotLatency SU(6) 10c
+# CHECK-NEXT:   Bump cycle to begin group
+# CHECK-NEXT: Cycle: 12 BotQ.A
+# CHECK-NEXT: BotQ.A @12c
+# CHECK-NEXT:   Retired: 7
+# CHECK-NEXT:   Executed: 12c
+# CHECK-NEXT:   Critical: 4c, 8 CortexA55UnitFPALU
+# CHECK-NEXT:   ExpectedLatency: 10c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT:   Resource booking (@12c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@12c): 
+# CHECK-NEXT:     Instance 0 available @13c
+# CHECK-NEXT:     Instance 1 available @12c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @12c
+# CHECK-NEXT:   Resource booking (@12c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@12c): 
+# CHECK-NEXT:     Instance 0 available @13c
+# CHECK-NEXT:     Instance 1 available @12c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @12c
+# CHECK-NEXT: Queue BotQ.P: 1 
+# CHECK-NEXT: Queue BotQ.A: 3 5 
+# CHECK-NEXT:   Cand SU(3) ORDER                              
+# CHECK-NEXT:   Cand SU(5) ORDER                              
+# CHECK-NEXT: Pick Bot ORDER     
+# CHECK-NEXT: Scheduling SU(5) %7:fpr128 = ANDv16i8 %2:fpr128, %6:fpr128
+# CHECK-NEXT:   Ready @12c
+# CHECK-NEXT:   CortexA55UnitFPALU +2x1u
+# CHECK-NEXT:   Resource booking (@12c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@12c): 
+# CHECK-NEXT:     Instance 0 available @13c
+# CHECK-NEXT:     Instance 1 available @12c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @12c
+# CHECK-NEXT:   Resource booking (@12c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@12c): 
+# CHECK-NEXT:     Instance 0 available @11c
+# CHECK-NEXT:     Instance 1 available @10c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @10c
+# CHECK-NEXT:   BotQ.A TopLatency SU(5) 4c
+# CHECK-NEXT:   Bump cycle to begin group
+# CHECK-NEXT: Cycle: 13 BotQ.A
+# CHECK-NEXT: BotQ.A @13c
+# CHECK-NEXT:   Retired: 8
+# CHECK-NEXT:   Executed: 13c
+# CHECK-NEXT:   Critical: 5c, 10 CortexA55UnitFPALU
+# CHECK-NEXT:   ExpectedLatency: 10c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT:   Resource booking (@13c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@13c): 
+# CHECK-NEXT:     Instance 0 available @13c
+# CHECK-NEXT:     Instance 1 available @14c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @13c
+# CHECK-NEXT: Queue BotQ.P: 1 4 0 
+# CHECK-NEXT: Queue BotQ.A: 3 
+# CHECK-NEXT: Scheduling SU(3) %3:fpr128 = EXTv16i8 %0:fpr128, %0:fpr128, 8
+# CHECK-NEXT:   Ready @13c
+# CHECK-NEXT:   CortexA55UnitFPALU +2x1u
+# CHECK-NEXT:   Resource booking (@13c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@13c): 
+# CHECK-NEXT:     Instance 0 available @13c
+# CHECK-NEXT:     Instance 1 available @14c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @13c
+# CHECK-NEXT:   Resource booking (@13c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@13c): 
+# CHECK-NEXT:     Instance 0 available @11c
+# CHECK-NEXT:     Instance 1 available @12c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @11c
+# CHECK-NEXT:   BotQ.A BotLatency SU(3) 11c
+# CHECK-NEXT:   Bump cycle to begin group
+# CHECK-NEXT: Cycle: 14 BotQ.A
+# CHECK-NEXT: BotQ.A @14c
+# CHECK-NEXT:   Retired: 9
+# CHECK-NEXT:   Executed: 14c
+# CHECK-NEXT:   Critical: 6c, 12 CortexA55UnitFPALU
+# CHECK-NEXT:   ExpectedLatency: 11c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT:   Resource booking (@14c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@14c): 
+# CHECK-NEXT:     Instance 0 available @4c
+# CHECK-NEXT:     Instance 1 available @4c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @4c
+# CHECK-NEXT:   Resource booking (@14c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@14c): 
+# CHECK-NEXT:     Instance 0 available @4c
+# CHECK-NEXT:     Instance 1 available @4c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @4c
+# CHECK-NEXT: Queue BotQ.P: 2 4 0 
+# CHECK-NEXT: Queue BotQ.A: 1 
+# CHECK-NEXT: Scheduling SU(1) %1:fpr128 = COPY $q1
+# CHECK-NEXT:   Ready @14c
+# CHECK-NEXT:   CortexA55UnitALU +1x1u
+# CHECK-NEXT:   Resource booking (@14c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@14c): 
+# CHECK-NEXT:     Instance 0 available @4c
+# CHECK-NEXT:     Instance 1 available @4c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @4c
+# CHECK-NEXT:   Resource booking (@14c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@14c): 
+# CHECK-NEXT:     Instance 0 available @3c
+# CHECK-NEXT:     Instance 1 available @3c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @3c
+# CHECK-NEXT:   BotQ.A BotLatency SU(1) 13c
+# CHECK-NEXT: BotQ.A @14c
+# CHECK-NEXT:   Retired: 10
+# CHECK-NEXT:   Executed: 14c
+# CHECK-NEXT:   Critical: 6c, 12 CortexA55UnitFPALU
+# CHECK-NEXT:   ExpectedLatency: 13c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 14
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT: Cycle: 15 BotQ.A
+# CHECK-NEXT:   Resource booking (@15c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 14
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@15c): 
+# CHECK-NEXT:     Instance 0 available @15c
+# CHECK-NEXT:     Instance 1 available @4c
+# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @4c
+# CHECK-NEXT: Queue BotQ.P: 2 4 
+# CHECK-NEXT: Queue BotQ.A: 0 
+# CHECK-NEXT: Scheduling SU(0) %2:fpr128 = COPY $q2
+# CHECK-NEXT:   Ready @15c
+# CHECK-NEXT:   CortexA55UnitALU +1x1u
+# CHECK-NEXT:   Resource booking (@15c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 14
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@15c): 
+# CHECK-NEXT:     Instance 0 available @15c
+# CHECK-NEXT:     Instance 1 available @4c
+# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @4c
+# CHECK-NEXT:   Resource booking (@15c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 14
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@15c): 
+# CHECK-NEXT:     Instance 0 available @14c
+# CHECK-NEXT:     Instance 1 available @3c
+# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @3c
+# CHECK-NEXT: BotQ.A @15c
+# CHECK-NEXT:   Retired: 11
+# CHECK-NEXT:   Executed: 15c
+# CHECK-NEXT:   Critical: 6c, 12 CortexA55UnitFPALU
+# CHECK-NEXT:   ExpectedLatency: 13c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 14
+# CHECK-NEXT: CortexA55UnitALU(1) = 15
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT: Cycle: 16 BotQ.A
+# CHECK-NEXT:   Resource booking (@16c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 14
+# CHECK-NEXT: CortexA55UnitALU(1) = 15
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@16c): 
+# CHECK-NEXT:     Instance 0 available @15c
+# CHECK-NEXT:     Instance 1 available @16c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @15c
+# CHECK-NEXT:   Resource booking (@16c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 14
+# CHECK-NEXT: CortexA55UnitALU(1) = 15
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@16c): 
+# CHECK-NEXT:     Instance 0 available @15c
+# CHECK-NEXT:     Instance 1 available @14c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @14c
+# CHECK-NEXT: Queue BotQ.P: 
+# CHECK-NEXT: Queue BotQ.A: 2 4 
+# CHECK-NEXT:   Cand SU(2) ORDER                              
+# CHECK-NEXT:   Cand SU(4) PHYS-REG                           
+# CHECK-NEXT: Pick Bot PHYS-REG  
+# CHECK-NEXT: Scheduling SU(4) %6:fpr128 = MOVIv2d_ns 17
+# CHECK-NEXT:   Ready @16c
+# CHECK-NEXT:   CortexA55UnitFPALU +2x1u
+# CHECK-NEXT:   Resource booking (@16c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 14
+# CHECK-NEXT: CortexA55UnitALU(1) = 15
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@16c): 
+# CHECK-NEXT:     Instance 0 available @15c
+# CHECK-NEXT:     Instance 1 available @14c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @14c
+# CHECK-NEXT:   Resource booking (@16c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 14
+# CHECK-NEXT: CortexA55UnitALU(1) = 15
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@16c): 
+# CHECK-NEXT:     Instance 0 available @13c
+# CHECK-NEXT:     Instance 1 available @12c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @12c
+# CHECK-NEXT:   BotQ.A BotLatency SU(4) 14c
+# CHECK-NEXT:   Bump cycle to begin group
+# CHECK-NEXT: Cycle: 17 BotQ.A
+# CHECK-NEXT: BotQ.A @17c
+# CHECK-NEXT:   Retired: 12
+# CHECK-NEXT:   Executed: 17c
+# CHECK-NEXT:   Critical: 7c, 14 CortexA55UnitFPALU
+# CHECK-NEXT:   ExpectedLatency: 14c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 14
+# CHECK-NEXT: CortexA55UnitALU(1) = 15
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 16
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT:   Resource booking (@17c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 14
+# CHECK-NEXT: CortexA55UnitALU(1) = 15
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 16
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@17c): 
+# CHECK-NEXT:     Instance 0 available @15c
+# CHECK-NEXT:     Instance 1 available @16c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @15c
+# CHECK-NEXT: Queue BotQ.P: 
+# CHECK-NEXT: Queue BotQ.A: 2 
+# CHECK-NEXT: Scheduling SU(2) %0:fpr128 = COPY $q0
+# CHECK-NEXT:   Ready @17c
+# CHECK-NEXT:   CortexA55UnitALU +1x1u
+# CHECK-NEXT:   Resource booking (@17c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 14
+# CHECK-NEXT: CortexA55UnitALU(1) = 15
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 16
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@17c): 
+# CHECK-NEXT:     Instance 0 available @15c
+# CHECK-NEXT:     Instance 1 available @16c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @15c
+# CHECK-NEXT:   Resource booking (@17c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 14
+# CHECK-NEXT: CortexA55UnitALU(1) = 15
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 16
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@17c): 
+# CHECK-NEXT:     Instance 0 available @14c
+# CHECK-NEXT:     Instance 1 available @15c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @14c
+# CHECK-NEXT: BotQ.A @17c
+# CHECK-NEXT:   Retired: 13
+# CHECK-NEXT:   Executed: 17c
+# CHECK-NEXT:   Critical: 7c, 14 CortexA55UnitFPALU
+# CHECK-NEXT:   ExpectedLatency: 14c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 17
+# CHECK-NEXT: CortexA55UnitALU(1) = 15
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 16
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT: *** Final schedule for %bb.0 ***
+# CHECK-NEXT:  * Schedule table (BottomUp):
+# CHECK-NEXT:   i: issue
+# CHECK-NEXT:   x: resource booked
+# CHECK-NEXT: Cycle              | 17 | 16 | 15 | 14 | 13 | 12 | 11 | 10 | 9  | 8  | 7  | 6  | 5  | 4  | 3  |
+# CHECK-NEXT: SU(2)              | i  |    |    |    |    |    |    |    |    |    |    |    |    |    |    |
+# CHECK-NEXT:   CortexA55UnitALU | x  |    |    |    |    |    |    |    |    |    |    |    |    |    |    | 
+# CHECK-NEXT: SU(4)              |    | i  |    |    |    |    |    |    |    |    |    |    |    |    |    |
+# CHECK-NEXT: CortexA55UnitFPALU |    | x  | x  |    |    |    |    |    |    |    |    |    |    |    |    | 
+# CHECK-NEXT: SU(0)              |    |    | i  |    |    |    |    |    |    |    |    |    |    |    |    |
+# CHECK-NEXT:   CortexA55UnitALU |    |    | x  |    |    |    |    |    |    |    |    |    |    |    |    | 
+# CHECK-NEXT: SU(1)              |    |    |    | i  |    |    |    |    |    |    |    |    |    |    |    |
+# CHECK-NEXT:   CortexA55UnitALU |    |    |    | x  |    |    |    |    |    |    |    |    |    |    |    | 
+# CHECK-NEXT: SU(3)              |    |    |    |    | i  |    |    |    |    |    |    |    |    |    |    |
+# CHECK-NEXT: CortexA55UnitFPALU |    |    |    |    | x  | x  |    |    |    |    |    |    |    |    |    | 
+# CHECK-NEXT: SU(5)              |    |    |    |    |    | i  |    |    |    |    |    |    |    |    |    |
+# CHECK-NEXT: CortexA55UnitFPALU |    |    |    |    |    | x  | x  |    |    |    |    |    |    |    |    | 
+# CHECK-NEXT: SU(6)              |    |    |    |    |    |    | i  |    |    |    |    |    |    |    |    |
+# CHECK-NEXT: CortexA55UnitFPALU |    |    |    |    |    |    | x  | x  |    |    |    |    |    |    |    | 
+# CHECK-NEXT: SU(7)              |    |    |    |    |    |    |    | i  |    |    |    |    |    |    |    |
+# CHECK-NEXT: CortexA55UnitFPALU |    |    |    |    |    |    |    | x  |    |    |    |    |    |    |    | 
+# CHECK-NEXT: SU(9)              |    |    |    |    |    |    |    |    | i  |    |    |    |    |    |    |
+# CHECK-NEXT: CortexA55UnitFPALU |    |    |    |    |    |    |    |    | x  |    |    |    |    |    |    | 
+# CHECK-NEXT: SU(8)              |    |    |    |    |    |    |    |    |    | i  |    |    |    |    |    |
+# CHECK-NEXT: CortexA55UnitFPALU |    |    |    |    |    |    |    |    |    | x  | x  |    |    |    |    | 
+# CHECK-NEXT: SU(10)             |    |    |    |    |    |    |    |    |    |    | i  |    |    |    |    |
+# CHECK-NEXT: CortexA55UnitFPALU |    |    |    |    |    |    |    |    |    |    | x  | x  |    |    |    | 
+# CHECK-NEXT: SU(11)             |    |    |    |    |    |    |    |    |    |    |    |    |    |    | i  |
+# CHECK-NEXT:   CortexA55UnitALU |    |    |    |    |    |    |    |    |    |    |    |    |    |    | x  | 
+# CHECK-NEXT: SU(12)             |    |    |    |    |    |    |    |    |    |    |    |    |    |    | i  |
+# CHECK-NEXT:   CortexA55UnitALU |    |    |    |    |    |    |    |    |    |    |    |    |    |    | x  | 
+# CHECK-NEXT: SU(2) [TopReadyCycle = 0, BottomReadyCycle = 17]:   %0:fpr128 = COPY $q0
+# CHECK-NEXT: SU(4) [TopReadyCycle = 0, BottomReadyCycle = 16]:   %6:fpr128 = MOVIv2d_ns 17
+# CHECK-NEXT: SU(0) [TopReadyCycle = 0, BottomReadyCycle = 15]:   %2:fpr128 = COPY $q2
+# CHECK-NEXT: SU(1) [TopReadyCycle = 0, BottomReadyCycle = 14]:   %1:fpr128 = COPY $q1
+# CHECK-NEXT: SU(3) [TopReadyCycle = 0, BottomReadyCycle = 13]:   %3:fpr128 = EXTv16i8 %0:fpr128, %0:fpr128, 8
+# CHECK-NEXT: SU(5) [TopReadyCycle = 0, BottomReadyCycle = 12]:   %7:fpr128 = ANDv16i8 %2:fpr128, %6:fpr128
+# CHECK-NEXT: SU(6) [TopReadyCycle = 0, BottomReadyCycle = 11]:   %8:fpr128 = ANDv16i8 %1:fpr128, %6:fpr128
+# CHECK-NEXT: SU(7) [TopReadyCycle = 0, BottomReadyCycle = 10]:   %9:fpr64 = XTNv4i16 %8:fpr128
+# CHECK-NEXT: SU(9) [TopReadyCycle = 0, BottomReadyCycle = 9]:   %11:fpr64 = XTNv4i16 %7:fpr128
+# CHECK-NEXT: SU(8) [TopReadyCycle = 0, BottomReadyCycle = 8]:   %10:fpr128 = UMULLv4i16_v4i32 %0.dsub:fpr128, %9:fpr64
+# CHECK-NEXT: SU(10) [TopReadyCycle = 0, BottomReadyCycle = 7]:   %12:fpr128 = UMULLv4i16_v4i32 %3.dsub:fpr128, %11:fpr64
+# CHECK-NEXT: SU(11) [TopReadyCycle = 0, BottomReadyCycle = 3]:   $q0 = COPY %10:fpr128
+# CHECK-NEXT: SU(12) [TopReadyCycle = 0, BottomReadyCycle = 3]:   $q1 = COPY %12:fpr128
+# CHECK-EMPTY: 
+# CHECK-NEXT: ********** INTERVALS **********
+# CHECK-NEXT: B0 [0B,48r:0)[192r,224r:1) 0@0B-phi 1@192r
+# CHECK-NEXT: B1 [0B,96r:0)[208r,224r:1) 0@0B-phi 1@208r
+# CHECK-NEXT: B2 [0B,88r:0) 0@0B-phi
+# CHECK-NEXT: %0 [48r,168r:0) 0@48r  weight:0.000000e+00
+# CHECK-NEXT: %1 [96r,120r:0) 0@96r  weight:0.000000e+00
+# CHECK-NEXT: %2 [88r,112r:0) 0@88r  weight:0.000000e+00
+# CHECK-NEXT: %3 [104r,176r:0) 0@104r  weight:0.000000e+00
+# CHECK-NEXT: %6 [80r,120r:0) 0@80r  weight:0.000000e+00
+# CHECK-NEXT: %7 [112r,160r:0) 0@112r  weight:0.000000e+00
+# CHECK-NEXT: %8 [120r,128r:0) 0@120r  weight:0.000000e+00
+# CHECK-NEXT: %9 [128r,168r:0) 0@128r  weight:0.000000e+00
+# CHECK-NEXT: %10 [168r,192r:0) 0@168r  weight:0.000000e+00
+# CHECK-NEXT: %11 [160r,176r:0) 0@160r  weight:0.000000e+00
+# CHECK-NEXT: %12 [176r,208r:0) 0@176r  weight:0.000000e+00
+# CHECK-NEXT: RegMasks:
+# CHECK-NEXT: ********** MACHINEINSTRS **********
+# CHECK-NEXT: # Machine code for function umull_and_v8i32: IsSSA, NoPHIs, TracksLiveness
+# CHECK-NEXT: Function Live Ins: $q0 in %0, $q1 in %1, $q2 in %2
+# CHECK-EMPTY: 
+# CHECK-NEXT: 0B	bb.0.entry:
+# CHECK-NEXT: 	  liveins: $q0, $q1, $q2
+# CHECK-NEXT: 48B	  %0:fpr128 = COPY $q0
+# CHECK-NEXT: 80B	  %6:fpr128 = MOVIv2d_ns 17
+# CHECK-NEXT: 88B	  %2:fpr128 = COPY $q2
+# CHECK-NEXT: 96B	  %1:fpr128 = COPY $q1
+# CHECK-NEXT: 104B	  %3:fpr128 = EXTv16i8 %0:fpr128, %0:fpr128, 8
+# CHECK-NEXT: 112B	  %7:fpr128 = ANDv16i8 %2:fpr128, %6:fpr128
+# CHECK-NEXT: 120B	  %8:fpr128 = ANDv16i8 %1:fpr128, %6:fpr128
+# CHECK-NEXT: 128B	  %9:fpr64 = XTNv4i16 %8:fpr128
+# CHECK-NEXT: 160B	  %11:fpr64 = XTNv4i16 %7:fpr128
+# CHECK-NEXT: 168B	  %10:fpr128 = UMULLv4i16_v4i32 %0.dsub:fpr128, %9:fpr64
+# CHECK-NEXT: 176B	  %12:fpr128 = UMULLv4i16_v4i32 %3.dsub:fpr128, %11:fpr64
+# CHECK-NEXT: 192B	  $q0 = COPY %10:fpr128
+# CHECK-NEXT: 208B	  $q1 = COPY %12:fpr128
+# CHECK-NEXT: 224B	  RET_ReallyLR implicit $q0, implicit $q1
+# CHECK-EMPTY: 
+# CHECK-NEXT: # End machine code for function umull_and_v8i32.
diff --git a/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-02.mir b/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-02.mir
new file mode 100644
index 0000000000000..0f5dc000c50b1
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-02.mir
@@ -0,0 +1,479 @@
+# RUN: llc -mtriple=aarch64-none-linux-gnu -mcpu=cortex-a55  \
+# RUN:  -run-pass=machine-scheduler -debug-only=machine-scheduler -o - %s \
+# RUN:  -misched-bottomup=true -sched-print-cycles=true \
+# RUN:  -misched-dump-reserved-cycles=true -misched-detail-resource-booking=true\
+# RUN:  -misched-dump-schedule-trace=true -misched-dump-schedule-trace-col-width=4 \
+# RUN:  2>&1 | FileCheck %s
+
+# REQUIRES: asserts, aarch64-registered-target
+---
+name: f
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1, $x2
+    $x3 = ADDXrr $x0, $x0
+    $x4 = ADDXrr $x1, $x1
+    $x5 = ADDXrr $x2, $x2
+    
+# CHECK-LABEL: Before MISched:
+# CHECK-NEXT: # Machine code for function f: IsSSA, NoPHIs, TracksLiveness, NoVRegs
+# CHECK-EMPTY: 
+# CHECK-NEXT: bb.0:
+# CHECK-NEXT:   liveins: $x0, $x1, $x2
+# CHECK-NEXT:   $x3 = ADDXrr $x0, $x0
+# CHECK-NEXT:   $x4 = ADDXrr $x1, $x1
+# CHECK-NEXT:   $x5 = ADDXrr $x2, $x2
+# CHECK-EMPTY: 
+# CHECK-NEXT: # End machine code for function f.
+# CHECK-EMPTY: 
+# CHECK-NEXT: ********** MI Scheduling **********
+# CHECK-NEXT: f:%bb.0 
+# CHECK-NEXT:   From: $x3 = ADDXrr $x0, $x0
+# CHECK-NEXT:     To: End
+# CHECK-NEXT:  RegionInstrs: 3
+# CHECK-NEXT: ScheduleDAGMILive::schedule starting
+# CHECK-NEXT: GenericScheduler RegionPolicy:  ShouldTrackPressure=0 OnlyTopDown=0 OnlyBottomUp=1
+# CHECK-NEXT: Disabled scoreboard hazard recognizer
+# CHECK-NEXT: Disabled scoreboard hazard recognizer
+# CHECK-NEXT: SU(0) [TopReadyCycle = 0, BottomReadyCycle = 0]:   $x3 = ADDXrr $x0, $x0
+# CHECK-NEXT:   # preds left       : 0
+# CHECK-NEXT:   # succs left       : 0
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 3
+# CHECK-NEXT:   Depth              : 0
+# CHECK-NEXT:   Height             : 0
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(1) [TopReadyCycle = 0, BottomReadyCycle = 0]:   $x4 = ADDXrr $x1, $x1
+# CHECK-NEXT:   # preds left       : 0
+# CHECK-NEXT:   # succs left       : 0
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 3
+# CHECK-NEXT:   Depth              : 0
+# CHECK-NEXT:   Height             : 0
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(2) [TopReadyCycle = 0, BottomReadyCycle = 0]:   $x5 = ADDXrr $x2, $x2
+# CHECK-NEXT:   # preds left       : 0
+# CHECK-NEXT:   # succs left       : 0
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 3
+# CHECK-NEXT:   Depth              : 0
+# CHECK-NEXT:   Height             : 0
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT: Critical Path(GS-RR ): 0
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT: Queue BotQ.P: 
+# CHECK-NEXT: Queue BotQ.A: 2 1 0 
+# CHECK-NEXT:   Cand SU(2) ORDER                              
+# CHECK-NEXT: Pick Bot ORDER     
+# CHECK-NEXT: Scheduling SU(2) $x5 = ADDXrr $x2, $x2
+# CHECK-NEXT:   Ready @0c
+# CHECK-NEXT:   CortexA55UnitALU +1x1u
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT: BotQ.A @0c
+# CHECK-NEXT:   Retired: 1
+# CHECK-NEXT:   Executed: 0c
+# CHECK-NEXT:   Critical: 0c, 1 MOps
+# CHECK-NEXT:   ExpectedLatency: 0c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 0
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 0
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @1c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @0c
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 0
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @1c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @0c
+# CHECK-NEXT: Queue BotQ.P: 
+# CHECK-NEXT: Queue BotQ.A: 0 1 
+# CHECK-NEXT:   Cand SU(0) ORDER                              
+# CHECK-NEXT:   Cand SU(1) ORDER                              
+# CHECK-NEXT: Pick Bot ORDER     
+# CHECK-NEXT: Scheduling SU(1) $x4 = ADDXrr $x1, $x1
+# CHECK-NEXT:   Ready @0c
+# CHECK-NEXT:   CortexA55UnitALU +1x1u
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 0
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @1c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @0c
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 0
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   *** Max MOps 2 at cycle 0
+# CHECK-NEXT: Cycle: 1 BotQ.A
+# CHECK-NEXT: BotQ.A @1c
+# CHECK-NEXT:   Retired: 2
+# CHECK-NEXT:   Executed: 1c
+# CHECK-NEXT:   Critical: 1c, 2 MOps
+# CHECK-NEXT:   ExpectedLatency: 0c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 0
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT:   Resource booking (@1c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 0
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@1c): 
+# CHECK-NEXT:     Instance 0 available @1c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @0c
+# CHECK-NEXT: Queue BotQ.P: 
+# CHECK-NEXT: Queue BotQ.A: 0 
+# CHECK-NEXT: Scheduling SU(0) $x3 = ADDXrr $x0, $x0
+# CHECK-NEXT:   Ready @1c
+# CHECK-NEXT:   CortexA55UnitALU +1x1u
+# CHECK-NEXT:   Resource booking (@1c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 0
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@1c): 
+# CHECK-NEXT:     Instance 0 available @1c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @0c
+# CHECK-NEXT:   Resource booking (@1c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 0
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@1c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT: BotQ.A @1c
+# CHECK-NEXT:   Retired: 3
+# CHECK-NEXT:   Executed: 1c
+# CHECK-NEXT:   Critical: 1c, 3 MOps
+# CHECK-NEXT:   ExpectedLatency: 0c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 1
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT: *** Final schedule for %bb.0 ***
+# CHECK-NEXT:  * Schedule table (BottomUp):
+# CHECK-NEXT:   i: issue
+# CHECK-NEXT:   x: resource booked
+# CHECK-NEXT: Cycle              | 1 | 0 |
+# CHECK-NEXT: SU(0)              | i |   |
+# CHECK-NEXT:   CortexA55UnitALU | x |   | 
+# CHECK-NEXT: SU(1)              |   | i |
+# CHECK-NEXT:   CortexA55UnitALU |   | x | 
+# CHECK-NEXT: SU(2)              |   | i |
+# CHECK-NEXT:   CortexA55UnitALU |   | x | 
+# CHECK-NEXT: SU(0) [TopReadyCycle = 0, BottomReadyCycle = 1]:   $x3 = ADDXrr $x0, $x0
+# CHECK-NEXT: SU(1) [TopReadyCycle = 0, BottomReadyCycle = 0]:   $x4 = ADDXrr $x1, $x1
+# CHECK-NEXT: SU(2) [TopReadyCycle = 0, BottomReadyCycle = 0]:   $x5 = ADDXrr $x2, $x2
+# CHECK-EMPTY: 
+# CHECK-NEXT: ********** INTERVALS **********
+# CHECK-NEXT: W0 [0B,16r:0) 0@0B-phi
+# CHECK-NEXT: W1 [0B,32r:0) 0@0B-phi
+# CHECK-NEXT: W2 [0B,48r:0) 0@0B-phi
+# CHECK-NEXT: RegMasks:
+# CHECK-NEXT: ********** MACHINEINSTRS **********
+# CHECK-NEXT: # Machine code for function f: IsSSA, NoPHIs, TracksLiveness, NoVRegs
+# CHECK-EMPTY: 
+# CHECK-NEXT: 0B	bb.0:
+# CHECK-NEXT: 	  liveins: $x0, $x1, $x2
+# CHECK-NEXT: 16B	  $x3 = ADDXrr $x0, $x0
+# CHECK-NEXT: 32B	  $x4 = ADDXrr $x1, $x1
+# CHECK-NEXT: 48B	  $x5 = ADDXrr $x2, $x2
+# CHECK-EMPTY: 
+# CHECK-NEXT: # End machine code for function f.
\ No newline at end of file

From 25f8b1a0a812717ecf095761dd6c7d6601408a1c Mon Sep 17 00:00:00 2001
From: Francesco Petrogalli <francesco.petrogalli@apple.com>
Date: Tue, 20 Jun 2023 11:28:45 +0200
Subject: [PATCH 105/130] Revert "[llc][MISched] Add
 `-misched-detail-resource-booking` to llc."

Reverting because of https://lab.llvm.org/buildbot#builders/75/builds/32485:

llvm-project/llvm/lib/CodeGen/MachineScheduler.cpp:2374:7: error: use of undeclared identifier 'MischedDetailResourceBooking'
 if (MischedDetailResourceBooking)

This reverts commit fc06262c1c365777e71207b6a5de281cba927c96.
---
 llvm/lib/CodeGen/MachineScheduler.cpp         |   18 +-
 .../misched-detail-resource-booking-01.mir    | 1618 -----------------
 .../misched-detail-resource-booking-02.mir    |  479 -----
 3 files changed, 1 insertion(+), 2114 deletions(-)
 delete mode 100644 llvm/test/CodeGen/AArch64/misched-detail-resource-booking-01.mir
 delete mode 100644 llvm/test/CodeGen/AArch64/misched-detail-resource-booking-02.mir

diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index 3adcf706f8388..28b17f76c99e2 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -98,15 +98,11 @@ cl::opt<bool> PrintDAGs("misched-print-dags", cl::Hidden,
 cl::opt<bool> MISchedDumpReservedCycles(
     "misched-dump-reserved-cycles", cl::Hidden, cl::init(false),
     cl::desc("Dump resource usage at schedule boundary."));
-cl::opt<bool> MischedDetailResourceBooking(
-    "misched-detail-resource-booking", cl::Hidden, cl::init(false),
-    cl::desc("Show details of invoking getNextResoufceCycle."));
 #else
 const bool ViewMISchedDAGs = false;
 const bool PrintDAGs = false;
 #ifdef LLVM_ENABLE_DUMP
 const bool MISchedDumpReservedCycles = false;
-const bool MischedDetailResourceBooking = false;
 #endif // LLVM_ENABLE_DUMP
 #endif // NDEBUG
 
@@ -2318,11 +2314,7 @@ unsigned SchedBoundary::getNextResourceCycleByInstance(unsigned InstanceIdx,
 std::pair<unsigned, unsigned>
 SchedBoundary::getNextResourceCycle(const MCSchedClassDesc *SC, unsigned PIdx,
                                     unsigned Cycles, unsigned StartAtCycle) {
-  if (MischedDetailResourceBooking) {
-    LLVM_DEBUG(dbgs() << "  Resource booking (@" << CurrCycle << "c): \n");
-    LLVM_DEBUG(dumpReservedCycles());
-    LLVM_DEBUG(dbgs() << "  getNextResourceCycle (@" << CurrCycle << "c): \n");
-  }
+
   unsigned MinNextUnreserved = InvalidCycle;
   unsigned InstanceIdx = 0;
   unsigned StartIndex = ReservedCyclesIndex[PIdx];
@@ -2363,19 +2355,11 @@ SchedBoundary::getNextResourceCycle(const MCSchedClassDesc *SC, unsigned PIdx,
        ++I) {
     unsigned NextUnreserved =
         getNextResourceCycleByInstance(I, Cycles, StartAtCycle);
-    if (MischedDetailResourceBooking)
-      LLVM_DEBUG(dbgs() << "    Instance " << I - StartIndex << " available @"
-                        << NextUnreserved << "c\n");
     if (MinNextUnreserved > NextUnreserved) {
       InstanceIdx = I;
       MinNextUnreserved = NextUnreserved;
     }
   }
-  if (MischedDetailResourceBooking)
-    LLVM_DEBUG(dbgs() << "    selecting " << SchedModel->getResourceName(PIdx)
-                      << "[" << InstanceIdx - StartIndex << "]"
-                      << " available @" << MinNextUnreserved << "c"
-                      << "\n");
   return std::make_pair(MinNextUnreserved, InstanceIdx);
 }
 
diff --git a/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-01.mir b/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-01.mir
deleted file mode 100644
index a012fc5608a3f..0000000000000
--- a/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-01.mir
+++ /dev/null
@@ -1,1618 +0,0 @@
-# RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+neon  %s -o - 2>&1 \
-# RUN:   -misched-dump-reserved-cycles=true \
-# RUN:   -run-pass=machine-scheduler -debug-only=machine-scheduler \
-# RUN:   -misched-bottomup=true -sched-print-cycles=true \
-# RUN:   -misched-detail-resource-booking=true \
-# RUN:   -misched-dump-schedule-trace=true -misched-dump-schedule-trace-col-header-width=21 \
-# RUN: | FileCheck  %s
-
---- |
-  ; ModuleID = '../llvm-project/llvm/test/CodeGen/AArch64/aarch64-smull.failing.ll'
-  source_filename = "../llvm-project/llvm/test/CodeGen/AArch64/aarch64-smull.failing.ll"
-  target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
-  target triple = "aarch64-none-linux-gnu"
-  
-  define <8 x i32> @umull_and_v8i32(<8 x i16> %src1, <8 x i32> %src2) #0 {
-  entry:
-    %in1 = zext <8 x i16> %src1 to <8 x i32>
-    %in2 = and <8 x i32> %src2, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
-    %out = mul nsw <8 x i32> %in1, %in2
-    ret <8 x i32> %out
-  }
-  
-  attributes #0 = { "target-features"="+neon" }
-
-...
----
-name:            umull_and_v8i32
-alignment:       4
-tracksRegLiveness: true
-registers:
-  - { id: 0, class: fpr128 }
-  - { id: 1, class: fpr128 }
-  - { id: 2, class: fpr128 }
-  - { id: 3, class: fpr128 }
-  - { id: 4, class: fpr64 }
-  - { id: 5, class: fpr64 }
-  - { id: 6, class: fpr128 }
-  - { id: 7, class: fpr128 }
-  - { id: 8, class: fpr128 }
-  - { id: 9, class: fpr64 }
-  - { id: 10, class: fpr128 }
-  - { id: 11, class: fpr64 }
-  - { id: 12, class: fpr128 }
-liveins:
-  - { reg: '$q0', virtual-reg: '%0' }
-  - { reg: '$q1', virtual-reg: '%1' }
-  - { reg: '$q2', virtual-reg: '%2' }
-frameInfo:
-  maxAlignment:    1
-  maxCallFrameSize: 0
-machineFunctionInfo: {}
-body:             |
-  bb.0.entry:
-    liveins: $q0, $q1, $q2
-  
-    %2:fpr128 = COPY $q2
-    %1:fpr128 = COPY $q1
-    %0:fpr128 = COPY $q0
-    %3:fpr128 = EXTv16i8 %0, %0, 8
-    %6:fpr128 = MOVIv2d_ns 17
-    %7:fpr128 = ANDv16i8 %2, %6
-    %8:fpr128 = ANDv16i8 %1, %6
-    %9:fpr64 = XTNv4i16 %8
-    %10:fpr128 = UMULLv4i16_v4i32 %0.dsub, %9
-    %11:fpr64 = XTNv4i16 %7
-    %12:fpr128 = UMULLv4i16_v4i32 %3.dsub, %11
-    $q0 = COPY %10
-    $q1 = COPY %12
-    RET_ReallyLR implicit $q0, implicit $q1
-
-...
-
-# CHECK-LABEL: Before MISched:
-# CHECK-NEXT: # Machine code for function umull_and_v8i32: IsSSA, NoPHIs, TracksLiveness
-# CHECK-NEXT: Function Live Ins: $q0 in %0, $q1 in %1, $q2 in %2
-# CHECK-EMPTY: 
-# CHECK-NEXT: bb.0.entry:
-# CHECK-NEXT:   liveins: $q0, $q1, $q2
-# CHECK-NEXT:   %2:fpr128 = COPY $q2
-# CHECK-NEXT:   %1:fpr128 = COPY $q1
-# CHECK-NEXT:   %0:fpr128 = COPY $q0
-# CHECK-NEXT:   %3:fpr128 = EXTv16i8 %0:fpr128, %0:fpr128, 8
-# CHECK-NEXT:   %6:fpr128 = MOVIv2d_ns 17
-# CHECK-NEXT:   %7:fpr128 = ANDv16i8 %2:fpr128, %6:fpr128
-# CHECK-NEXT:   %8:fpr128 = ANDv16i8 %1:fpr128, %6:fpr128
-# CHECK-NEXT:   %9:fpr64 = XTNv4i16 %8:fpr128
-# CHECK-NEXT:   %10:fpr128 = UMULLv4i16_v4i32 %0.dsub:fpr128, %9:fpr64
-# CHECK-NEXT:   %11:fpr64 = XTNv4i16 %7:fpr128
-# CHECK-NEXT:   %12:fpr128 = UMULLv4i16_v4i32 %3.dsub:fpr128, %11:fpr64
-# CHECK-NEXT:   $q0 = COPY %10:fpr128
-# CHECK-NEXT:   $q1 = COPY %12:fpr128
-# CHECK-NEXT:   RET_ReallyLR implicit $q0, implicit $q1
-# CHECK-EMPTY: 
-# CHECK-NEXT: # End machine code for function umull_and_v8i32.
-# CHECK-EMPTY: 
-# CHECK-NEXT: ********** MI Scheduling **********
-# CHECK-NEXT: umull_and_v8i32:%bb.0 entry
-# CHECK-NEXT:   From: %2:fpr128 = COPY $q2
-# CHECK-NEXT:     To: RET_ReallyLR implicit $q0, implicit $q1
-# CHECK-NEXT:  RegionInstrs: 13
-# CHECK-NEXT: ScheduleDAGMILive::schedule starting
-# CHECK-NEXT: GenericScheduler RegionPolicy:  ShouldTrackPressure=0 OnlyTopDown=0 OnlyBottomUp=1
-# CHECK-NEXT: Disabled scoreboard hazard recognizer
-# CHECK-NEXT: Disabled scoreboard hazard recognizer
-# CHECK-NEXT: SU(0) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %2:fpr128 = COPY $q2
-# CHECK-NEXT:   # preds left       : 0
-# CHECK-NEXT:   # succs left       : 1
-# CHECK-NEXT:   # rdefs left       : 0
-# CHECK-NEXT:   Latency            : 3
-# CHECK-NEXT:   Depth              : 0
-# CHECK-NEXT:   Height             : 13
-# CHECK-NEXT:   Successors:
-# CHECK-NEXT:     SU(5): Data Latency=3 Reg=%2
-# CHECK-NEXT:   Single Issue       : false;
-# CHECK-NEXT: SU(1) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %1:fpr128 = COPY $q1
-# CHECK-NEXT:   # preds left       : 0
-# CHECK-NEXT:   # succs left       : 2
-# CHECK-NEXT:   # rdefs left       : 0
-# CHECK-NEXT:   Latency            : 3
-# CHECK-NEXT:   Depth              : 0
-# CHECK-NEXT:   Height             : 13
-# CHECK-NEXT:   Successors:
-# CHECK-NEXT:     SU(6): Data Latency=3 Reg=%1
-# CHECK-NEXT:     SU(12): Anti Latency=0
-# CHECK-NEXT:   Single Issue       : false;
-# CHECK-NEXT: SU(2) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %0:fpr128 = COPY $q0
-# CHECK-NEXT:   # preds left       : 0
-# CHECK-NEXT:   # succs left       : 3
-# CHECK-NEXT:   # rdefs left       : 0
-# CHECK-NEXT:   Latency            : 3
-# CHECK-NEXT:   Depth              : 0
-# CHECK-NEXT:   Height             : 14
-# CHECK-NEXT:   Successors:
-# CHECK-NEXT:     SU(8): Data Latency=3 Reg=%0
-# CHECK-NEXT:     SU(3): Data Latency=3 Reg=%0
-# CHECK-NEXT:     SU(11): Anti Latency=0
-# CHECK-NEXT:   Single Issue       : false;
-# CHECK-NEXT: SU(3) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %3:fpr128 = EXTv16i8 %0:fpr128, %0:fpr128, 8
-# CHECK-NEXT:   # preds left       : 1
-# CHECK-NEXT:   # succs left       : 1
-# CHECK-NEXT:   # rdefs left       : 0
-# CHECK-NEXT:   Latency            : 4
-# CHECK-NEXT:   Depth              : 3
-# CHECK-NEXT:   Height             : 11
-# CHECK-NEXT:   Predecessors:
-# CHECK-NEXT:     SU(2): Data Latency=3 Reg=%0
-# CHECK-NEXT:   Successors:
-# CHECK-NEXT:     SU(10): Data Latency=4 Reg=%3
-# CHECK-NEXT:   Single Issue       : false;
-# CHECK-NEXT: SU(4) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %6:fpr128 = MOVIv2d_ns 17
-# CHECK-NEXT:   # preds left       : 0
-# CHECK-NEXT:   # succs left       : 2
-# CHECK-NEXT:   # rdefs left       : 0
-# CHECK-NEXT:   Latency            : 4
-# CHECK-NEXT:   Depth              : 0
-# CHECK-NEXT:   Height             : 14
-# CHECK-NEXT:   Successors:
-# CHECK-NEXT:     SU(6): Data Latency=4 Reg=%6
-# CHECK-NEXT:     SU(5): Data Latency=4 Reg=%6
-# CHECK-NEXT:   Single Issue       : false;
-# CHECK-NEXT: SU(5) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %7:fpr128 = ANDv16i8 %2:fpr128, %6:fpr128
-# CHECK-NEXT:   # preds left       : 2
-# CHECK-NEXT:   # succs left       : 1
-# CHECK-NEXT:   # rdefs left       : 0
-# CHECK-NEXT:   Latency            : 1
-# CHECK-NEXT:   Depth              : 4
-# CHECK-NEXT:   Height             : 10
-# CHECK-NEXT:   Predecessors:
-# CHECK-NEXT:     SU(4): Data Latency=4 Reg=%6
-# CHECK-NEXT:     SU(0): Data Latency=3 Reg=%2
-# CHECK-NEXT:   Successors:
-# CHECK-NEXT:     SU(9): Data Latency=1 Reg=%7
-# CHECK-NEXT:   Single Issue       : false;
-# CHECK-NEXT: SU(6) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %8:fpr128 = ANDv16i8 %1:fpr128, %6:fpr128
-# CHECK-NEXT:   # preds left       : 2
-# CHECK-NEXT:   # succs left       : 1
-# CHECK-NEXT:   # rdefs left       : 0
-# CHECK-NEXT:   Latency            : 1
-# CHECK-NEXT:   Depth              : 4
-# CHECK-NEXT:   Height             : 10
-# CHECK-NEXT:   Predecessors:
-# CHECK-NEXT:     SU(4): Data Latency=4 Reg=%6
-# CHECK-NEXT:     SU(1): Data Latency=3 Reg=%1
-# CHECK-NEXT:   Successors:
-# CHECK-NEXT:     SU(7): Data Latency=1 Reg=%8
-# CHECK-NEXT:   Single Issue       : false;
-# CHECK-NEXT: SU(7) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %9:fpr64 = XTNv4i16 %8:fpr128
-# CHECK-NEXT:   # preds left       : 1
-# CHECK-NEXT:   # succs left       : 1
-# CHECK-NEXT:   # rdefs left       : 0
-# CHECK-NEXT:   Latency            : 2
-# CHECK-NEXT:   Depth              : 5
-# CHECK-NEXT:   Height             : 9
-# CHECK-NEXT:   Predecessors:
-# CHECK-NEXT:     SU(6): Data Latency=1 Reg=%8
-# CHECK-NEXT:   Successors:
-# CHECK-NEXT:     SU(8): Data Latency=2 Reg=%9
-# CHECK-NEXT:   Single Issue       : false;
-# CHECK-NEXT: SU(8) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %10:fpr128 = UMULLv4i16_v4i32 %0.dsub:fpr128, %9:fpr64
-# CHECK-NEXT:   # preds left       : 2
-# CHECK-NEXT:   # succs left       : 1
-# CHECK-NEXT:   # rdefs left       : 0
-# CHECK-NEXT:   Latency            : 4
-# CHECK-NEXT:   Depth              : 7
-# CHECK-NEXT:   Height             : 7
-# CHECK-NEXT:   Predecessors:
-# CHECK-NEXT:     SU(7): Data Latency=2 Reg=%9
-# CHECK-NEXT:     SU(2): Data Latency=3 Reg=%0
-# CHECK-NEXT:   Successors:
-# CHECK-NEXT:     SU(11): Data Latency=4 Reg=%10
-# CHECK-NEXT:   Single Issue       : false;
-# CHECK-NEXT: SU(9) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %11:fpr64 = XTNv4i16 %7:fpr128
-# CHECK-NEXT:   # preds left       : 1
-# CHECK-NEXT:   # succs left       : 1
-# CHECK-NEXT:   # rdefs left       : 0
-# CHECK-NEXT:   Latency            : 2
-# CHECK-NEXT:   Depth              : 5
-# CHECK-NEXT:   Height             : 9
-# CHECK-NEXT:   Predecessors:
-# CHECK-NEXT:     SU(5): Data Latency=1 Reg=%7
-# CHECK-NEXT:   Successors:
-# CHECK-NEXT:     SU(10): Data Latency=2 Reg=%11
-# CHECK-NEXT:   Single Issue       : false;
-# CHECK-NEXT: SU(10) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %12:fpr128 = UMULLv4i16_v4i32 %3.dsub:fpr128, %11:fpr64
-# CHECK-NEXT:   # preds left       : 2
-# CHECK-NEXT:   # succs left       : 1
-# CHECK-NEXT:   # rdefs left       : 0
-# CHECK-NEXT:   Latency            : 4
-# CHECK-NEXT:   Depth              : 7
-# CHECK-NEXT:   Height             : 7
-# CHECK-NEXT:   Predecessors:
-# CHECK-NEXT:     SU(9): Data Latency=2 Reg=%11
-# CHECK-NEXT:     SU(3): Data Latency=4 Reg=%3
-# CHECK-NEXT:   Successors:
-# CHECK-NEXT:     SU(12): Data Latency=4 Reg=%12
-# CHECK-NEXT:   Single Issue       : false;
-# CHECK-NEXT: SU(11) [TopReadyCycle = 0, BottomReadyCycle = 0]:   $q0 = COPY %10:fpr128
-# CHECK-NEXT:   # preds left       : 2
-# CHECK-NEXT:   # succs left       : 1
-# CHECK-NEXT:   # rdefs left       : 0
-# CHECK-NEXT:   Latency            : 3
-# CHECK-NEXT:   Depth              : 11
-# CHECK-NEXT:   Height             : 3
-# CHECK-NEXT:   Predecessors:
-# CHECK-NEXT:     SU(8): Data Latency=4 Reg=%10
-# CHECK-NEXT:     SU(2): Anti Latency=0
-# CHECK-NEXT:   Successors:
-# CHECK-NEXT:     ExitSU: Ord  Latency=3 Artificial
-# CHECK-NEXT:   Single Issue       : false;
-# CHECK-NEXT: SU(12) [TopReadyCycle = 0, BottomReadyCycle = 0]:   $q1 = COPY %12:fpr128
-# CHECK-NEXT:   # preds left       : 2
-# CHECK-NEXT:   # succs left       : 1
-# CHECK-NEXT:   # rdefs left       : 0
-# CHECK-NEXT:   Latency            : 3
-# CHECK-NEXT:   Depth              : 11
-# CHECK-NEXT:   Height             : 3
-# CHECK-NEXT:   Predecessors:
-# CHECK-NEXT:     SU(10): Data Latency=4 Reg=%12
-# CHECK-NEXT:     SU(1): Anti Latency=0
-# CHECK-NEXT:   Successors:
-# CHECK-NEXT:     ExitSU: Ord  Latency=3 Artificial
-# CHECK-NEXT:   Single Issue       : false;
-# CHECK-NEXT: ExitSU [TopReadyCycle = 0, BottomReadyCycle = 0]:   RET_ReallyLR implicit $q0, implicit $q1
-# CHECK-NEXT:   # preds left       : 2
-# CHECK-NEXT:   # succs left       : 0
-# CHECK-NEXT:   # rdefs left       : 0
-# CHECK-NEXT:   Latency            : 0
-# CHECK-NEXT:   Depth              : 14
-# CHECK-NEXT:   Height             : 0
-# CHECK-NEXT:   Predecessors:
-# CHECK-NEXT:     SU(12): Ord  Latency=3 Artificial
-# CHECK-NEXT:     SU(11): Ord  Latency=3 Artificial
-# CHECK-NEXT:   Resource booking (@0c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@0c): 
-# CHECK-NEXT:     Instance 0 available @0c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
-# CHECK-NEXT:   Resource booking (@0c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@0c): 
-# CHECK-NEXT:     Instance 0 available @0c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
-# CHECK-NEXT:   Resource booking (@0c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@0c): 
-# CHECK-NEXT:     Instance 0 available @0c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
-# CHECK-NEXT:   Resource booking (@0c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@0c): 
-# CHECK-NEXT:     Instance 0 available @0c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @0c
-# CHECK-NEXT: Critical Path(GS-RR ): 14
-# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
-# CHECK-NEXT: Cycle: 3 BotQ.A
-# CHECK-NEXT:   Resource booking (@3c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@3c): 
-# CHECK-NEXT:     Instance 0 available @0c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
-# CHECK-NEXT:   Resource booking (@3c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@3c): 
-# CHECK-NEXT:     Instance 0 available @0c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
-# CHECK-NEXT: Queue BotQ.P: 
-# CHECK-NEXT: Queue BotQ.A: 12 11 
-# CHECK-NEXT:   Cand SU(12) ORDER                              
-# CHECK-NEXT: Pick Bot ORDER     
-# CHECK-NEXT: Scheduling SU(12) $q1 = COPY %12:fpr128
-# CHECK-NEXT:   Ready @3c
-# CHECK-NEXT:   CortexA55UnitALU +1x1u
-# CHECK-NEXT:   Resource booking (@3c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@3c): 
-# CHECK-NEXT:     Instance 0 available @0c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
-# CHECK-NEXT:   Resource booking (@3c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@3c): 
-# CHECK-NEXT:     Instance 0 available @0c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
-# CHECK-NEXT:   BotQ.A TopLatency SU(12) 11c
-# CHECK-NEXT:   BotQ.A BotLatency SU(12) 3c
-# CHECK-NEXT: BotQ.A @3c
-# CHECK-NEXT:   Retired: 1
-# CHECK-NEXT:   Executed: 3c
-# CHECK-NEXT:   Critical: 0c, 1 MOps
-# CHECK-NEXT:   ExpectedLatency: 3c
-# CHECK-NEXT:   - Latency limited.
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
-# CHECK-NEXT:   Resource booking (@3c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@3c): 
-# CHECK-NEXT:     Instance 0 available @4c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @0c
-# CHECK-NEXT: Queue BotQ.P: 10 
-# CHECK-NEXT: Queue BotQ.A: 11 
-# CHECK-NEXT: Scheduling SU(11) $q0 = COPY %10:fpr128
-# CHECK-NEXT:   Ready @3c
-# CHECK-NEXT:   CortexA55UnitALU +1x1u
-# CHECK-NEXT:   Resource booking (@3c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@3c): 
-# CHECK-NEXT:     Instance 0 available @4c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @0c
-# CHECK-NEXT:   Resource booking (@3c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@3c): 
-# CHECK-NEXT:     Instance 0 available @3c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @0c
-# CHECK-NEXT:   *** Max MOps 2 at cycle 3
-# CHECK-NEXT: Cycle: 4 BotQ.A
-# CHECK-NEXT: BotQ.A @4c
-# CHECK-NEXT:   Retired: 2
-# CHECK-NEXT:   Executed: 4c
-# CHECK-NEXT:   Critical: 1c, 2 MOps
-# CHECK-NEXT:   ExpectedLatency: 3c
-# CHECK-NEXT:   - Latency limited.
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
-# CHECK-NEXT: Cycle: 7 BotQ.A
-# CHECK-NEXT:   Resource booking (@7c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@7c): 
-# CHECK-NEXT:     Instance 0 available @0c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @0c
-# CHECK-NEXT:   Resource booking (@7c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@7c): 
-# CHECK-NEXT:     Instance 0 available @0c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @0c
-# CHECK-NEXT: Queue BotQ.P: 
-# CHECK-NEXT: Queue BotQ.A: 10 8 
-# CHECK-NEXT:   Cand SU(10) ORDER                              
-# CHECK-NEXT: Pick Bot ORDER     
-# CHECK-NEXT: Scheduling SU(10) %12:fpr128 = UMULLv4i16_v4i32 %3.dsub:fpr128, %11:fpr64
-# CHECK-NEXT:   Ready @7c
-# CHECK-NEXT:   CortexA55UnitFPALU +2x1u
-# CHECK-NEXT:   Resource booking (@7c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@7c): 
-# CHECK-NEXT:     Instance 0 available @0c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @0c
-# CHECK-NEXT:   Resource booking (@7c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@7c): 
-# CHECK-NEXT:     Instance 0 available @0c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @0c
-# CHECK-NEXT:   BotQ.A BotLatency SU(10) 7c
-# CHECK-NEXT:   Bump cycle to begin group
-# CHECK-NEXT: Cycle: 8 BotQ.A
-# CHECK-NEXT: BotQ.A @8c
-# CHECK-NEXT:   Retired: 3
-# CHECK-NEXT:   Executed: 8c
-# CHECK-NEXT:   Critical: 1c, 3 MOps
-# CHECK-NEXT:   ExpectedLatency: 7c
-# CHECK-NEXT:   - Latency limited.
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
-# CHECK-NEXT:   Resource booking (@8c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@8c): 
-# CHECK-NEXT:     Instance 0 available @9c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @0c
-# CHECK-NEXT: Queue BotQ.P: 9 3 
-# CHECK-NEXT: Queue BotQ.A: 8 
-# CHECK-NEXT: Scheduling SU(8) %10:fpr128 = UMULLv4i16_v4i32 %0.dsub:fpr128, %9:fpr64
-# CHECK-NEXT:   Ready @8c
-# CHECK-NEXT:   CortexA55UnitFPALU +2x1u
-# CHECK-NEXT:   Resource booking (@8c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@8c): 
-# CHECK-NEXT:     Instance 0 available @9c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @0c
-# CHECK-NEXT:   Resource booking (@8c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@8c): 
-# CHECK-NEXT:     Instance 0 available @7c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @0c
-# CHECK-NEXT:   BotQ.A TopLatency SU(8) 7c
-# CHECK-NEXT:   Bump cycle to begin group
-# CHECK-NEXT: Cycle: 9 BotQ.A
-# CHECK-NEXT: BotQ.A @9c
-# CHECK-NEXT:   Retired: 4
-# CHECK-NEXT:   Executed: 9c
-# CHECK-NEXT:   Critical: 2c, 4 MOps
-# CHECK-NEXT:   ExpectedLatency: 7c
-# CHECK-NEXT:   - Latency limited.
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
-# CHECK-NEXT:   Resource booking (@9c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@9c): 
-# CHECK-NEXT:     Instance 0 available @8c
-# CHECK-NEXT:     Instance 1 available @9c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @8c
-# CHECK-NEXT:   Resource booking (@9c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@9c): 
-# CHECK-NEXT:     Instance 0 available @8c
-# CHECK-NEXT:     Instance 1 available @9c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @8c
-# CHECK-NEXT: Queue BotQ.P: 7 3 
-# CHECK-NEXT: Queue BotQ.A: 9 
-# CHECK-NEXT: Scheduling SU(9) %11:fpr64 = XTNv4i16 %7:fpr128
-# CHECK-NEXT:   Ready @9c
-# CHECK-NEXT:   CortexA55UnitFPALU +1x1u
-# CHECK-NEXT:   Resource booking (@9c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@9c): 
-# CHECK-NEXT:     Instance 0 available @8c
-# CHECK-NEXT:     Instance 1 available @9c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @8c
-# CHECK-NEXT:   Resource booking (@9c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@9c): 
-# CHECK-NEXT:     Instance 0 available @7c
-# CHECK-NEXT:     Instance 1 available @8c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @7c
-# CHECK-NEXT:   BotQ.A BotLatency SU(9) 9c
-# CHECK-NEXT: BotQ.A @9c
-# CHECK-NEXT:   Retired: 5
-# CHECK-NEXT:   Executed: 9c
-# CHECK-NEXT:   Critical: 2c, 5 MOps
-# CHECK-NEXT:   ExpectedLatency: 9c
-# CHECK-NEXT:   - Latency limited.
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
-# CHECK-NEXT: Cycle: 10 BotQ.A
-# CHECK-NEXT:   Resource booking (@10c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@10c): 
-# CHECK-NEXT:     Instance 0 available @10c
-# CHECK-NEXT:     Instance 1 available @9c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @9c
-# CHECK-NEXT:   Resource booking (@10c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@10c): 
-# CHECK-NEXT:     Instance 0 available @11c
-# CHECK-NEXT:     Instance 1 available @10c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @10c
-# CHECK-NEXT: Queue BotQ.P: 3 
-# CHECK-NEXT: Queue BotQ.A: 7 5 
-# CHECK-NEXT:   Cand SU(7) ORDER                              
-# CHECK-NEXT: Pick Bot ORDER     
-# CHECK-NEXT: Scheduling SU(7) %9:fpr64 = XTNv4i16 %8:fpr128
-# CHECK-NEXT:   Ready @10c
-# CHECK-NEXT:   CortexA55UnitFPALU +1x1u
-# CHECK-NEXT:   Resource booking (@10c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@10c): 
-# CHECK-NEXT:     Instance 0 available @10c
-# CHECK-NEXT:     Instance 1 available @9c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @9c
-# CHECK-NEXT:   Resource booking (@10c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@10c): 
-# CHECK-NEXT:     Instance 0 available @9c
-# CHECK-NEXT:     Instance 1 available @8c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @8c
-# CHECK-NEXT: BotQ.A @10c
-# CHECK-NEXT:   Retired: 6
-# CHECK-NEXT:   Executed: 10c
-# CHECK-NEXT:   Critical: 3c, 6 MOps
-# CHECK-NEXT:   ExpectedLatency: 9c
-# CHECK-NEXT:   - Latency limited.
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
-# CHECK-NEXT:   Resource booking (@10c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@10c): 
-# CHECK-NEXT:     Instance 0 available @11c
-# CHECK-NEXT:     Instance 1 available @12c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @11c
-# CHECK-NEXT:   SU(5) CortexA55UnitFPALU[0]=11c
-# CHECK-NEXT: Cycle: 11 BotQ.A
-# CHECK-NEXT:   Resource booking (@11c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@11c): 
-# CHECK-NEXT:     Instance 0 available @11c
-# CHECK-NEXT:     Instance 1 available @12c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @11c
-# CHECK-NEXT:   Resource booking (@11c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@11c): 
-# CHECK-NEXT:     Instance 0 available @11c
-# CHECK-NEXT:     Instance 1 available @12c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @11c
-# CHECK-NEXT:   Resource booking (@11c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@11c): 
-# CHECK-NEXT:     Instance 0 available @11c
-# CHECK-NEXT:     Instance 1 available @12c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @11c
-# CHECK-NEXT: Queue BotQ.P: 
-# CHECK-NEXT: Queue BotQ.A: 3 5 6 
-# CHECK-NEXT:   Cand SU(3) ORDER                              
-# CHECK-NEXT:   Cand SU(5) ORDER                              
-# CHECK-NEXT:   Cand SU(6) ORDER                              
-# CHECK-NEXT: Pick Bot ORDER     
-# CHECK-NEXT: Scheduling SU(6) %8:fpr128 = ANDv16i8 %1:fpr128, %6:fpr128
-# CHECK-NEXT:   Ready @11c
-# CHECK-NEXT:   CortexA55UnitFPALU +2x1u
-# CHECK-NEXT:   *** Critical resource CortexA55UnitFPALU: 4c
-# CHECK-NEXT:   Resource booking (@11c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@11c): 
-# CHECK-NEXT:     Instance 0 available @11c
-# CHECK-NEXT:     Instance 1 available @12c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @11c
-# CHECK-NEXT:   Resource booking (@11c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@11c): 
-# CHECK-NEXT:     Instance 0 available @9c
-# CHECK-NEXT:     Instance 1 available @10c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @9c
-# CHECK-NEXT:   BotQ.A BotLatency SU(6) 10c
-# CHECK-NEXT:   Bump cycle to begin group
-# CHECK-NEXT: Cycle: 12 BotQ.A
-# CHECK-NEXT: BotQ.A @12c
-# CHECK-NEXT:   Retired: 7
-# CHECK-NEXT:   Executed: 12c
-# CHECK-NEXT:   Critical: 4c, 8 CortexA55UnitFPALU
-# CHECK-NEXT:   ExpectedLatency: 10c
-# CHECK-NEXT:   - Latency limited.
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
-# CHECK-NEXT:   Resource booking (@12c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@12c): 
-# CHECK-NEXT:     Instance 0 available @13c
-# CHECK-NEXT:     Instance 1 available @12c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @12c
-# CHECK-NEXT:   Resource booking (@12c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@12c): 
-# CHECK-NEXT:     Instance 0 available @13c
-# CHECK-NEXT:     Instance 1 available @12c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @12c
-# CHECK-NEXT: Queue BotQ.P: 1 
-# CHECK-NEXT: Queue BotQ.A: 3 5 
-# CHECK-NEXT:   Cand SU(3) ORDER                              
-# CHECK-NEXT:   Cand SU(5) ORDER                              
-# CHECK-NEXT: Pick Bot ORDER     
-# CHECK-NEXT: Scheduling SU(5) %7:fpr128 = ANDv16i8 %2:fpr128, %6:fpr128
-# CHECK-NEXT:   Ready @12c
-# CHECK-NEXT:   CortexA55UnitFPALU +2x1u
-# CHECK-NEXT:   Resource booking (@12c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@12c): 
-# CHECK-NEXT:     Instance 0 available @13c
-# CHECK-NEXT:     Instance 1 available @12c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @12c
-# CHECK-NEXT:   Resource booking (@12c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@12c): 
-# CHECK-NEXT:     Instance 0 available @11c
-# CHECK-NEXT:     Instance 1 available @10c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @10c
-# CHECK-NEXT:   BotQ.A TopLatency SU(5) 4c
-# CHECK-NEXT:   Bump cycle to begin group
-# CHECK-NEXT: Cycle: 13 BotQ.A
-# CHECK-NEXT: BotQ.A @13c
-# CHECK-NEXT:   Retired: 8
-# CHECK-NEXT:   Executed: 13c
-# CHECK-NEXT:   Critical: 5c, 10 CortexA55UnitFPALU
-# CHECK-NEXT:   ExpectedLatency: 10c
-# CHECK-NEXT:   - Latency limited.
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
-# CHECK-NEXT:   Resource booking (@13c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@13c): 
-# CHECK-NEXT:     Instance 0 available @13c
-# CHECK-NEXT:     Instance 1 available @14c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @13c
-# CHECK-NEXT: Queue BotQ.P: 1 4 0 
-# CHECK-NEXT: Queue BotQ.A: 3 
-# CHECK-NEXT: Scheduling SU(3) %3:fpr128 = EXTv16i8 %0:fpr128, %0:fpr128, 8
-# CHECK-NEXT:   Ready @13c
-# CHECK-NEXT:   CortexA55UnitFPALU +2x1u
-# CHECK-NEXT:   Resource booking (@13c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@13c): 
-# CHECK-NEXT:     Instance 0 available @13c
-# CHECK-NEXT:     Instance 1 available @14c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @13c
-# CHECK-NEXT:   Resource booking (@13c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@13c): 
-# CHECK-NEXT:     Instance 0 available @11c
-# CHECK-NEXT:     Instance 1 available @12c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @11c
-# CHECK-NEXT:   BotQ.A BotLatency SU(3) 11c
-# CHECK-NEXT:   Bump cycle to begin group
-# CHECK-NEXT: Cycle: 14 BotQ.A
-# CHECK-NEXT: BotQ.A @14c
-# CHECK-NEXT:   Retired: 9
-# CHECK-NEXT:   Executed: 14c
-# CHECK-NEXT:   Critical: 6c, 12 CortexA55UnitFPALU
-# CHECK-NEXT:   ExpectedLatency: 11c
-# CHECK-NEXT:   - Latency limited.
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
-# CHECK-NEXT:   Resource booking (@14c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@14c): 
-# CHECK-NEXT:     Instance 0 available @4c
-# CHECK-NEXT:     Instance 1 available @4c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @4c
-# CHECK-NEXT:   Resource booking (@14c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@14c): 
-# CHECK-NEXT:     Instance 0 available @4c
-# CHECK-NEXT:     Instance 1 available @4c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @4c
-# CHECK-NEXT: Queue BotQ.P: 2 4 0 
-# CHECK-NEXT: Queue BotQ.A: 1 
-# CHECK-NEXT: Scheduling SU(1) %1:fpr128 = COPY $q1
-# CHECK-NEXT:   Ready @14c
-# CHECK-NEXT:   CortexA55UnitALU +1x1u
-# CHECK-NEXT:   Resource booking (@14c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@14c): 
-# CHECK-NEXT:     Instance 0 available @4c
-# CHECK-NEXT:     Instance 1 available @4c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @4c
-# CHECK-NEXT:   Resource booking (@14c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 3
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@14c): 
-# CHECK-NEXT:     Instance 0 available @3c
-# CHECK-NEXT:     Instance 1 available @3c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @3c
-# CHECK-NEXT:   BotQ.A BotLatency SU(1) 13c
-# CHECK-NEXT: BotQ.A @14c
-# CHECK-NEXT:   Retired: 10
-# CHECK-NEXT:   Executed: 14c
-# CHECK-NEXT:   Critical: 6c, 12 CortexA55UnitFPALU
-# CHECK-NEXT:   ExpectedLatency: 13c
-# CHECK-NEXT:   - Latency limited.
-# CHECK-NEXT: CortexA55UnitALU(0) = 14
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
-# CHECK-NEXT: Cycle: 15 BotQ.A
-# CHECK-NEXT:   Resource booking (@15c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 14
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@15c): 
-# CHECK-NEXT:     Instance 0 available @15c
-# CHECK-NEXT:     Instance 1 available @4c
-# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @4c
-# CHECK-NEXT: Queue BotQ.P: 2 4 
-# CHECK-NEXT: Queue BotQ.A: 0 
-# CHECK-NEXT: Scheduling SU(0) %2:fpr128 = COPY $q2
-# CHECK-NEXT:   Ready @15c
-# CHECK-NEXT:   CortexA55UnitALU +1x1u
-# CHECK-NEXT:   Resource booking (@15c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 14
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@15c): 
-# CHECK-NEXT:     Instance 0 available @15c
-# CHECK-NEXT:     Instance 1 available @4c
-# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @4c
-# CHECK-NEXT:   Resource booking (@15c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 14
-# CHECK-NEXT: CortexA55UnitALU(1) = 3
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@15c): 
-# CHECK-NEXT:     Instance 0 available @14c
-# CHECK-NEXT:     Instance 1 available @3c
-# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @3c
-# CHECK-NEXT: BotQ.A @15c
-# CHECK-NEXT:   Retired: 11
-# CHECK-NEXT:   Executed: 15c
-# CHECK-NEXT:   Critical: 6c, 12 CortexA55UnitFPALU
-# CHECK-NEXT:   ExpectedLatency: 13c
-# CHECK-NEXT:   - Latency limited.
-# CHECK-NEXT: CortexA55UnitALU(0) = 14
-# CHECK-NEXT: CortexA55UnitALU(1) = 15
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
-# CHECK-NEXT: Cycle: 16 BotQ.A
-# CHECK-NEXT:   Resource booking (@16c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 14
-# CHECK-NEXT: CortexA55UnitALU(1) = 15
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@16c): 
-# CHECK-NEXT:     Instance 0 available @15c
-# CHECK-NEXT:     Instance 1 available @16c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @15c
-# CHECK-NEXT:   Resource booking (@16c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 14
-# CHECK-NEXT: CortexA55UnitALU(1) = 15
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@16c): 
-# CHECK-NEXT:     Instance 0 available @15c
-# CHECK-NEXT:     Instance 1 available @14c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @14c
-# CHECK-NEXT: Queue BotQ.P: 
-# CHECK-NEXT: Queue BotQ.A: 2 4 
-# CHECK-NEXT:   Cand SU(2) ORDER                              
-# CHECK-NEXT:   Cand SU(4) PHYS-REG                           
-# CHECK-NEXT: Pick Bot PHYS-REG  
-# CHECK-NEXT: Scheduling SU(4) %6:fpr128 = MOVIv2d_ns 17
-# CHECK-NEXT:   Ready @16c
-# CHECK-NEXT:   CortexA55UnitFPALU +2x1u
-# CHECK-NEXT:   Resource booking (@16c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 14
-# CHECK-NEXT: CortexA55UnitALU(1) = 15
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@16c): 
-# CHECK-NEXT:     Instance 0 available @15c
-# CHECK-NEXT:     Instance 1 available @14c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @14c
-# CHECK-NEXT:   Resource booking (@16c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 14
-# CHECK-NEXT: CortexA55UnitALU(1) = 15
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@16c): 
-# CHECK-NEXT:     Instance 0 available @13c
-# CHECK-NEXT:     Instance 1 available @12c
-# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @12c
-# CHECK-NEXT:   BotQ.A BotLatency SU(4) 14c
-# CHECK-NEXT:   Bump cycle to begin group
-# CHECK-NEXT: Cycle: 17 BotQ.A
-# CHECK-NEXT: BotQ.A @17c
-# CHECK-NEXT:   Retired: 12
-# CHECK-NEXT:   Executed: 17c
-# CHECK-NEXT:   Critical: 7c, 14 CortexA55UnitFPALU
-# CHECK-NEXT:   ExpectedLatency: 14c
-# CHECK-NEXT:   - Latency limited.
-# CHECK-NEXT: CortexA55UnitALU(0) = 14
-# CHECK-NEXT: CortexA55UnitALU(1) = 15
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 16
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
-# CHECK-NEXT:   Resource booking (@17c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 14
-# CHECK-NEXT: CortexA55UnitALU(1) = 15
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 16
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@17c): 
-# CHECK-NEXT:     Instance 0 available @15c
-# CHECK-NEXT:     Instance 1 available @16c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @15c
-# CHECK-NEXT: Queue BotQ.P: 
-# CHECK-NEXT: Queue BotQ.A: 2 
-# CHECK-NEXT: Scheduling SU(2) %0:fpr128 = COPY $q0
-# CHECK-NEXT:   Ready @17c
-# CHECK-NEXT:   CortexA55UnitALU +1x1u
-# CHECK-NEXT:   Resource booking (@17c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 14
-# CHECK-NEXT: CortexA55UnitALU(1) = 15
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 16
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@17c): 
-# CHECK-NEXT:     Instance 0 available @15c
-# CHECK-NEXT:     Instance 1 available @16c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @15c
-# CHECK-NEXT:   Resource booking (@17c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 14
-# CHECK-NEXT: CortexA55UnitALU(1) = 15
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 16
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@17c): 
-# CHECK-NEXT:     Instance 0 available @14c
-# CHECK-NEXT:     Instance 1 available @15c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @14c
-# CHECK-NEXT: BotQ.A @17c
-# CHECK-NEXT:   Retired: 13
-# CHECK-NEXT:   Executed: 17c
-# CHECK-NEXT:   Critical: 7c, 14 CortexA55UnitFPALU
-# CHECK-NEXT:   ExpectedLatency: 14c
-# CHECK-NEXT:   - Latency limited.
-# CHECK-NEXT: CortexA55UnitALU(0) = 17
-# CHECK-NEXT: CortexA55UnitALU(1) = 15
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 16
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
-# CHECK-NEXT: *** Final schedule for %bb.0 ***
-# CHECK-NEXT:  * Schedule table (BottomUp):
-# CHECK-NEXT:   i: issue
-# CHECK-NEXT:   x: resource booked
-# CHECK-NEXT: Cycle              | 17 | 16 | 15 | 14 | 13 | 12 | 11 | 10 | 9  | 8  | 7  | 6  | 5  | 4  | 3  |
-# CHECK-NEXT: SU(2)              | i  |    |    |    |    |    |    |    |    |    |    |    |    |    |    |
-# CHECK-NEXT:   CortexA55UnitALU | x  |    |    |    |    |    |    |    |    |    |    |    |    |    |    | 
-# CHECK-NEXT: SU(4)              |    | i  |    |    |    |    |    |    |    |    |    |    |    |    |    |
-# CHECK-NEXT: CortexA55UnitFPALU |    | x  | x  |    |    |    |    |    |    |    |    |    |    |    |    | 
-# CHECK-NEXT: SU(0)              |    |    | i  |    |    |    |    |    |    |    |    |    |    |    |    |
-# CHECK-NEXT:   CortexA55UnitALU |    |    | x  |    |    |    |    |    |    |    |    |    |    |    |    | 
-# CHECK-NEXT: SU(1)              |    |    |    | i  |    |    |    |    |    |    |    |    |    |    |    |
-# CHECK-NEXT:   CortexA55UnitALU |    |    |    | x  |    |    |    |    |    |    |    |    |    |    |    | 
-# CHECK-NEXT: SU(3)              |    |    |    |    | i  |    |    |    |    |    |    |    |    |    |    |
-# CHECK-NEXT: CortexA55UnitFPALU |    |    |    |    | x  | x  |    |    |    |    |    |    |    |    |    | 
-# CHECK-NEXT: SU(5)              |    |    |    |    |    | i  |    |    |    |    |    |    |    |    |    |
-# CHECK-NEXT: CortexA55UnitFPALU |    |    |    |    |    | x  | x  |    |    |    |    |    |    |    |    | 
-# CHECK-NEXT: SU(6)              |    |    |    |    |    |    | i  |    |    |    |    |    |    |    |    |
-# CHECK-NEXT: CortexA55UnitFPALU |    |    |    |    |    |    | x  | x  |    |    |    |    |    |    |    | 
-# CHECK-NEXT: SU(7)              |    |    |    |    |    |    |    | i  |    |    |    |    |    |    |    |
-# CHECK-NEXT: CortexA55UnitFPALU |    |    |    |    |    |    |    | x  |    |    |    |    |    |    |    | 
-# CHECK-NEXT: SU(9)              |    |    |    |    |    |    |    |    | i  |    |    |    |    |    |    |
-# CHECK-NEXT: CortexA55UnitFPALU |    |    |    |    |    |    |    |    | x  |    |    |    |    |    |    | 
-# CHECK-NEXT: SU(8)              |    |    |    |    |    |    |    |    |    | i  |    |    |    |    |    |
-# CHECK-NEXT: CortexA55UnitFPALU |    |    |    |    |    |    |    |    |    | x  | x  |    |    |    |    | 
-# CHECK-NEXT: SU(10)             |    |    |    |    |    |    |    |    |    |    | i  |    |    |    |    |
-# CHECK-NEXT: CortexA55UnitFPALU |    |    |    |    |    |    |    |    |    |    | x  | x  |    |    |    | 
-# CHECK-NEXT: SU(11)             |    |    |    |    |    |    |    |    |    |    |    |    |    |    | i  |
-# CHECK-NEXT:   CortexA55UnitALU |    |    |    |    |    |    |    |    |    |    |    |    |    |    | x  | 
-# CHECK-NEXT: SU(12)             |    |    |    |    |    |    |    |    |    |    |    |    |    |    | i  |
-# CHECK-NEXT:   CortexA55UnitALU |    |    |    |    |    |    |    |    |    |    |    |    |    |    | x  | 
-# CHECK-NEXT: SU(2) [TopReadyCycle = 0, BottomReadyCycle = 17]:   %0:fpr128 = COPY $q0
-# CHECK-NEXT: SU(4) [TopReadyCycle = 0, BottomReadyCycle = 16]:   %6:fpr128 = MOVIv2d_ns 17
-# CHECK-NEXT: SU(0) [TopReadyCycle = 0, BottomReadyCycle = 15]:   %2:fpr128 = COPY $q2
-# CHECK-NEXT: SU(1) [TopReadyCycle = 0, BottomReadyCycle = 14]:   %1:fpr128 = COPY $q1
-# CHECK-NEXT: SU(3) [TopReadyCycle = 0, BottomReadyCycle = 13]:   %3:fpr128 = EXTv16i8 %0:fpr128, %0:fpr128, 8
-# CHECK-NEXT: SU(5) [TopReadyCycle = 0, BottomReadyCycle = 12]:   %7:fpr128 = ANDv16i8 %2:fpr128, %6:fpr128
-# CHECK-NEXT: SU(6) [TopReadyCycle = 0, BottomReadyCycle = 11]:   %8:fpr128 = ANDv16i8 %1:fpr128, %6:fpr128
-# CHECK-NEXT: SU(7) [TopReadyCycle = 0, BottomReadyCycle = 10]:   %9:fpr64 = XTNv4i16 %8:fpr128
-# CHECK-NEXT: SU(9) [TopReadyCycle = 0, BottomReadyCycle = 9]:   %11:fpr64 = XTNv4i16 %7:fpr128
-# CHECK-NEXT: SU(8) [TopReadyCycle = 0, BottomReadyCycle = 8]:   %10:fpr128 = UMULLv4i16_v4i32 %0.dsub:fpr128, %9:fpr64
-# CHECK-NEXT: SU(10) [TopReadyCycle = 0, BottomReadyCycle = 7]:   %12:fpr128 = UMULLv4i16_v4i32 %3.dsub:fpr128, %11:fpr64
-# CHECK-NEXT: SU(11) [TopReadyCycle = 0, BottomReadyCycle = 3]:   $q0 = COPY %10:fpr128
-# CHECK-NEXT: SU(12) [TopReadyCycle = 0, BottomReadyCycle = 3]:   $q1 = COPY %12:fpr128
-# CHECK-EMPTY: 
-# CHECK-NEXT: ********** INTERVALS **********
-# CHECK-NEXT: B0 [0B,48r:0)[192r,224r:1) 0@0B-phi 1@192r
-# CHECK-NEXT: B1 [0B,96r:0)[208r,224r:1) 0@0B-phi 1@208r
-# CHECK-NEXT: B2 [0B,88r:0) 0@0B-phi
-# CHECK-NEXT: %0 [48r,168r:0) 0@48r  weight:0.000000e+00
-# CHECK-NEXT: %1 [96r,120r:0) 0@96r  weight:0.000000e+00
-# CHECK-NEXT: %2 [88r,112r:0) 0@88r  weight:0.000000e+00
-# CHECK-NEXT: %3 [104r,176r:0) 0@104r  weight:0.000000e+00
-# CHECK-NEXT: %6 [80r,120r:0) 0@80r  weight:0.000000e+00
-# CHECK-NEXT: %7 [112r,160r:0) 0@112r  weight:0.000000e+00
-# CHECK-NEXT: %8 [120r,128r:0) 0@120r  weight:0.000000e+00
-# CHECK-NEXT: %9 [128r,168r:0) 0@128r  weight:0.000000e+00
-# CHECK-NEXT: %10 [168r,192r:0) 0@168r  weight:0.000000e+00
-# CHECK-NEXT: %11 [160r,176r:0) 0@160r  weight:0.000000e+00
-# CHECK-NEXT: %12 [176r,208r:0) 0@176r  weight:0.000000e+00
-# CHECK-NEXT: RegMasks:
-# CHECK-NEXT: ********** MACHINEINSTRS **********
-# CHECK-NEXT: # Machine code for function umull_and_v8i32: IsSSA, NoPHIs, TracksLiveness
-# CHECK-NEXT: Function Live Ins: $q0 in %0, $q1 in %1, $q2 in %2
-# CHECK-EMPTY: 
-# CHECK-NEXT: 0B	bb.0.entry:
-# CHECK-NEXT: 	  liveins: $q0, $q1, $q2
-# CHECK-NEXT: 48B	  %0:fpr128 = COPY $q0
-# CHECK-NEXT: 80B	  %6:fpr128 = MOVIv2d_ns 17
-# CHECK-NEXT: 88B	  %2:fpr128 = COPY $q2
-# CHECK-NEXT: 96B	  %1:fpr128 = COPY $q1
-# CHECK-NEXT: 104B	  %3:fpr128 = EXTv16i8 %0:fpr128, %0:fpr128, 8
-# CHECK-NEXT: 112B	  %7:fpr128 = ANDv16i8 %2:fpr128, %6:fpr128
-# CHECK-NEXT: 120B	  %8:fpr128 = ANDv16i8 %1:fpr128, %6:fpr128
-# CHECK-NEXT: 128B	  %9:fpr64 = XTNv4i16 %8:fpr128
-# CHECK-NEXT: 160B	  %11:fpr64 = XTNv4i16 %7:fpr128
-# CHECK-NEXT: 168B	  %10:fpr128 = UMULLv4i16_v4i32 %0.dsub:fpr128, %9:fpr64
-# CHECK-NEXT: 176B	  %12:fpr128 = UMULLv4i16_v4i32 %3.dsub:fpr128, %11:fpr64
-# CHECK-NEXT: 192B	  $q0 = COPY %10:fpr128
-# CHECK-NEXT: 208B	  $q1 = COPY %12:fpr128
-# CHECK-NEXT: 224B	  RET_ReallyLR implicit $q0, implicit $q1
-# CHECK-EMPTY: 
-# CHECK-NEXT: # End machine code for function umull_and_v8i32.
diff --git a/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-02.mir b/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-02.mir
deleted file mode 100644
index 0f5dc000c50b1..0000000000000
--- a/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-02.mir
+++ /dev/null
@@ -1,479 +0,0 @@
-# RUN: llc -mtriple=aarch64-none-linux-gnu -mcpu=cortex-a55  \
-# RUN:  -run-pass=machine-scheduler -debug-only=machine-scheduler -o - %s \
-# RUN:  -misched-bottomup=true -sched-print-cycles=true \
-# RUN:  -misched-dump-reserved-cycles=true -misched-detail-resource-booking=true\
-# RUN:  -misched-dump-schedule-trace=true -misched-dump-schedule-trace-col-width=4 \
-# RUN:  2>&1 | FileCheck %s
-
-# REQUIRES: asserts, aarch64-registered-target
----
-name: f
-tracksRegLiveness: true
-body:             |
-  bb.0:
-    liveins: $x0, $x1, $x2
-    $x3 = ADDXrr $x0, $x0
-    $x4 = ADDXrr $x1, $x1
-    $x5 = ADDXrr $x2, $x2
-    
-# CHECK-LABEL: Before MISched:
-# CHECK-NEXT: # Machine code for function f: IsSSA, NoPHIs, TracksLiveness, NoVRegs
-# CHECK-EMPTY: 
-# CHECK-NEXT: bb.0:
-# CHECK-NEXT:   liveins: $x0, $x1, $x2
-# CHECK-NEXT:   $x3 = ADDXrr $x0, $x0
-# CHECK-NEXT:   $x4 = ADDXrr $x1, $x1
-# CHECK-NEXT:   $x5 = ADDXrr $x2, $x2
-# CHECK-EMPTY: 
-# CHECK-NEXT: # End machine code for function f.
-# CHECK-EMPTY: 
-# CHECK-NEXT: ********** MI Scheduling **********
-# CHECK-NEXT: f:%bb.0 
-# CHECK-NEXT:   From: $x3 = ADDXrr $x0, $x0
-# CHECK-NEXT:     To: End
-# CHECK-NEXT:  RegionInstrs: 3
-# CHECK-NEXT: ScheduleDAGMILive::schedule starting
-# CHECK-NEXT: GenericScheduler RegionPolicy:  ShouldTrackPressure=0 OnlyTopDown=0 OnlyBottomUp=1
-# CHECK-NEXT: Disabled scoreboard hazard recognizer
-# CHECK-NEXT: Disabled scoreboard hazard recognizer
-# CHECK-NEXT: SU(0) [TopReadyCycle = 0, BottomReadyCycle = 0]:   $x3 = ADDXrr $x0, $x0
-# CHECK-NEXT:   # preds left       : 0
-# CHECK-NEXT:   # succs left       : 0
-# CHECK-NEXT:   # rdefs left       : 0
-# CHECK-NEXT:   Latency            : 3
-# CHECK-NEXT:   Depth              : 0
-# CHECK-NEXT:   Height             : 0
-# CHECK-NEXT:   Single Issue       : false;
-# CHECK-NEXT: SU(1) [TopReadyCycle = 0, BottomReadyCycle = 0]:   $x4 = ADDXrr $x1, $x1
-# CHECK-NEXT:   # preds left       : 0
-# CHECK-NEXT:   # succs left       : 0
-# CHECK-NEXT:   # rdefs left       : 0
-# CHECK-NEXT:   Latency            : 3
-# CHECK-NEXT:   Depth              : 0
-# CHECK-NEXT:   Height             : 0
-# CHECK-NEXT:   Single Issue       : false;
-# CHECK-NEXT: SU(2) [TopReadyCycle = 0, BottomReadyCycle = 0]:   $x5 = ADDXrr $x2, $x2
-# CHECK-NEXT:   # preds left       : 0
-# CHECK-NEXT:   # succs left       : 0
-# CHECK-NEXT:   # rdefs left       : 0
-# CHECK-NEXT:   Latency            : 3
-# CHECK-NEXT:   Depth              : 0
-# CHECK-NEXT:   Height             : 0
-# CHECK-NEXT:   Single Issue       : false;
-# CHECK-NEXT:   Resource booking (@0c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@0c): 
-# CHECK-NEXT:     Instance 0 available @0c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
-# CHECK-NEXT:   Resource booking (@0c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@0c): 
-# CHECK-NEXT:     Instance 0 available @0c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
-# CHECK-NEXT:   Resource booking (@0c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@0c): 
-# CHECK-NEXT:     Instance 0 available @0c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
-# CHECK-NEXT:   Resource booking (@0c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@0c): 
-# CHECK-NEXT:     Instance 0 available @0c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
-# CHECK-NEXT:   Resource booking (@0c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@0c): 
-# CHECK-NEXT:     Instance 0 available @0c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
-# CHECK-NEXT:   Resource booking (@0c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@0c): 
-# CHECK-NEXT:     Instance 0 available @0c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
-# CHECK-NEXT: Critical Path(GS-RR ): 0
-# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
-# CHECK-NEXT:   Resource booking (@0c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@0c): 
-# CHECK-NEXT:     Instance 0 available @0c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
-# CHECK-NEXT:   Resource booking (@0c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@0c): 
-# CHECK-NEXT:     Instance 0 available @0c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
-# CHECK-NEXT:   Resource booking (@0c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@0c): 
-# CHECK-NEXT:     Instance 0 available @0c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
-# CHECK-NEXT: Queue BotQ.P: 
-# CHECK-NEXT: Queue BotQ.A: 2 1 0 
-# CHECK-NEXT:   Cand SU(2) ORDER                              
-# CHECK-NEXT: Pick Bot ORDER     
-# CHECK-NEXT: Scheduling SU(2) $x5 = ADDXrr $x2, $x2
-# CHECK-NEXT:   Ready @0c
-# CHECK-NEXT:   CortexA55UnitALU +1x1u
-# CHECK-NEXT:   Resource booking (@0c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@0c): 
-# CHECK-NEXT:     Instance 0 available @0c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
-# CHECK-NEXT:   Resource booking (@0c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@0c): 
-# CHECK-NEXT:     Instance 0 available @0c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
-# CHECK-NEXT: BotQ.A @0c
-# CHECK-NEXT:   Retired: 1
-# CHECK-NEXT:   Executed: 0c
-# CHECK-NEXT:   Critical: 0c, 1 MOps
-# CHECK-NEXT:   ExpectedLatency: 0c
-# CHECK-NEXT:   - Latency limited.
-# CHECK-NEXT: CortexA55UnitALU(0) = 0
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
-# CHECK-NEXT:   Resource booking (@0c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 0
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@0c): 
-# CHECK-NEXT:     Instance 0 available @1c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @0c
-# CHECK-NEXT:   Resource booking (@0c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 0
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@0c): 
-# CHECK-NEXT:     Instance 0 available @1c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @0c
-# CHECK-NEXT: Queue BotQ.P: 
-# CHECK-NEXT: Queue BotQ.A: 0 1 
-# CHECK-NEXT:   Cand SU(0) ORDER                              
-# CHECK-NEXT:   Cand SU(1) ORDER                              
-# CHECK-NEXT: Pick Bot ORDER     
-# CHECK-NEXT: Scheduling SU(1) $x4 = ADDXrr $x1, $x1
-# CHECK-NEXT:   Ready @0c
-# CHECK-NEXT:   CortexA55UnitALU +1x1u
-# CHECK-NEXT:   Resource booking (@0c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 0
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@0c): 
-# CHECK-NEXT:     Instance 0 available @1c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @0c
-# CHECK-NEXT:   Resource booking (@0c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 0
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@0c): 
-# CHECK-NEXT:     Instance 0 available @0c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
-# CHECK-NEXT:   *** Max MOps 2 at cycle 0
-# CHECK-NEXT: Cycle: 1 BotQ.A
-# CHECK-NEXT: BotQ.A @1c
-# CHECK-NEXT:   Retired: 2
-# CHECK-NEXT:   Executed: 1c
-# CHECK-NEXT:   Critical: 1c, 2 MOps
-# CHECK-NEXT:   ExpectedLatency: 0c
-# CHECK-NEXT:   - Latency limited.
-# CHECK-NEXT: CortexA55UnitALU(0) = 0
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
-# CHECK-NEXT:   Resource booking (@1c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 0
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@1c): 
-# CHECK-NEXT:     Instance 0 available @1c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @0c
-# CHECK-NEXT: Queue BotQ.P: 
-# CHECK-NEXT: Queue BotQ.A: 0 
-# CHECK-NEXT: Scheduling SU(0) $x3 = ADDXrr $x0, $x0
-# CHECK-NEXT:   Ready @1c
-# CHECK-NEXT:   CortexA55UnitALU +1x1u
-# CHECK-NEXT:   Resource booking (@1c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 0
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@1c): 
-# CHECK-NEXT:     Instance 0 available @1c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @0c
-# CHECK-NEXT:   Resource booking (@1c): 
-# CHECK-NEXT: CortexA55UnitALU(0) = 0
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT:   getNextResourceCycle (@1c): 
-# CHECK-NEXT:     Instance 0 available @0c
-# CHECK-NEXT:     Instance 1 available @0c
-# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
-# CHECK-NEXT: BotQ.A @1c
-# CHECK-NEXT:   Retired: 3
-# CHECK-NEXT:   Executed: 1c
-# CHECK-NEXT:   Critical: 1c, 3 MOps
-# CHECK-NEXT:   ExpectedLatency: 0c
-# CHECK-NEXT:   - Latency limited.
-# CHECK-NEXT: CortexA55UnitALU(0) = 1
-# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
-# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
-# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
-# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
-# CHECK-NEXT: *** Final schedule for %bb.0 ***
-# CHECK-NEXT:  * Schedule table (BottomUp):
-# CHECK-NEXT:   i: issue
-# CHECK-NEXT:   x: resource booked
-# CHECK-NEXT: Cycle              | 1 | 0 |
-# CHECK-NEXT: SU(0)              | i |   |
-# CHECK-NEXT:   CortexA55UnitALU | x |   | 
-# CHECK-NEXT: SU(1)              |   | i |
-# CHECK-NEXT:   CortexA55UnitALU |   | x | 
-# CHECK-NEXT: SU(2)              |   | i |
-# CHECK-NEXT:   CortexA55UnitALU |   | x | 
-# CHECK-NEXT: SU(0) [TopReadyCycle = 0, BottomReadyCycle = 1]:   $x3 = ADDXrr $x0, $x0
-# CHECK-NEXT: SU(1) [TopReadyCycle = 0, BottomReadyCycle = 0]:   $x4 = ADDXrr $x1, $x1
-# CHECK-NEXT: SU(2) [TopReadyCycle = 0, BottomReadyCycle = 0]:   $x5 = ADDXrr $x2, $x2
-# CHECK-EMPTY: 
-# CHECK-NEXT: ********** INTERVALS **********
-# CHECK-NEXT: W0 [0B,16r:0) 0@0B-phi
-# CHECK-NEXT: W1 [0B,32r:0) 0@0B-phi
-# CHECK-NEXT: W2 [0B,48r:0) 0@0B-phi
-# CHECK-NEXT: RegMasks:
-# CHECK-NEXT: ********** MACHINEINSTRS **********
-# CHECK-NEXT: # Machine code for function f: IsSSA, NoPHIs, TracksLiveness, NoVRegs
-# CHECK-EMPTY: 
-# CHECK-NEXT: 0B	bb.0:
-# CHECK-NEXT: 	  liveins: $x0, $x1, $x2
-# CHECK-NEXT: 16B	  $x3 = ADDXrr $x0, $x0
-# CHECK-NEXT: 32B	  $x4 = ADDXrr $x1, $x1
-# CHECK-NEXT: 48B	  $x5 = ADDXrr $x2, $x2
-# CHECK-EMPTY: 
-# CHECK-NEXT: # End machine code for function f.
\ No newline at end of file

From 37db9cae2bc0d26d8f3f19750739ca0a636f3a29 Mon Sep 17 00:00:00 2001
From: Francesco Petrogalli <francesco.petrogalli@apple.com>
Date: Tue, 20 Jun 2023 11:42:01 +0200
Subject: [PATCH 106/130] [llc][MISched] Add `-misched-detail-resource-booking`
 to llc.

The option `-misched-detail-resource-booking` prints the following
information every time the method
`SchedBoundary::getNextResourceCycle` is invoked:

1. counters of the resources that have already been booked;

2. the values returned by `getNextResourceCycle`, which is the next
available cycle in which a resource can be booked.

The method is useful to debug low-level checks inside the machine
scheduler that make decisions based on the values returned by
`getNextResourceCycle`.

Reviewed By: andreadb

Differential Revision: https://reviews.llvm.org/D153116
---
 llvm/lib/CodeGen/MachineScheduler.cpp         |   18 +-
 .../misched-detail-resource-booking-01.mir    | 1618 +++++++++++++++++
 .../misched-detail-resource-booking-02.mir    |  479 +++++
 3 files changed, 2114 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/AArch64/misched-detail-resource-booking-01.mir
 create mode 100644 llvm/test/CodeGen/AArch64/misched-detail-resource-booking-02.mir

diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp
index 28b17f76c99e2..ed5dff788b7cb 100644
--- a/llvm/lib/CodeGen/MachineScheduler.cpp
+++ b/llvm/lib/CodeGen/MachineScheduler.cpp
@@ -98,9 +98,13 @@ cl::opt<bool> PrintDAGs("misched-print-dags", cl::Hidden,
 cl::opt<bool> MISchedDumpReservedCycles(
     "misched-dump-reserved-cycles", cl::Hidden, cl::init(false),
     cl::desc("Dump resource usage at schedule boundary."));
+cl::opt<bool> MischedDetailResourceBooking(
+    "misched-detail-resource-booking", cl::Hidden, cl::init(false),
+    cl::desc("Show details of invoking getNextResoufceCycle."));
 #else
 const bool ViewMISchedDAGs = false;
 const bool PrintDAGs = false;
+const bool MischedDetailResourceBooking = false;
 #ifdef LLVM_ENABLE_DUMP
 const bool MISchedDumpReservedCycles = false;
 #endif // LLVM_ENABLE_DUMP
@@ -2314,7 +2318,11 @@ unsigned SchedBoundary::getNextResourceCycleByInstance(unsigned InstanceIdx,
 std::pair<unsigned, unsigned>
 SchedBoundary::getNextResourceCycle(const MCSchedClassDesc *SC, unsigned PIdx,
                                     unsigned Cycles, unsigned StartAtCycle) {
-
+  if (MischedDetailResourceBooking) {
+    LLVM_DEBUG(dbgs() << "  Resource booking (@" << CurrCycle << "c): \n");
+    LLVM_DEBUG(dumpReservedCycles());
+    LLVM_DEBUG(dbgs() << "  getNextResourceCycle (@" << CurrCycle << "c): \n");
+  }
   unsigned MinNextUnreserved = InvalidCycle;
   unsigned InstanceIdx = 0;
   unsigned StartIndex = ReservedCyclesIndex[PIdx];
@@ -2355,11 +2363,19 @@ SchedBoundary::getNextResourceCycle(const MCSchedClassDesc *SC, unsigned PIdx,
        ++I) {
     unsigned NextUnreserved =
         getNextResourceCycleByInstance(I, Cycles, StartAtCycle);
+    if (MischedDetailResourceBooking)
+      LLVM_DEBUG(dbgs() << "    Instance " << I - StartIndex << " available @"
+                        << NextUnreserved << "c\n");
     if (MinNextUnreserved > NextUnreserved) {
       InstanceIdx = I;
       MinNextUnreserved = NextUnreserved;
     }
   }
+  if (MischedDetailResourceBooking)
+    LLVM_DEBUG(dbgs() << "    selecting " << SchedModel->getResourceName(PIdx)
+                      << "[" << InstanceIdx - StartIndex << "]"
+                      << " available @" << MinNextUnreserved << "c"
+                      << "\n");
   return std::make_pair(MinNextUnreserved, InstanceIdx);
 }
 
diff --git a/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-01.mir b/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-01.mir
new file mode 100644
index 0000000000000..a012fc5608a3f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-01.mir
@@ -0,0 +1,1618 @@
+# RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+neon  %s -o - 2>&1 \
+# RUN:   -misched-dump-reserved-cycles=true \
+# RUN:   -run-pass=machine-scheduler -debug-only=machine-scheduler \
+# RUN:   -misched-bottomup=true -sched-print-cycles=true \
+# RUN:   -misched-detail-resource-booking=true \
+# RUN:   -misched-dump-schedule-trace=true -misched-dump-schedule-trace-col-header-width=21 \
+# RUN: | FileCheck  %s
+
+--- |
+  ; ModuleID = '../llvm-project/llvm/test/CodeGen/AArch64/aarch64-smull.failing.ll'
+  source_filename = "../llvm-project/llvm/test/CodeGen/AArch64/aarch64-smull.failing.ll"
+  target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+  target triple = "aarch64-none-linux-gnu"
+  
+  define <8 x i32> @umull_and_v8i32(<8 x i16> %src1, <8 x i32> %src2) #0 {
+  entry:
+    %in1 = zext <8 x i16> %src1 to <8 x i32>
+    %in2 = and <8 x i32> %src2, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
+    %out = mul nsw <8 x i32> %in1, %in2
+    ret <8 x i32> %out
+  }
+  
+  attributes #0 = { "target-features"="+neon" }
+
+...
+---
+name:            umull_and_v8i32
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: fpr128 }
+  - { id: 1, class: fpr128 }
+  - { id: 2, class: fpr128 }
+  - { id: 3, class: fpr128 }
+  - { id: 4, class: fpr64 }
+  - { id: 5, class: fpr64 }
+  - { id: 6, class: fpr128 }
+  - { id: 7, class: fpr128 }
+  - { id: 8, class: fpr128 }
+  - { id: 9, class: fpr64 }
+  - { id: 10, class: fpr128 }
+  - { id: 11, class: fpr64 }
+  - { id: 12, class: fpr128 }
+liveins:
+  - { reg: '$q0', virtual-reg: '%0' }
+  - { reg: '$q1', virtual-reg: '%1' }
+  - { reg: '$q2', virtual-reg: '%2' }
+frameInfo:
+  maxAlignment:    1
+  maxCallFrameSize: 0
+machineFunctionInfo: {}
+body:             |
+  bb.0.entry:
+    liveins: $q0, $q1, $q2
+  
+    %2:fpr128 = COPY $q2
+    %1:fpr128 = COPY $q1
+    %0:fpr128 = COPY $q0
+    %3:fpr128 = EXTv16i8 %0, %0, 8
+    %6:fpr128 = MOVIv2d_ns 17
+    %7:fpr128 = ANDv16i8 %2, %6
+    %8:fpr128 = ANDv16i8 %1, %6
+    %9:fpr64 = XTNv4i16 %8
+    %10:fpr128 = UMULLv4i16_v4i32 %0.dsub, %9
+    %11:fpr64 = XTNv4i16 %7
+    %12:fpr128 = UMULLv4i16_v4i32 %3.dsub, %11
+    $q0 = COPY %10
+    $q1 = COPY %12
+    RET_ReallyLR implicit $q0, implicit $q1
+
+...
+
+# CHECK-LABEL: Before MISched:
+# CHECK-NEXT: # Machine code for function umull_and_v8i32: IsSSA, NoPHIs, TracksLiveness
+# CHECK-NEXT: Function Live Ins: $q0 in %0, $q1 in %1, $q2 in %2
+# CHECK-EMPTY: 
+# CHECK-NEXT: bb.0.entry:
+# CHECK-NEXT:   liveins: $q0, $q1, $q2
+# CHECK-NEXT:   %2:fpr128 = COPY $q2
+# CHECK-NEXT:   %1:fpr128 = COPY $q1
+# CHECK-NEXT:   %0:fpr128 = COPY $q0
+# CHECK-NEXT:   %3:fpr128 = EXTv16i8 %0:fpr128, %0:fpr128, 8
+# CHECK-NEXT:   %6:fpr128 = MOVIv2d_ns 17
+# CHECK-NEXT:   %7:fpr128 = ANDv16i8 %2:fpr128, %6:fpr128
+# CHECK-NEXT:   %8:fpr128 = ANDv16i8 %1:fpr128, %6:fpr128
+# CHECK-NEXT:   %9:fpr64 = XTNv4i16 %8:fpr128
+# CHECK-NEXT:   %10:fpr128 = UMULLv4i16_v4i32 %0.dsub:fpr128, %9:fpr64
+# CHECK-NEXT:   %11:fpr64 = XTNv4i16 %7:fpr128
+# CHECK-NEXT:   %12:fpr128 = UMULLv4i16_v4i32 %3.dsub:fpr128, %11:fpr64
+# CHECK-NEXT:   $q0 = COPY %10:fpr128
+# CHECK-NEXT:   $q1 = COPY %12:fpr128
+# CHECK-NEXT:   RET_ReallyLR implicit $q0, implicit $q1
+# CHECK-EMPTY: 
+# CHECK-NEXT: # End machine code for function umull_and_v8i32.
+# CHECK-EMPTY: 
+# CHECK-NEXT: ********** MI Scheduling **********
+# CHECK-NEXT: umull_and_v8i32:%bb.0 entry
+# CHECK-NEXT:   From: %2:fpr128 = COPY $q2
+# CHECK-NEXT:     To: RET_ReallyLR implicit $q0, implicit $q1
+# CHECK-NEXT:  RegionInstrs: 13
+# CHECK-NEXT: ScheduleDAGMILive::schedule starting
+# CHECK-NEXT: GenericScheduler RegionPolicy:  ShouldTrackPressure=0 OnlyTopDown=0 OnlyBottomUp=1
+# CHECK-NEXT: Disabled scoreboard hazard recognizer
+# CHECK-NEXT: Disabled scoreboard hazard recognizer
+# CHECK-NEXT: SU(0) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %2:fpr128 = COPY $q2
+# CHECK-NEXT:   # preds left       : 0
+# CHECK-NEXT:   # succs left       : 1
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 3
+# CHECK-NEXT:   Depth              : 0
+# CHECK-NEXT:   Height             : 13
+# CHECK-NEXT:   Successors:
+# CHECK-NEXT:     SU(5): Data Latency=3 Reg=%2
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(1) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %1:fpr128 = COPY $q1
+# CHECK-NEXT:   # preds left       : 0
+# CHECK-NEXT:   # succs left       : 2
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 3
+# CHECK-NEXT:   Depth              : 0
+# CHECK-NEXT:   Height             : 13
+# CHECK-NEXT:   Successors:
+# CHECK-NEXT:     SU(6): Data Latency=3 Reg=%1
+# CHECK-NEXT:     SU(12): Anti Latency=0
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(2) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %0:fpr128 = COPY $q0
+# CHECK-NEXT:   # preds left       : 0
+# CHECK-NEXT:   # succs left       : 3
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 3
+# CHECK-NEXT:   Depth              : 0
+# CHECK-NEXT:   Height             : 14
+# CHECK-NEXT:   Successors:
+# CHECK-NEXT:     SU(8): Data Latency=3 Reg=%0
+# CHECK-NEXT:     SU(3): Data Latency=3 Reg=%0
+# CHECK-NEXT:     SU(11): Anti Latency=0
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(3) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %3:fpr128 = EXTv16i8 %0:fpr128, %0:fpr128, 8
+# CHECK-NEXT:   # preds left       : 1
+# CHECK-NEXT:   # succs left       : 1
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 4
+# CHECK-NEXT:   Depth              : 3
+# CHECK-NEXT:   Height             : 11
+# CHECK-NEXT:   Predecessors:
+# CHECK-NEXT:     SU(2): Data Latency=3 Reg=%0
+# CHECK-NEXT:   Successors:
+# CHECK-NEXT:     SU(10): Data Latency=4 Reg=%3
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(4) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %6:fpr128 = MOVIv2d_ns 17
+# CHECK-NEXT:   # preds left       : 0
+# CHECK-NEXT:   # succs left       : 2
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 4
+# CHECK-NEXT:   Depth              : 0
+# CHECK-NEXT:   Height             : 14
+# CHECK-NEXT:   Successors:
+# CHECK-NEXT:     SU(6): Data Latency=4 Reg=%6
+# CHECK-NEXT:     SU(5): Data Latency=4 Reg=%6
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(5) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %7:fpr128 = ANDv16i8 %2:fpr128, %6:fpr128
+# CHECK-NEXT:   # preds left       : 2
+# CHECK-NEXT:   # succs left       : 1
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 1
+# CHECK-NEXT:   Depth              : 4
+# CHECK-NEXT:   Height             : 10
+# CHECK-NEXT:   Predecessors:
+# CHECK-NEXT:     SU(4): Data Latency=4 Reg=%6
+# CHECK-NEXT:     SU(0): Data Latency=3 Reg=%2
+# CHECK-NEXT:   Successors:
+# CHECK-NEXT:     SU(9): Data Latency=1 Reg=%7
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(6) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %8:fpr128 = ANDv16i8 %1:fpr128, %6:fpr128
+# CHECK-NEXT:   # preds left       : 2
+# CHECK-NEXT:   # succs left       : 1
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 1
+# CHECK-NEXT:   Depth              : 4
+# CHECK-NEXT:   Height             : 10
+# CHECK-NEXT:   Predecessors:
+# CHECK-NEXT:     SU(4): Data Latency=4 Reg=%6
+# CHECK-NEXT:     SU(1): Data Latency=3 Reg=%1
+# CHECK-NEXT:   Successors:
+# CHECK-NEXT:     SU(7): Data Latency=1 Reg=%8
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(7) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %9:fpr64 = XTNv4i16 %8:fpr128
+# CHECK-NEXT:   # preds left       : 1
+# CHECK-NEXT:   # succs left       : 1
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 2
+# CHECK-NEXT:   Depth              : 5
+# CHECK-NEXT:   Height             : 9
+# CHECK-NEXT:   Predecessors:
+# CHECK-NEXT:     SU(6): Data Latency=1 Reg=%8
+# CHECK-NEXT:   Successors:
+# CHECK-NEXT:     SU(8): Data Latency=2 Reg=%9
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(8) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %10:fpr128 = UMULLv4i16_v4i32 %0.dsub:fpr128, %9:fpr64
+# CHECK-NEXT:   # preds left       : 2
+# CHECK-NEXT:   # succs left       : 1
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 4
+# CHECK-NEXT:   Depth              : 7
+# CHECK-NEXT:   Height             : 7
+# CHECK-NEXT:   Predecessors:
+# CHECK-NEXT:     SU(7): Data Latency=2 Reg=%9
+# CHECK-NEXT:     SU(2): Data Latency=3 Reg=%0
+# CHECK-NEXT:   Successors:
+# CHECK-NEXT:     SU(11): Data Latency=4 Reg=%10
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(9) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %11:fpr64 = XTNv4i16 %7:fpr128
+# CHECK-NEXT:   # preds left       : 1
+# CHECK-NEXT:   # succs left       : 1
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 2
+# CHECK-NEXT:   Depth              : 5
+# CHECK-NEXT:   Height             : 9
+# CHECK-NEXT:   Predecessors:
+# CHECK-NEXT:     SU(5): Data Latency=1 Reg=%7
+# CHECK-NEXT:   Successors:
+# CHECK-NEXT:     SU(10): Data Latency=2 Reg=%11
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(10) [TopReadyCycle = 0, BottomReadyCycle = 0]:   %12:fpr128 = UMULLv4i16_v4i32 %3.dsub:fpr128, %11:fpr64
+# CHECK-NEXT:   # preds left       : 2
+# CHECK-NEXT:   # succs left       : 1
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 4
+# CHECK-NEXT:   Depth              : 7
+# CHECK-NEXT:   Height             : 7
+# CHECK-NEXT:   Predecessors:
+# CHECK-NEXT:     SU(9): Data Latency=2 Reg=%11
+# CHECK-NEXT:     SU(3): Data Latency=4 Reg=%3
+# CHECK-NEXT:   Successors:
+# CHECK-NEXT:     SU(12): Data Latency=4 Reg=%12
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(11) [TopReadyCycle = 0, BottomReadyCycle = 0]:   $q0 = COPY %10:fpr128
+# CHECK-NEXT:   # preds left       : 2
+# CHECK-NEXT:   # succs left       : 1
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 3
+# CHECK-NEXT:   Depth              : 11
+# CHECK-NEXT:   Height             : 3
+# CHECK-NEXT:   Predecessors:
+# CHECK-NEXT:     SU(8): Data Latency=4 Reg=%10
+# CHECK-NEXT:     SU(2): Anti Latency=0
+# CHECK-NEXT:   Successors:
+# CHECK-NEXT:     ExitSU: Ord  Latency=3 Artificial
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(12) [TopReadyCycle = 0, BottomReadyCycle = 0]:   $q1 = COPY %12:fpr128
+# CHECK-NEXT:   # preds left       : 2
+# CHECK-NEXT:   # succs left       : 1
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 3
+# CHECK-NEXT:   Depth              : 11
+# CHECK-NEXT:   Height             : 3
+# CHECK-NEXT:   Predecessors:
+# CHECK-NEXT:     SU(10): Data Latency=4 Reg=%12
+# CHECK-NEXT:     SU(1): Anti Latency=0
+# CHECK-NEXT:   Successors:
+# CHECK-NEXT:     ExitSU: Ord  Latency=3 Artificial
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: ExitSU [TopReadyCycle = 0, BottomReadyCycle = 0]:   RET_ReallyLR implicit $q0, implicit $q1
+# CHECK-NEXT:   # preds left       : 2
+# CHECK-NEXT:   # succs left       : 0
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 0
+# CHECK-NEXT:   Depth              : 14
+# CHECK-NEXT:   Height             : 0
+# CHECK-NEXT:   Predecessors:
+# CHECK-NEXT:     SU(12): Ord  Latency=3 Artificial
+# CHECK-NEXT:     SU(11): Ord  Latency=3 Artificial
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @0c
+# CHECK-NEXT: Critical Path(GS-RR ): 14
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT: Cycle: 3 BotQ.A
+# CHECK-NEXT:   Resource booking (@3c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@3c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@3c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@3c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT: Queue BotQ.P: 
+# CHECK-NEXT: Queue BotQ.A: 12 11 
+# CHECK-NEXT:   Cand SU(12) ORDER                              
+# CHECK-NEXT: Pick Bot ORDER     
+# CHECK-NEXT: Scheduling SU(12) $q1 = COPY %12:fpr128
+# CHECK-NEXT:   Ready @3c
+# CHECK-NEXT:   CortexA55UnitALU +1x1u
+# CHECK-NEXT:   Resource booking (@3c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@3c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@3c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@3c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   BotQ.A TopLatency SU(12) 11c
+# CHECK-NEXT:   BotQ.A BotLatency SU(12) 3c
+# CHECK-NEXT: BotQ.A @3c
+# CHECK-NEXT:   Retired: 1
+# CHECK-NEXT:   Executed: 3c
+# CHECK-NEXT:   Critical: 0c, 1 MOps
+# CHECK-NEXT:   ExpectedLatency: 3c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT:   Resource booking (@3c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@3c): 
+# CHECK-NEXT:     Instance 0 available @4c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @0c
+# CHECK-NEXT: Queue BotQ.P: 10 
+# CHECK-NEXT: Queue BotQ.A: 11 
+# CHECK-NEXT: Scheduling SU(11) $q0 = COPY %10:fpr128
+# CHECK-NEXT:   Ready @3c
+# CHECK-NEXT:   CortexA55UnitALU +1x1u
+# CHECK-NEXT:   Resource booking (@3c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@3c): 
+# CHECK-NEXT:     Instance 0 available @4c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @0c
+# CHECK-NEXT:   Resource booking (@3c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@3c): 
+# CHECK-NEXT:     Instance 0 available @3c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @0c
+# CHECK-NEXT:   *** Max MOps 2 at cycle 3
+# CHECK-NEXT: Cycle: 4 BotQ.A
+# CHECK-NEXT: BotQ.A @4c
+# CHECK-NEXT:   Retired: 2
+# CHECK-NEXT:   Executed: 4c
+# CHECK-NEXT:   Critical: 1c, 2 MOps
+# CHECK-NEXT:   ExpectedLatency: 3c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT: Cycle: 7 BotQ.A
+# CHECK-NEXT:   Resource booking (@7c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@7c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@7c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@7c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @0c
+# CHECK-NEXT: Queue BotQ.P: 
+# CHECK-NEXT: Queue BotQ.A: 10 8 
+# CHECK-NEXT:   Cand SU(10) ORDER                              
+# CHECK-NEXT: Pick Bot ORDER     
+# CHECK-NEXT: Scheduling SU(10) %12:fpr128 = UMULLv4i16_v4i32 %3.dsub:fpr128, %11:fpr64
+# CHECK-NEXT:   Ready @7c
+# CHECK-NEXT:   CortexA55UnitFPALU +2x1u
+# CHECK-NEXT:   Resource booking (@7c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@7c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@7c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@7c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @0c
+# CHECK-NEXT:   BotQ.A BotLatency SU(10) 7c
+# CHECK-NEXT:   Bump cycle to begin group
+# CHECK-NEXT: Cycle: 8 BotQ.A
+# CHECK-NEXT: BotQ.A @8c
+# CHECK-NEXT:   Retired: 3
+# CHECK-NEXT:   Executed: 8c
+# CHECK-NEXT:   Critical: 1c, 3 MOps
+# CHECK-NEXT:   ExpectedLatency: 7c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT:   Resource booking (@8c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@8c): 
+# CHECK-NEXT:     Instance 0 available @9c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @0c
+# CHECK-NEXT: Queue BotQ.P: 9 3 
+# CHECK-NEXT: Queue BotQ.A: 8 
+# CHECK-NEXT: Scheduling SU(8) %10:fpr128 = UMULLv4i16_v4i32 %0.dsub:fpr128, %9:fpr64
+# CHECK-NEXT:   Ready @8c
+# CHECK-NEXT:   CortexA55UnitFPALU +2x1u
+# CHECK-NEXT:   Resource booking (@8c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@8c): 
+# CHECK-NEXT:     Instance 0 available @9c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @0c
+# CHECK-NEXT:   Resource booking (@8c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@8c): 
+# CHECK-NEXT:     Instance 0 available @7c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @0c
+# CHECK-NEXT:   BotQ.A TopLatency SU(8) 7c
+# CHECK-NEXT:   Bump cycle to begin group
+# CHECK-NEXT: Cycle: 9 BotQ.A
+# CHECK-NEXT: BotQ.A @9c
+# CHECK-NEXT:   Retired: 4
+# CHECK-NEXT:   Executed: 9c
+# CHECK-NEXT:   Critical: 2c, 4 MOps
+# CHECK-NEXT:   ExpectedLatency: 7c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT:   Resource booking (@9c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@9c): 
+# CHECK-NEXT:     Instance 0 available @8c
+# CHECK-NEXT:     Instance 1 available @9c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @8c
+# CHECK-NEXT:   Resource booking (@9c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@9c): 
+# CHECK-NEXT:     Instance 0 available @8c
+# CHECK-NEXT:     Instance 1 available @9c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @8c
+# CHECK-NEXT: Queue BotQ.P: 7 3 
+# CHECK-NEXT: Queue BotQ.A: 9 
+# CHECK-NEXT: Scheduling SU(9) %11:fpr64 = XTNv4i16 %7:fpr128
+# CHECK-NEXT:   Ready @9c
+# CHECK-NEXT:   CortexA55UnitFPALU +1x1u
+# CHECK-NEXT:   Resource booking (@9c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@9c): 
+# CHECK-NEXT:     Instance 0 available @8c
+# CHECK-NEXT:     Instance 1 available @9c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @8c
+# CHECK-NEXT:   Resource booking (@9c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 7
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@9c): 
+# CHECK-NEXT:     Instance 0 available @7c
+# CHECK-NEXT:     Instance 1 available @8c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @7c
+# CHECK-NEXT:   BotQ.A BotLatency SU(9) 9c
+# CHECK-NEXT: BotQ.A @9c
+# CHECK-NEXT:   Retired: 5
+# CHECK-NEXT:   Executed: 9c
+# CHECK-NEXT:   Critical: 2c, 5 MOps
+# CHECK-NEXT:   ExpectedLatency: 9c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT: Cycle: 10 BotQ.A
+# CHECK-NEXT:   Resource booking (@10c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@10c): 
+# CHECK-NEXT:     Instance 0 available @10c
+# CHECK-NEXT:     Instance 1 available @9c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @9c
+# CHECK-NEXT:   Resource booking (@10c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@10c): 
+# CHECK-NEXT:     Instance 0 available @11c
+# CHECK-NEXT:     Instance 1 available @10c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @10c
+# CHECK-NEXT: Queue BotQ.P: 3 
+# CHECK-NEXT: Queue BotQ.A: 7 5 
+# CHECK-NEXT:   Cand SU(7) ORDER                              
+# CHECK-NEXT: Pick Bot ORDER     
+# CHECK-NEXT: Scheduling SU(7) %9:fpr64 = XTNv4i16 %8:fpr128
+# CHECK-NEXT:   Ready @10c
+# CHECK-NEXT:   CortexA55UnitFPALU +1x1u
+# CHECK-NEXT:   Resource booking (@10c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@10c): 
+# CHECK-NEXT:     Instance 0 available @10c
+# CHECK-NEXT:     Instance 1 available @9c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @9c
+# CHECK-NEXT:   Resource booking (@10c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 8
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@10c): 
+# CHECK-NEXT:     Instance 0 available @9c
+# CHECK-NEXT:     Instance 1 available @8c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @8c
+# CHECK-NEXT: BotQ.A @10c
+# CHECK-NEXT:   Retired: 6
+# CHECK-NEXT:   Executed: 10c
+# CHECK-NEXT:   Critical: 3c, 6 MOps
+# CHECK-NEXT:   ExpectedLatency: 9c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT:   Resource booking (@10c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@10c): 
+# CHECK-NEXT:     Instance 0 available @11c
+# CHECK-NEXT:     Instance 1 available @12c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @11c
+# CHECK-NEXT:   SU(5) CortexA55UnitFPALU[0]=11c
+# CHECK-NEXT: Cycle: 11 BotQ.A
+# CHECK-NEXT:   Resource booking (@11c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@11c): 
+# CHECK-NEXT:     Instance 0 available @11c
+# CHECK-NEXT:     Instance 1 available @12c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @11c
+# CHECK-NEXT:   Resource booking (@11c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@11c): 
+# CHECK-NEXT:     Instance 0 available @11c
+# CHECK-NEXT:     Instance 1 available @12c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @11c
+# CHECK-NEXT:   Resource booking (@11c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@11c): 
+# CHECK-NEXT:     Instance 0 available @11c
+# CHECK-NEXT:     Instance 1 available @12c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @11c
+# CHECK-NEXT: Queue BotQ.P: 
+# CHECK-NEXT: Queue BotQ.A: 3 5 6 
+# CHECK-NEXT:   Cand SU(3) ORDER                              
+# CHECK-NEXT:   Cand SU(5) ORDER                              
+# CHECK-NEXT:   Cand SU(6) ORDER                              
+# CHECK-NEXT: Pick Bot ORDER     
+# CHECK-NEXT: Scheduling SU(6) %8:fpr128 = ANDv16i8 %1:fpr128, %6:fpr128
+# CHECK-NEXT:   Ready @11c
+# CHECK-NEXT:   CortexA55UnitFPALU +2x1u
+# CHECK-NEXT:   *** Critical resource CortexA55UnitFPALU: 4c
+# CHECK-NEXT:   Resource booking (@11c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@11c): 
+# CHECK-NEXT:     Instance 0 available @11c
+# CHECK-NEXT:     Instance 1 available @12c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @11c
+# CHECK-NEXT:   Resource booking (@11c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 9
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@11c): 
+# CHECK-NEXT:     Instance 0 available @9c
+# CHECK-NEXT:     Instance 1 available @10c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @9c
+# CHECK-NEXT:   BotQ.A BotLatency SU(6) 10c
+# CHECK-NEXT:   Bump cycle to begin group
+# CHECK-NEXT: Cycle: 12 BotQ.A
+# CHECK-NEXT: BotQ.A @12c
+# CHECK-NEXT:   Retired: 7
+# CHECK-NEXT:   Executed: 12c
+# CHECK-NEXT:   Critical: 4c, 8 CortexA55UnitFPALU
+# CHECK-NEXT:   ExpectedLatency: 10c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT:   Resource booking (@12c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@12c): 
+# CHECK-NEXT:     Instance 0 available @13c
+# CHECK-NEXT:     Instance 1 available @12c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @12c
+# CHECK-NEXT:   Resource booking (@12c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@12c): 
+# CHECK-NEXT:     Instance 0 available @13c
+# CHECK-NEXT:     Instance 1 available @12c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @12c
+# CHECK-NEXT: Queue BotQ.P: 1 
+# CHECK-NEXT: Queue BotQ.A: 3 5 
+# CHECK-NEXT:   Cand SU(3) ORDER                              
+# CHECK-NEXT:   Cand SU(5) ORDER                              
+# CHECK-NEXT: Pick Bot ORDER     
+# CHECK-NEXT: Scheduling SU(5) %7:fpr128 = ANDv16i8 %2:fpr128, %6:fpr128
+# CHECK-NEXT:   Ready @12c
+# CHECK-NEXT:   CortexA55UnitFPALU +2x1u
+# CHECK-NEXT:   Resource booking (@12c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@12c): 
+# CHECK-NEXT:     Instance 0 available @13c
+# CHECK-NEXT:     Instance 1 available @12c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @12c
+# CHECK-NEXT:   Resource booking (@12c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 10
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@12c): 
+# CHECK-NEXT:     Instance 0 available @11c
+# CHECK-NEXT:     Instance 1 available @10c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @10c
+# CHECK-NEXT:   BotQ.A TopLatency SU(5) 4c
+# CHECK-NEXT:   Bump cycle to begin group
+# CHECK-NEXT: Cycle: 13 BotQ.A
+# CHECK-NEXT: BotQ.A @13c
+# CHECK-NEXT:   Retired: 8
+# CHECK-NEXT:   Executed: 13c
+# CHECK-NEXT:   Critical: 5c, 10 CortexA55UnitFPALU
+# CHECK-NEXT:   ExpectedLatency: 10c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT:   Resource booking (@13c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@13c): 
+# CHECK-NEXT:     Instance 0 available @13c
+# CHECK-NEXT:     Instance 1 available @14c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @13c
+# CHECK-NEXT: Queue BotQ.P: 1 4 0 
+# CHECK-NEXT: Queue BotQ.A: 3 
+# CHECK-NEXT: Scheduling SU(3) %3:fpr128 = EXTv16i8 %0:fpr128, %0:fpr128, 8
+# CHECK-NEXT:   Ready @13c
+# CHECK-NEXT:   CortexA55UnitFPALU +2x1u
+# CHECK-NEXT:   Resource booking (@13c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@13c): 
+# CHECK-NEXT:     Instance 0 available @13c
+# CHECK-NEXT:     Instance 1 available @14c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @13c
+# CHECK-NEXT:   Resource booking (@13c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 11
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@13c): 
+# CHECK-NEXT:     Instance 0 available @11c
+# CHECK-NEXT:     Instance 1 available @12c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[0] available @11c
+# CHECK-NEXT:   BotQ.A BotLatency SU(3) 11c
+# CHECK-NEXT:   Bump cycle to begin group
+# CHECK-NEXT: Cycle: 14 BotQ.A
+# CHECK-NEXT: BotQ.A @14c
+# CHECK-NEXT:   Retired: 9
+# CHECK-NEXT:   Executed: 14c
+# CHECK-NEXT:   Critical: 6c, 12 CortexA55UnitFPALU
+# CHECK-NEXT:   ExpectedLatency: 11c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT:   Resource booking (@14c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@14c): 
+# CHECK-NEXT:     Instance 0 available @4c
+# CHECK-NEXT:     Instance 1 available @4c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @4c
+# CHECK-NEXT:   Resource booking (@14c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@14c): 
+# CHECK-NEXT:     Instance 0 available @4c
+# CHECK-NEXT:     Instance 1 available @4c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @4c
+# CHECK-NEXT: Queue BotQ.P: 2 4 0 
+# CHECK-NEXT: Queue BotQ.A: 1 
+# CHECK-NEXT: Scheduling SU(1) %1:fpr128 = COPY $q1
+# CHECK-NEXT:   Ready @14c
+# CHECK-NEXT:   CortexA55UnitALU +1x1u
+# CHECK-NEXT:   Resource booking (@14c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@14c): 
+# CHECK-NEXT:     Instance 0 available @4c
+# CHECK-NEXT:     Instance 1 available @4c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @4c
+# CHECK-NEXT:   Resource booking (@14c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 3
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@14c): 
+# CHECK-NEXT:     Instance 0 available @3c
+# CHECK-NEXT:     Instance 1 available @3c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @3c
+# CHECK-NEXT:   BotQ.A BotLatency SU(1) 13c
+# CHECK-NEXT: BotQ.A @14c
+# CHECK-NEXT:   Retired: 10
+# CHECK-NEXT:   Executed: 14c
+# CHECK-NEXT:   Critical: 6c, 12 CortexA55UnitFPALU
+# CHECK-NEXT:   ExpectedLatency: 13c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 14
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT: Cycle: 15 BotQ.A
+# CHECK-NEXT:   Resource booking (@15c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 14
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@15c): 
+# CHECK-NEXT:     Instance 0 available @15c
+# CHECK-NEXT:     Instance 1 available @4c
+# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @4c
+# CHECK-NEXT: Queue BotQ.P: 2 4 
+# CHECK-NEXT: Queue BotQ.A: 0 
+# CHECK-NEXT: Scheduling SU(0) %2:fpr128 = COPY $q2
+# CHECK-NEXT:   Ready @15c
+# CHECK-NEXT:   CortexA55UnitALU +1x1u
+# CHECK-NEXT:   Resource booking (@15c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 14
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@15c): 
+# CHECK-NEXT:     Instance 0 available @15c
+# CHECK-NEXT:     Instance 1 available @4c
+# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @4c
+# CHECK-NEXT:   Resource booking (@15c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 14
+# CHECK-NEXT: CortexA55UnitALU(1) = 3
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@15c): 
+# CHECK-NEXT:     Instance 0 available @14c
+# CHECK-NEXT:     Instance 1 available @3c
+# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @3c
+# CHECK-NEXT: BotQ.A @15c
+# CHECK-NEXT:   Retired: 11
+# CHECK-NEXT:   Executed: 15c
+# CHECK-NEXT:   Critical: 6c, 12 CortexA55UnitFPALU
+# CHECK-NEXT:   ExpectedLatency: 13c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 14
+# CHECK-NEXT: CortexA55UnitALU(1) = 15
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT: Cycle: 16 BotQ.A
+# CHECK-NEXT:   Resource booking (@16c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 14
+# CHECK-NEXT: CortexA55UnitALU(1) = 15
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@16c): 
+# CHECK-NEXT:     Instance 0 available @15c
+# CHECK-NEXT:     Instance 1 available @16c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @15c
+# CHECK-NEXT:   Resource booking (@16c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 14
+# CHECK-NEXT: CortexA55UnitALU(1) = 15
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@16c): 
+# CHECK-NEXT:     Instance 0 available @15c
+# CHECK-NEXT:     Instance 1 available @14c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @14c
+# CHECK-NEXT: Queue BotQ.P: 
+# CHECK-NEXT: Queue BotQ.A: 2 4 
+# CHECK-NEXT:   Cand SU(2) ORDER                              
+# CHECK-NEXT:   Cand SU(4) PHYS-REG                           
+# CHECK-NEXT: Pick Bot PHYS-REG  
+# CHECK-NEXT: Scheduling SU(4) %6:fpr128 = MOVIv2d_ns 17
+# CHECK-NEXT:   Ready @16c
+# CHECK-NEXT:   CortexA55UnitFPALU +2x1u
+# CHECK-NEXT:   Resource booking (@16c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 14
+# CHECK-NEXT: CortexA55UnitALU(1) = 15
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@16c): 
+# CHECK-NEXT:     Instance 0 available @15c
+# CHECK-NEXT:     Instance 1 available @14c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @14c
+# CHECK-NEXT:   Resource booking (@16c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 14
+# CHECK-NEXT: CortexA55UnitALU(1) = 15
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 12
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@16c): 
+# CHECK-NEXT:     Instance 0 available @13c
+# CHECK-NEXT:     Instance 1 available @12c
+# CHECK-NEXT:     selecting CortexA55UnitFPALU[1] available @12c
+# CHECK-NEXT:   BotQ.A BotLatency SU(4) 14c
+# CHECK-NEXT:   Bump cycle to begin group
+# CHECK-NEXT: Cycle: 17 BotQ.A
+# CHECK-NEXT: BotQ.A @17c
+# CHECK-NEXT:   Retired: 12
+# CHECK-NEXT:   Executed: 17c
+# CHECK-NEXT:   Critical: 7c, 14 CortexA55UnitFPALU
+# CHECK-NEXT:   ExpectedLatency: 14c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 14
+# CHECK-NEXT: CortexA55UnitALU(1) = 15
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 16
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT:   Resource booking (@17c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 14
+# CHECK-NEXT: CortexA55UnitALU(1) = 15
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 16
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@17c): 
+# CHECK-NEXT:     Instance 0 available @15c
+# CHECK-NEXT:     Instance 1 available @16c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @15c
+# CHECK-NEXT: Queue BotQ.P: 
+# CHECK-NEXT: Queue BotQ.A: 2 
+# CHECK-NEXT: Scheduling SU(2) %0:fpr128 = COPY $q0
+# CHECK-NEXT:   Ready @17c
+# CHECK-NEXT:   CortexA55UnitALU +1x1u
+# CHECK-NEXT:   Resource booking (@17c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 14
+# CHECK-NEXT: CortexA55UnitALU(1) = 15
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 16
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@17c): 
+# CHECK-NEXT:     Instance 0 available @15c
+# CHECK-NEXT:     Instance 1 available @16c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @15c
+# CHECK-NEXT:   Resource booking (@17c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 14
+# CHECK-NEXT: CortexA55UnitALU(1) = 15
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 16
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@17c): 
+# CHECK-NEXT:     Instance 0 available @14c
+# CHECK-NEXT:     Instance 1 available @15c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @14c
+# CHECK-NEXT: BotQ.A @17c
+# CHECK-NEXT:   Retired: 13
+# CHECK-NEXT:   Executed: 17c
+# CHECK-NEXT:   Critical: 7c, 14 CortexA55UnitFPALU
+# CHECK-NEXT:   ExpectedLatency: 14c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 17
+# CHECK-NEXT: CortexA55UnitALU(1) = 15
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 13
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 16
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT: *** Final schedule for %bb.0 ***
+# CHECK-NEXT:  * Schedule table (BottomUp):
+# CHECK-NEXT:   i: issue
+# CHECK-NEXT:   x: resource booked
+# CHECK-NEXT: Cycle              | 17 | 16 | 15 | 14 | 13 | 12 | 11 | 10 | 9  | 8  | 7  | 6  | 5  | 4  | 3  |
+# CHECK-NEXT: SU(2)              | i  |    |    |    |    |    |    |    |    |    |    |    |    |    |    |
+# CHECK-NEXT:   CortexA55UnitALU | x  |    |    |    |    |    |    |    |    |    |    |    |    |    |    | 
+# CHECK-NEXT: SU(4)              |    | i  |    |    |    |    |    |    |    |    |    |    |    |    |    |
+# CHECK-NEXT: CortexA55UnitFPALU |    | x  | x  |    |    |    |    |    |    |    |    |    |    |    |    | 
+# CHECK-NEXT: SU(0)              |    |    | i  |    |    |    |    |    |    |    |    |    |    |    |    |
+# CHECK-NEXT:   CortexA55UnitALU |    |    | x  |    |    |    |    |    |    |    |    |    |    |    |    | 
+# CHECK-NEXT: SU(1)              |    |    |    | i  |    |    |    |    |    |    |    |    |    |    |    |
+# CHECK-NEXT:   CortexA55UnitALU |    |    |    | x  |    |    |    |    |    |    |    |    |    |    |    | 
+# CHECK-NEXT: SU(3)              |    |    |    |    | i  |    |    |    |    |    |    |    |    |    |    |
+# CHECK-NEXT: CortexA55UnitFPALU |    |    |    |    | x  | x  |    |    |    |    |    |    |    |    |    | 
+# CHECK-NEXT: SU(5)              |    |    |    |    |    | i  |    |    |    |    |    |    |    |    |    |
+# CHECK-NEXT: CortexA55UnitFPALU |    |    |    |    |    | x  | x  |    |    |    |    |    |    |    |    | 
+# CHECK-NEXT: SU(6)              |    |    |    |    |    |    | i  |    |    |    |    |    |    |    |    |
+# CHECK-NEXT: CortexA55UnitFPALU |    |    |    |    |    |    | x  | x  |    |    |    |    |    |    |    | 
+# CHECK-NEXT: SU(7)              |    |    |    |    |    |    |    | i  |    |    |    |    |    |    |    |
+# CHECK-NEXT: CortexA55UnitFPALU |    |    |    |    |    |    |    | x  |    |    |    |    |    |    |    | 
+# CHECK-NEXT: SU(9)              |    |    |    |    |    |    |    |    | i  |    |    |    |    |    |    |
+# CHECK-NEXT: CortexA55UnitFPALU |    |    |    |    |    |    |    |    | x  |    |    |    |    |    |    | 
+# CHECK-NEXT: SU(8)              |    |    |    |    |    |    |    |    |    | i  |    |    |    |    |    |
+# CHECK-NEXT: CortexA55UnitFPALU |    |    |    |    |    |    |    |    |    | x  | x  |    |    |    |    | 
+# CHECK-NEXT: SU(10)             |    |    |    |    |    |    |    |    |    |    | i  |    |    |    |    |
+# CHECK-NEXT: CortexA55UnitFPALU |    |    |    |    |    |    |    |    |    |    | x  | x  |    |    |    | 
+# CHECK-NEXT: SU(11)             |    |    |    |    |    |    |    |    |    |    |    |    |    |    | i  |
+# CHECK-NEXT:   CortexA55UnitALU |    |    |    |    |    |    |    |    |    |    |    |    |    |    | x  | 
+# CHECK-NEXT: SU(12)             |    |    |    |    |    |    |    |    |    |    |    |    |    |    | i  |
+# CHECK-NEXT:   CortexA55UnitALU |    |    |    |    |    |    |    |    |    |    |    |    |    |    | x  | 
+# CHECK-NEXT: SU(2) [TopReadyCycle = 0, BottomReadyCycle = 17]:   %0:fpr128 = COPY $q0
+# CHECK-NEXT: SU(4) [TopReadyCycle = 0, BottomReadyCycle = 16]:   %6:fpr128 = MOVIv2d_ns 17
+# CHECK-NEXT: SU(0) [TopReadyCycle = 0, BottomReadyCycle = 15]:   %2:fpr128 = COPY $q2
+# CHECK-NEXT: SU(1) [TopReadyCycle = 0, BottomReadyCycle = 14]:   %1:fpr128 = COPY $q1
+# CHECK-NEXT: SU(3) [TopReadyCycle = 0, BottomReadyCycle = 13]:   %3:fpr128 = EXTv16i8 %0:fpr128, %0:fpr128, 8
+# CHECK-NEXT: SU(5) [TopReadyCycle = 0, BottomReadyCycle = 12]:   %7:fpr128 = ANDv16i8 %2:fpr128, %6:fpr128
+# CHECK-NEXT: SU(6) [TopReadyCycle = 0, BottomReadyCycle = 11]:   %8:fpr128 = ANDv16i8 %1:fpr128, %6:fpr128
+# CHECK-NEXT: SU(7) [TopReadyCycle = 0, BottomReadyCycle = 10]:   %9:fpr64 = XTNv4i16 %8:fpr128
+# CHECK-NEXT: SU(9) [TopReadyCycle = 0, BottomReadyCycle = 9]:   %11:fpr64 = XTNv4i16 %7:fpr128
+# CHECK-NEXT: SU(8) [TopReadyCycle = 0, BottomReadyCycle = 8]:   %10:fpr128 = UMULLv4i16_v4i32 %0.dsub:fpr128, %9:fpr64
+# CHECK-NEXT: SU(10) [TopReadyCycle = 0, BottomReadyCycle = 7]:   %12:fpr128 = UMULLv4i16_v4i32 %3.dsub:fpr128, %11:fpr64
+# CHECK-NEXT: SU(11) [TopReadyCycle = 0, BottomReadyCycle = 3]:   $q0 = COPY %10:fpr128
+# CHECK-NEXT: SU(12) [TopReadyCycle = 0, BottomReadyCycle = 3]:   $q1 = COPY %12:fpr128
+# CHECK-EMPTY: 
+# CHECK-NEXT: ********** INTERVALS **********
+# CHECK-NEXT: B0 [0B,48r:0)[192r,224r:1) 0@0B-phi 1@192r
+# CHECK-NEXT: B1 [0B,96r:0)[208r,224r:1) 0@0B-phi 1@208r
+# CHECK-NEXT: B2 [0B,88r:0) 0@0B-phi
+# CHECK-NEXT: %0 [48r,168r:0) 0@48r  weight:0.000000e+00
+# CHECK-NEXT: %1 [96r,120r:0) 0@96r  weight:0.000000e+00
+# CHECK-NEXT: %2 [88r,112r:0) 0@88r  weight:0.000000e+00
+# CHECK-NEXT: %3 [104r,176r:0) 0@104r  weight:0.000000e+00
+# CHECK-NEXT: %6 [80r,120r:0) 0@80r  weight:0.000000e+00
+# CHECK-NEXT: %7 [112r,160r:0) 0@112r  weight:0.000000e+00
+# CHECK-NEXT: %8 [120r,128r:0) 0@120r  weight:0.000000e+00
+# CHECK-NEXT: %9 [128r,168r:0) 0@128r  weight:0.000000e+00
+# CHECK-NEXT: %10 [168r,192r:0) 0@168r  weight:0.000000e+00
+# CHECK-NEXT: %11 [160r,176r:0) 0@160r  weight:0.000000e+00
+# CHECK-NEXT: %12 [176r,208r:0) 0@176r  weight:0.000000e+00
+# CHECK-NEXT: RegMasks:
+# CHECK-NEXT: ********** MACHINEINSTRS **********
+# CHECK-NEXT: # Machine code for function umull_and_v8i32: IsSSA, NoPHIs, TracksLiveness
+# CHECK-NEXT: Function Live Ins: $q0 in %0, $q1 in %1, $q2 in %2
+# CHECK-EMPTY: 
+# CHECK-NEXT: 0B	bb.0.entry:
+# CHECK-NEXT: 	  liveins: $q0, $q1, $q2
+# CHECK-NEXT: 48B	  %0:fpr128 = COPY $q0
+# CHECK-NEXT: 80B	  %6:fpr128 = MOVIv2d_ns 17
+# CHECK-NEXT: 88B	  %2:fpr128 = COPY $q2
+# CHECK-NEXT: 96B	  %1:fpr128 = COPY $q1
+# CHECK-NEXT: 104B	  %3:fpr128 = EXTv16i8 %0:fpr128, %0:fpr128, 8
+# CHECK-NEXT: 112B	  %7:fpr128 = ANDv16i8 %2:fpr128, %6:fpr128
+# CHECK-NEXT: 120B	  %8:fpr128 = ANDv16i8 %1:fpr128, %6:fpr128
+# CHECK-NEXT: 128B	  %9:fpr64 = XTNv4i16 %8:fpr128
+# CHECK-NEXT: 160B	  %11:fpr64 = XTNv4i16 %7:fpr128
+# CHECK-NEXT: 168B	  %10:fpr128 = UMULLv4i16_v4i32 %0.dsub:fpr128, %9:fpr64
+# CHECK-NEXT: 176B	  %12:fpr128 = UMULLv4i16_v4i32 %3.dsub:fpr128, %11:fpr64
+# CHECK-NEXT: 192B	  $q0 = COPY %10:fpr128
+# CHECK-NEXT: 208B	  $q1 = COPY %12:fpr128
+# CHECK-NEXT: 224B	  RET_ReallyLR implicit $q0, implicit $q1
+# CHECK-EMPTY: 
+# CHECK-NEXT: # End machine code for function umull_and_v8i32.
diff --git a/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-02.mir b/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-02.mir
new file mode 100644
index 0000000000000..0f5dc000c50b1
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-02.mir
@@ -0,0 +1,479 @@
+# RUN: llc -mtriple=aarch64-none-linux-gnu -mcpu=cortex-a55  \
+# RUN:  -run-pass=machine-scheduler -debug-only=machine-scheduler -o - %s \
+# RUN:  -misched-bottomup=true -sched-print-cycles=true \
+# RUN:  -misched-dump-reserved-cycles=true -misched-detail-resource-booking=true\
+# RUN:  -misched-dump-schedule-trace=true -misched-dump-schedule-trace-col-width=4 \
+# RUN:  2>&1 | FileCheck %s
+
+# REQUIRES: asserts, aarch64-registered-target
+---
+name: f
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    liveins: $x0, $x1, $x2
+    $x3 = ADDXrr $x0, $x0
+    $x4 = ADDXrr $x1, $x1
+    $x5 = ADDXrr $x2, $x2
+    
+# CHECK-LABEL: Before MISched:
+# CHECK-NEXT: # Machine code for function f: IsSSA, NoPHIs, TracksLiveness, NoVRegs
+# CHECK-EMPTY: 
+# CHECK-NEXT: bb.0:
+# CHECK-NEXT:   liveins: $x0, $x1, $x2
+# CHECK-NEXT:   $x3 = ADDXrr $x0, $x0
+# CHECK-NEXT:   $x4 = ADDXrr $x1, $x1
+# CHECK-NEXT:   $x5 = ADDXrr $x2, $x2
+# CHECK-EMPTY: 
+# CHECK-NEXT: # End machine code for function f.
+# CHECK-EMPTY: 
+# CHECK-NEXT: ********** MI Scheduling **********
+# CHECK-NEXT: f:%bb.0 
+# CHECK-NEXT:   From: $x3 = ADDXrr $x0, $x0
+# CHECK-NEXT:     To: End
+# CHECK-NEXT:  RegionInstrs: 3
+# CHECK-NEXT: ScheduleDAGMILive::schedule starting
+# CHECK-NEXT: GenericScheduler RegionPolicy:  ShouldTrackPressure=0 OnlyTopDown=0 OnlyBottomUp=1
+# CHECK-NEXT: Disabled scoreboard hazard recognizer
+# CHECK-NEXT: Disabled scoreboard hazard recognizer
+# CHECK-NEXT: SU(0) [TopReadyCycle = 0, BottomReadyCycle = 0]:   $x3 = ADDXrr $x0, $x0
+# CHECK-NEXT:   # preds left       : 0
+# CHECK-NEXT:   # succs left       : 0
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 3
+# CHECK-NEXT:   Depth              : 0
+# CHECK-NEXT:   Height             : 0
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(1) [TopReadyCycle = 0, BottomReadyCycle = 0]:   $x4 = ADDXrr $x1, $x1
+# CHECK-NEXT:   # preds left       : 0
+# CHECK-NEXT:   # succs left       : 0
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 3
+# CHECK-NEXT:   Depth              : 0
+# CHECK-NEXT:   Height             : 0
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT: SU(2) [TopReadyCycle = 0, BottomReadyCycle = 0]:   $x5 = ADDXrr $x2, $x2
+# CHECK-NEXT:   # preds left       : 0
+# CHECK-NEXT:   # succs left       : 0
+# CHECK-NEXT:   # rdefs left       : 0
+# CHECK-NEXT:   Latency            : 3
+# CHECK-NEXT:   Depth              : 0
+# CHECK-NEXT:   Height             : 0
+# CHECK-NEXT:   Single Issue       : false;
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT: Critical Path(GS-RR ): 0
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT: Queue BotQ.P: 
+# CHECK-NEXT: Queue BotQ.A: 2 1 0 
+# CHECK-NEXT:   Cand SU(2) ORDER                              
+# CHECK-NEXT: Pick Bot ORDER     
+# CHECK-NEXT: Scheduling SU(2) $x5 = ADDXrr $x2, $x2
+# CHECK-NEXT:   Ready @0c
+# CHECK-NEXT:   CortexA55UnitALU +1x1u
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT: BotQ.A @0c
+# CHECK-NEXT:   Retired: 1
+# CHECK-NEXT:   Executed: 0c
+# CHECK-NEXT:   Critical: 0c, 1 MOps
+# CHECK-NEXT:   ExpectedLatency: 0c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 0
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 0
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @1c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @0c
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 0
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @1c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @0c
+# CHECK-NEXT: Queue BotQ.P: 
+# CHECK-NEXT: Queue BotQ.A: 0 1 
+# CHECK-NEXT:   Cand SU(0) ORDER                              
+# CHECK-NEXT:   Cand SU(1) ORDER                              
+# CHECK-NEXT: Pick Bot ORDER     
+# CHECK-NEXT: Scheduling SU(1) $x4 = ADDXrr $x1, $x1
+# CHECK-NEXT:   Ready @0c
+# CHECK-NEXT:   CortexA55UnitALU +1x1u
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 0
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @1c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @0c
+# CHECK-NEXT:   Resource booking (@0c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 0
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@0c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT:   *** Max MOps 2 at cycle 0
+# CHECK-NEXT: Cycle: 1 BotQ.A
+# CHECK-NEXT: BotQ.A @1c
+# CHECK-NEXT:   Retired: 2
+# CHECK-NEXT:   Executed: 1c
+# CHECK-NEXT:   Critical: 1c, 2 MOps
+# CHECK-NEXT:   ExpectedLatency: 0c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 0
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT:   Resource booking (@1c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 0
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@1c): 
+# CHECK-NEXT:     Instance 0 available @1c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @0c
+# CHECK-NEXT: Queue BotQ.P: 
+# CHECK-NEXT: Queue BotQ.A: 0 
+# CHECK-NEXT: Scheduling SU(0) $x3 = ADDXrr $x0, $x0
+# CHECK-NEXT:   Ready @1c
+# CHECK-NEXT:   CortexA55UnitALU +1x1u
+# CHECK-NEXT:   Resource booking (@1c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 0
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@1c): 
+# CHECK-NEXT:     Instance 0 available @1c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[1] available @0c
+# CHECK-NEXT:   Resource booking (@1c): 
+# CHECK-NEXT: CortexA55UnitALU(0) = 0
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT:   getNextResourceCycle (@1c): 
+# CHECK-NEXT:     Instance 0 available @0c
+# CHECK-NEXT:     Instance 1 available @0c
+# CHECK-NEXT:     selecting CortexA55UnitALU[0] available @0c
+# CHECK-NEXT: BotQ.A @1c
+# CHECK-NEXT:   Retired: 3
+# CHECK-NEXT:   Executed: 1c
+# CHECK-NEXT:   Critical: 1c, 3 MOps
+# CHECK-NEXT:   ExpectedLatency: 0c
+# CHECK-NEXT:   - Latency limited.
+# CHECK-NEXT: CortexA55UnitALU(0) = 1
+# CHECK-NEXT: CortexA55UnitALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitB(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitDiv(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPALU(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPDIV(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitFPMAC(1) = 4294967295
+# CHECK-NEXT: CortexA55UnitLd(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitMAC(0) = 4294967295
+# CHECK-NEXT: CortexA55UnitSt(0) = 4294967295
+# CHECK-NEXT: ** ScheduleDAGMILive::schedule picking next node
+# CHECK-NEXT: *** Final schedule for %bb.0 ***
+# CHECK-NEXT:  * Schedule table (BottomUp):
+# CHECK-NEXT:   i: issue
+# CHECK-NEXT:   x: resource booked
+# CHECK-NEXT: Cycle              | 1 | 0 |
+# CHECK-NEXT: SU(0)              | i |   |
+# CHECK-NEXT:   CortexA55UnitALU | x |   | 
+# CHECK-NEXT: SU(1)              |   | i |
+# CHECK-NEXT:   CortexA55UnitALU |   | x | 
+# CHECK-NEXT: SU(2)              |   | i |
+# CHECK-NEXT:   CortexA55UnitALU |   | x | 
+# CHECK-NEXT: SU(0) [TopReadyCycle = 0, BottomReadyCycle = 1]:   $x3 = ADDXrr $x0, $x0
+# CHECK-NEXT: SU(1) [TopReadyCycle = 0, BottomReadyCycle = 0]:   $x4 = ADDXrr $x1, $x1
+# CHECK-NEXT: SU(2) [TopReadyCycle = 0, BottomReadyCycle = 0]:   $x5 = ADDXrr $x2, $x2
+# CHECK-EMPTY: 
+# CHECK-NEXT: ********** INTERVALS **********
+# CHECK-NEXT: W0 [0B,16r:0) 0@0B-phi
+# CHECK-NEXT: W1 [0B,32r:0) 0@0B-phi
+# CHECK-NEXT: W2 [0B,48r:0) 0@0B-phi
+# CHECK-NEXT: RegMasks:
+# CHECK-NEXT: ********** MACHINEINSTRS **********
+# CHECK-NEXT: # Machine code for function f: IsSSA, NoPHIs, TracksLiveness, NoVRegs
+# CHECK-EMPTY: 
+# CHECK-NEXT: 0B	bb.0:
+# CHECK-NEXT: 	  liveins: $x0, $x1, $x2
+# CHECK-NEXT: 16B	  $x3 = ADDXrr $x0, $x0
+# CHECK-NEXT: 32B	  $x4 = ADDXrr $x1, $x1
+# CHECK-NEXT: 48B	  $x5 = ADDXrr $x2, $x2
+# CHECK-EMPTY: 
+# CHECK-NEXT: # End machine code for function f.
\ No newline at end of file

From dec42ffa2869faab6384fd534d2d90072bf46db3 Mon Sep 17 00:00:00 2001
From: Ivan Kosarev <ivan.kosarev@amd.com>
Date: Tue, 20 Jun 2023 10:32:12 +0100
Subject: [PATCH 107/130] [AMDGPU][GFX11] Add test coverage for FMA
 instructions.

Reviewed By: arsenm

Differential Revision: https://reviews.llvm.org/D153269
---
 .../GlobalISel/combine-fma-add-fma-mul.ll     |  358 +++
 .../AMDGPU/GlobalISel/combine-fma-sub-mul.ll  |  250 ++
 .../CodeGen/AMDGPU/dagcombine-fma-fmad.ll     |  352 ++-
 llvm/test/CodeGen/AMDGPU/fma-combine.ll       | 2034 +++++++++++++++--
 llvm/test/CodeGen/AMDGPU/fma.f16.ll           |  107 +
 llvm/test/CodeGen/AMDGPU/fma.f64.ll           |   37 +-
 6 files changed, 2801 insertions(+), 337 deletions(-)

diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-fma-mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-fma-mul.ll
index 88ab58d2b2126..a4261adffeb2f 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-fma-mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-add-fma-mul.ll
@@ -3,6 +3,8 @@
 ; RUN: llc -global-isel -march=amdgcn -mcpu=gfx900 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX9-DENORM %s
 ; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -fp-contract=fast < %s | FileCheck -check-prefix=GFX10-CONTRACT %s
 ; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1100 -fp-contract=fast < %s | FileCheck -check-prefix=GFX11-CONTRACT %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1100 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX11-DENORM %s
 
 ; fadd (fma a, b, (fmul c, d)), e --> fma a, b, (fma c, d, e)
 ; fadd e, (fma a, b, (fmul c, d)) --> fma a, b, (fma c, d, e)
@@ -40,6 +42,26 @@ define float @test_f32_add_mul(float %a, float %b, float %c, float %d, float %e)
 ; GFX10-DENORM-NEXT:    v_fmac_f32_e32 v2, v0, v1
 ; GFX10-DENORM-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX10-DENORM-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-CONTRACT-LABEL: test_f32_add_mul:
+; GFX11-CONTRACT:       ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-CONTRACT-NEXT:    v_fma_f32 v2, v2, v3, v4
+; GFX11-CONTRACT-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-CONTRACT-NEXT:    v_fmac_f32_e32 v2, v0, v1
+; GFX11-CONTRACT-NEXT:    v_mov_b32_e32 v0, v2
+; GFX11-CONTRACT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-DENORM-LABEL: test_f32_add_mul:
+; GFX11-DENORM:       ; %bb.0: ; %.entry
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-DENORM-NEXT:    v_fma_f32 v2, v2, v3, v4
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-DENORM-NEXT:    v_fmac_f32_e32 v2, v0, v1
+; GFX11-DENORM-NEXT:    v_mov_b32_e32 v0, v2
+; GFX11-DENORM-NEXT:    s_setpc_b64 s[30:31]
 .entry:
   %x = fmul fast float %c, %d
   %y = call fast float @llvm.fmuladd.f32(float %a, float %b, float %x)
@@ -80,6 +102,26 @@ define float @test_f32_add_mul_rhs(float %a, float %b, float %c, float %d, float
 ; GFX10-DENORM-NEXT:    v_fmac_f32_e32 v2, v0, v1
 ; GFX10-DENORM-NEXT:    v_mov_b32_e32 v0, v2
 ; GFX10-DENORM-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-CONTRACT-LABEL: test_f32_add_mul_rhs:
+; GFX11-CONTRACT:       ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-CONTRACT-NEXT:    v_fma_f32 v2, v2, v3, v4
+; GFX11-CONTRACT-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-CONTRACT-NEXT:    v_fmac_f32_e32 v2, v0, v1
+; GFX11-CONTRACT-NEXT:    v_mov_b32_e32 v0, v2
+; GFX11-CONTRACT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-DENORM-LABEL: test_f32_add_mul_rhs:
+; GFX11-DENORM:       ; %bb.0: ; %.entry
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-DENORM-NEXT:    v_fma_f32 v2, v2, v3, v4
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-DENORM-NEXT:    v_fmac_f32_e32 v2, v0, v1
+; GFX11-DENORM-NEXT:    v_mov_b32_e32 v0, v2
+; GFX11-DENORM-NEXT:    s_setpc_b64 s[30:31]
 .entry:
   %x = fmul fast float %c, %d
   %y = call fast float @llvm.fmuladd.f32(float %a, float %b, float %x)
@@ -121,6 +163,27 @@ define half @test_half_add_mul(half %a, half %b, half %c, half %d, half %e) {
 ; GFX10-DENORM-NEXT:    v_add_f16_e32 v0, v0, v2
 ; GFX10-DENORM-NEXT:    v_add_f16_e32 v0, v0, v4
 ; GFX10-DENORM-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-CONTRACT-LABEL: test_half_add_mul:
+; GFX11-CONTRACT:       ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-CONTRACT-NEXT:    v_fma_f16 v2, v2, v3, v4
+; GFX11-CONTRACT-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-CONTRACT-NEXT:    v_fmac_f16_e32 v2, v0, v1
+; GFX11-CONTRACT-NEXT:    v_mov_b32_e32 v0, v2
+; GFX11-CONTRACT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-DENORM-LABEL: test_half_add_mul:
+; GFX11-DENORM:       ; %bb.0: ; %.entry
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-DENORM-NEXT:    v_mul_f16_e32 v2, v2, v3
+; GFX11-DENORM-NEXT:    v_mul_f16_e32 v0, v0, v1
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-DENORM-NEXT:    v_add_f16_e32 v0, v0, v2
+; GFX11-DENORM-NEXT:    v_add_f16_e32 v0, v0, v4
+; GFX11-DENORM-NEXT:    s_setpc_b64 s[30:31]
 .entry:
   %x = fmul fast half %c, %d
   %y = call fast half @llvm.fmuladd.f16(half %a, half %b, half %x)
@@ -162,6 +225,27 @@ define half @test_half_add_mul_rhs(half %a, half %b, half %c, half %d, half %e)
 ; GFX10-DENORM-NEXT:    v_add_f16_e32 v0, v0, v2
 ; GFX10-DENORM-NEXT:    v_add_f16_e32 v0, v4, v0
 ; GFX10-DENORM-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-CONTRACT-LABEL: test_half_add_mul_rhs:
+; GFX11-CONTRACT:       ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-CONTRACT-NEXT:    v_fma_f16 v2, v2, v3, v4
+; GFX11-CONTRACT-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-CONTRACT-NEXT:    v_fmac_f16_e32 v2, v0, v1
+; GFX11-CONTRACT-NEXT:    v_mov_b32_e32 v0, v2
+; GFX11-CONTRACT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-DENORM-LABEL: test_half_add_mul_rhs:
+; GFX11-DENORM:       ; %bb.0: ; %.entry
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-DENORM-NEXT:    v_mul_f16_e32 v2, v2, v3
+; GFX11-DENORM-NEXT:    v_mul_f16_e32 v0, v0, v1
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-DENORM-NEXT:    v_add_f16_e32 v0, v0, v2
+; GFX11-DENORM-NEXT:    v_add_f16_e32 v0, v4, v0
+; GFX11-DENORM-NEXT:    s_setpc_b64 s[30:31]
 .entry:
   %x = fmul fast half %c, %d
   %y = call fast half @llvm.fmuladd.f16(half %a, half %b, half %x)
@@ -199,6 +283,24 @@ define double @test_double_add_mul(double %a, double %b, double %c, double %d, d
 ; GFX10-DENORM-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[8:9]
 ; GFX10-DENORM-NEXT:    v_fma_f64 v[0:1], v[0:1], v[2:3], v[4:5]
 ; GFX10-DENORM-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-CONTRACT-LABEL: test_double_add_mul:
+; GFX11-CONTRACT:       ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[8:9]
+; GFX11-CONTRACT-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[0:1], v[0:1], v[2:3], v[4:5]
+; GFX11-CONTRACT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-DENORM-LABEL: test_double_add_mul:
+; GFX11-DENORM:       ; %bb.0: ; %.entry
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-DENORM-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[8:9]
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-DENORM-NEXT:    v_fma_f64 v[0:1], v[0:1], v[2:3], v[4:5]
+; GFX11-DENORM-NEXT:    s_setpc_b64 s[30:31]
 .entry:
   %x = fmul fast double %c, %d
   %y = call fast double @llvm.fmuladd.f64(double %a, double %b, double %x)
@@ -236,6 +338,24 @@ define double @test_double_add_mul_rhs(double %a, double %b, double %c, double %
 ; GFX10-DENORM-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[8:9]
 ; GFX10-DENORM-NEXT:    v_fma_f64 v[0:1], v[0:1], v[2:3], v[4:5]
 ; GFX10-DENORM-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-CONTRACT-LABEL: test_double_add_mul_rhs:
+; GFX11-CONTRACT:       ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[8:9]
+; GFX11-CONTRACT-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[0:1], v[0:1], v[2:3], v[4:5]
+; GFX11-CONTRACT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-DENORM-LABEL: test_double_add_mul_rhs:
+; GFX11-DENORM:       ; %bb.0: ; %.entry
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-DENORM-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[8:9]
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-DENORM-NEXT:    v_fma_f64 v[0:1], v[0:1], v[2:3], v[4:5]
+; GFX11-DENORM-NEXT:    s_setpc_b64 s[30:31]
 .entry:
   %x = fmul fast double %c, %d
   %y = call fast double @llvm.fmuladd.f64(double %a, double %b, double %x)
@@ -309,6 +429,38 @@ define <4 x float> @test_v4f32_add_mul(<4 x float> %a, <4 x float> %b, <4 x floa
 ; GFX10-DENORM-NEXT:    v_mov_b32_e32 v2, v10
 ; GFX10-DENORM-NEXT:    v_mov_b32_e32 v3, v11
 ; GFX10-DENORM-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-CONTRACT-LABEL: test_v4f32_add_mul:
+; GFX11-CONTRACT:       ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-CONTRACT-NEXT:    v_fma_f32 v8, v8, v12, v16
+; GFX11-CONTRACT-NEXT:    v_fma_f32 v9, v9, v13, v17
+; GFX11-CONTRACT-NEXT:    v_fma_f32 v10, v10, v14, v18
+; GFX11-CONTRACT-NEXT:    v_fma_f32 v11, v11, v15, v19
+; GFX11-CONTRACT-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-CONTRACT-NEXT:    v_dual_fmac_f32 v8, v0, v4 :: v_dual_fmac_f32 v9, v1, v5
+; GFX11-CONTRACT-NEXT:    v_dual_fmac_f32 v10, v2, v6 :: v_dual_fmac_f32 v11, v3, v7
+; GFX11-CONTRACT-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-CONTRACT-NEXT:    v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9
+; GFX11-CONTRACT-NEXT:    v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11
+; GFX11-CONTRACT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-DENORM-LABEL: test_v4f32_add_mul:
+; GFX11-DENORM:       ; %bb.0: ; %.entry
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-DENORM-NEXT:    v_fma_f32 v8, v8, v12, v16
+; GFX11-DENORM-NEXT:    v_fma_f32 v9, v9, v13, v17
+; GFX11-DENORM-NEXT:    v_fma_f32 v10, v10, v14, v18
+; GFX11-DENORM-NEXT:    v_fma_f32 v11, v11, v15, v19
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-DENORM-NEXT:    v_dual_fmac_f32 v8, v0, v4 :: v_dual_fmac_f32 v9, v1, v5
+; GFX11-DENORM-NEXT:    v_dual_fmac_f32 v10, v2, v6 :: v_dual_fmac_f32 v11, v3, v7
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-DENORM-NEXT:    v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9
+; GFX11-DENORM-NEXT:    v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11
+; GFX11-DENORM-NEXT:    s_setpc_b64 s[30:31]
 .entry:
   %x = fmul fast <4 x float> %c, %d
   %y = call fast <4 x float> @llvm.fmuladd.v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %x)
@@ -382,6 +534,38 @@ define <4 x float> @test_v4f32_add_mul_rhs(<4 x float> %a, <4 x float> %b, <4 x
 ; GFX10-DENORM-NEXT:    v_mov_b32_e32 v2, v10
 ; GFX10-DENORM-NEXT:    v_mov_b32_e32 v3, v11
 ; GFX10-DENORM-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-CONTRACT-LABEL: test_v4f32_add_mul_rhs:
+; GFX11-CONTRACT:       ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-CONTRACT-NEXT:    v_fma_f32 v8, v8, v12, v16
+; GFX11-CONTRACT-NEXT:    v_fma_f32 v9, v9, v13, v17
+; GFX11-CONTRACT-NEXT:    v_fma_f32 v10, v10, v14, v18
+; GFX11-CONTRACT-NEXT:    v_fma_f32 v11, v11, v15, v19
+; GFX11-CONTRACT-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-CONTRACT-NEXT:    v_dual_fmac_f32 v8, v0, v4 :: v_dual_fmac_f32 v9, v1, v5
+; GFX11-CONTRACT-NEXT:    v_dual_fmac_f32 v10, v2, v6 :: v_dual_fmac_f32 v11, v3, v7
+; GFX11-CONTRACT-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-CONTRACT-NEXT:    v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9
+; GFX11-CONTRACT-NEXT:    v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11
+; GFX11-CONTRACT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-DENORM-LABEL: test_v4f32_add_mul_rhs:
+; GFX11-DENORM:       ; %bb.0: ; %.entry
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-DENORM-NEXT:    v_fma_f32 v8, v8, v12, v16
+; GFX11-DENORM-NEXT:    v_fma_f32 v9, v9, v13, v17
+; GFX11-DENORM-NEXT:    v_fma_f32 v10, v10, v14, v18
+; GFX11-DENORM-NEXT:    v_fma_f32 v11, v11, v15, v19
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-DENORM-NEXT:    v_dual_fmac_f32 v8, v0, v4 :: v_dual_fmac_f32 v9, v1, v5
+; GFX11-DENORM-NEXT:    v_dual_fmac_f32 v10, v2, v6 :: v_dual_fmac_f32 v11, v3, v7
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-DENORM-NEXT:    v_dual_mov_b32 v0, v8 :: v_dual_mov_b32 v1, v9
+; GFX11-DENORM-NEXT:    v_dual_mov_b32 v2, v10 :: v_dual_mov_b32 v3, v11
+; GFX11-DENORM-NEXT:    s_setpc_b64 s[30:31]
 .entry:
   %x = fmul fast <4 x float> %c, %d
   %y = call fast <4 x float> @llvm.fmuladd.v4f32(<4 x float> %a, <4 x float> %b, <4 x float> %x)
@@ -435,6 +619,33 @@ define <4 x half> @test_f16_add_mul(<4 x half> %a, <4 x half> %b, <4 x half> %c,
 ; GFX10-DENORM-NEXT:    v_pk_add_f16 v0, v0, v8
 ; GFX10-DENORM-NEXT:    v_pk_add_f16 v1, v1, v9
 ; GFX10-DENORM-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-CONTRACT-LABEL: test_f16_add_mul:
+; GFX11-CONTRACT:       ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-CONTRACT-NEXT:    v_pk_fma_f16 v4, v4, v6, v8
+; GFX11-CONTRACT-NEXT:    v_pk_fma_f16 v5, v5, v7, v9
+; GFX11-CONTRACT-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-CONTRACT-NEXT:    v_pk_fma_f16 v0, v0, v2, v4
+; GFX11-CONTRACT-NEXT:    v_pk_fma_f16 v1, v1, v3, v5
+; GFX11-CONTRACT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-DENORM-LABEL: test_f16_add_mul:
+; GFX11-DENORM:       ; %bb.0: ; %.entry
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-DENORM-NEXT:    v_pk_mul_f16 v4, v4, v6
+; GFX11-DENORM-NEXT:    v_pk_mul_f16 v5, v5, v7
+; GFX11-DENORM-NEXT:    v_pk_mul_f16 v0, v0, v2
+; GFX11-DENORM-NEXT:    v_pk_mul_f16 v1, v1, v3
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-DENORM-NEXT:    v_pk_add_f16 v0, v0, v4
+; GFX11-DENORM-NEXT:    v_pk_add_f16 v1, v1, v5
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-DENORM-NEXT:    v_pk_add_f16 v0, v0, v8
+; GFX11-DENORM-NEXT:    v_pk_add_f16 v1, v1, v9
+; GFX11-DENORM-NEXT:    s_setpc_b64 s[30:31]
 .entry:
   %x = fmul fast <4 x half> %c, %d
   %y = call fast <4 x half> @llvm.fmuladd.v4f16(<4 x half> %a, <4 x half> %b, <4 x half> %x)
@@ -488,6 +699,33 @@ define <4 x half> @test_f16_add_mul_rhs(<4 x half> %a, <4 x half> %b, <4 x half>
 ; GFX10-DENORM-NEXT:    v_pk_add_f16 v0, v8, v0
 ; GFX10-DENORM-NEXT:    v_pk_add_f16 v1, v9, v1
 ; GFX10-DENORM-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-CONTRACT-LABEL: test_f16_add_mul_rhs:
+; GFX11-CONTRACT:       ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-CONTRACT-NEXT:    v_pk_fma_f16 v4, v4, v6, v8
+; GFX11-CONTRACT-NEXT:    v_pk_fma_f16 v5, v5, v7, v9
+; GFX11-CONTRACT-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-CONTRACT-NEXT:    v_pk_fma_f16 v0, v0, v2, v4
+; GFX11-CONTRACT-NEXT:    v_pk_fma_f16 v1, v1, v3, v5
+; GFX11-CONTRACT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-DENORM-LABEL: test_f16_add_mul_rhs:
+; GFX11-DENORM:       ; %bb.0: ; %.entry
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-DENORM-NEXT:    v_pk_mul_f16 v4, v4, v6
+; GFX11-DENORM-NEXT:    v_pk_mul_f16 v5, v5, v7
+; GFX11-DENORM-NEXT:    v_pk_mul_f16 v0, v0, v2
+; GFX11-DENORM-NEXT:    v_pk_mul_f16 v1, v1, v3
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-DENORM-NEXT:    v_pk_add_f16 v0, v0, v4
+; GFX11-DENORM-NEXT:    v_pk_add_f16 v1, v1, v5
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-DENORM-NEXT:    v_pk_add_f16 v0, v8, v0
+; GFX11-DENORM-NEXT:    v_pk_add_f16 v1, v9, v1
+; GFX11-DENORM-NEXT:    s_setpc_b64 s[30:31]
 .entry:
   %x = fmul fast <4 x half> %c, %d
   %y = call fast <4 x half> @llvm.fmuladd.v4f16(<4 x half> %a, <4 x half> %b, <4 x half> %x)
@@ -603,6 +841,66 @@ define <4 x double> @test_f64_add_mul(<4 x double> %a, <4 x double> %b, <4 x dou
 ; GFX10-DENORM-NEXT:    v_fma_f64 v[4:5], v[4:5], v[12:13], v[20:21]
 ; GFX10-DENORM-NEXT:    v_fma_f64 v[6:7], v[6:7], v[14:15], v[22:23]
 ; GFX10-DENORM-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-CONTRACT-LABEL: test_f64_add_mul:
+; GFX11-CONTRACT:       ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-CONTRACT-NEXT:    s_clause 0x8
+; GFX11-CONTRACT-NEXT:    scratch_load_b32 v31, off, s32
+; GFX11-CONTRACT-NEXT:    scratch_load_b32 v32, off, s32 offset:4
+; GFX11-CONTRACT-NEXT:    scratch_load_b32 v33, off, s32 offset:8
+; GFX11-CONTRACT-NEXT:    scratch_load_b32 v34, off, s32 offset:12
+; GFX11-CONTRACT-NEXT:    scratch_load_b32 v35, off, s32 offset:16
+; GFX11-CONTRACT-NEXT:    scratch_load_b32 v36, off, s32 offset:20
+; GFX11-CONTRACT-NEXT:    scratch_load_b32 v37, off, s32 offset:24
+; GFX11-CONTRACT-NEXT:    scratch_load_b32 v38, off, s32 offset:28
+; GFX11-CONTRACT-NEXT:    scratch_load_b32 v39, off, s32 offset:32
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(6)
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[16:17], v[16:17], v[24:25], v[32:33]
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(4)
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[18:19], v[18:19], v[26:27], v[34:35]
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(2)
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[20:21], v[20:21], v[28:29], v[36:37]
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[22:23], v[22:23], v[30:31], v[38:39]
+; GFX11-CONTRACT-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[0:1], v[0:1], v[8:9], v[16:17]
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[2:3], v[2:3], v[10:11], v[18:19]
+; GFX11-CONTRACT-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[4:5], v[4:5], v[12:13], v[20:21]
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[6:7], v[6:7], v[14:15], v[22:23]
+; GFX11-CONTRACT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-DENORM-LABEL: test_f64_add_mul:
+; GFX11-DENORM:       ; %bb.0: ; %.entry
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-DENORM-NEXT:    s_clause 0x8
+; GFX11-DENORM-NEXT:    scratch_load_b32 v31, off, s32
+; GFX11-DENORM-NEXT:    scratch_load_b32 v32, off, s32 offset:4
+; GFX11-DENORM-NEXT:    scratch_load_b32 v33, off, s32 offset:8
+; GFX11-DENORM-NEXT:    scratch_load_b32 v34, off, s32 offset:12
+; GFX11-DENORM-NEXT:    scratch_load_b32 v35, off, s32 offset:16
+; GFX11-DENORM-NEXT:    scratch_load_b32 v36, off, s32 offset:20
+; GFX11-DENORM-NEXT:    scratch_load_b32 v37, off, s32 offset:24
+; GFX11-DENORM-NEXT:    scratch_load_b32 v38, off, s32 offset:28
+; GFX11-DENORM-NEXT:    scratch_load_b32 v39, off, s32 offset:32
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(6)
+; GFX11-DENORM-NEXT:    v_fma_f64 v[16:17], v[16:17], v[24:25], v[32:33]
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(4)
+; GFX11-DENORM-NEXT:    v_fma_f64 v[18:19], v[18:19], v[26:27], v[34:35]
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(2)
+; GFX11-DENORM-NEXT:    v_fma_f64 v[20:21], v[20:21], v[28:29], v[36:37]
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-DENORM-NEXT:    v_fma_f64 v[22:23], v[22:23], v[30:31], v[38:39]
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-DENORM-NEXT:    v_fma_f64 v[0:1], v[0:1], v[8:9], v[16:17]
+; GFX11-DENORM-NEXT:    v_fma_f64 v[2:3], v[2:3], v[10:11], v[18:19]
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-DENORM-NEXT:    v_fma_f64 v[4:5], v[4:5], v[12:13], v[20:21]
+; GFX11-DENORM-NEXT:    v_fma_f64 v[6:7], v[6:7], v[14:15], v[22:23]
+; GFX11-DENORM-NEXT:    s_setpc_b64 s[30:31]
 .entry:
   %x = fmul fast <4 x double> %c, %d
   %y = call fast <4 x double> @llvm.fmuladd.v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %x)
@@ -718,6 +1016,66 @@ define <4 x double> @test_f64_add_mul_rhs(<4 x double> %a, <4 x double> %b, <4 x
 ; GFX10-DENORM-NEXT:    v_fma_f64 v[4:5], v[4:5], v[12:13], v[20:21]
 ; GFX10-DENORM-NEXT:    v_fma_f64 v[6:7], v[6:7], v[14:15], v[22:23]
 ; GFX10-DENORM-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-CONTRACT-LABEL: test_f64_add_mul_rhs:
+; GFX11-CONTRACT:       ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-CONTRACT-NEXT:    s_clause 0x8
+; GFX11-CONTRACT-NEXT:    scratch_load_b32 v31, off, s32
+; GFX11-CONTRACT-NEXT:    scratch_load_b32 v32, off, s32 offset:4
+; GFX11-CONTRACT-NEXT:    scratch_load_b32 v33, off, s32 offset:8
+; GFX11-CONTRACT-NEXT:    scratch_load_b32 v34, off, s32 offset:12
+; GFX11-CONTRACT-NEXT:    scratch_load_b32 v35, off, s32 offset:16
+; GFX11-CONTRACT-NEXT:    scratch_load_b32 v36, off, s32 offset:20
+; GFX11-CONTRACT-NEXT:    scratch_load_b32 v37, off, s32 offset:24
+; GFX11-CONTRACT-NEXT:    scratch_load_b32 v38, off, s32 offset:28
+; GFX11-CONTRACT-NEXT:    scratch_load_b32 v39, off, s32 offset:32
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(6)
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[16:17], v[16:17], v[24:25], v[32:33]
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(4)
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[18:19], v[18:19], v[26:27], v[34:35]
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(2)
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[20:21], v[20:21], v[28:29], v[36:37]
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[22:23], v[22:23], v[30:31], v[38:39]
+; GFX11-CONTRACT-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[0:1], v[0:1], v[8:9], v[16:17]
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[2:3], v[2:3], v[10:11], v[18:19]
+; GFX11-CONTRACT-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[4:5], v[4:5], v[12:13], v[20:21]
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[6:7], v[6:7], v[14:15], v[22:23]
+; GFX11-CONTRACT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-DENORM-LABEL: test_f64_add_mul_rhs:
+; GFX11-DENORM:       ; %bb.0: ; %.entry
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-DENORM-NEXT:    s_clause 0x8
+; GFX11-DENORM-NEXT:    scratch_load_b32 v31, off, s32
+; GFX11-DENORM-NEXT:    scratch_load_b32 v32, off, s32 offset:4
+; GFX11-DENORM-NEXT:    scratch_load_b32 v33, off, s32 offset:8
+; GFX11-DENORM-NEXT:    scratch_load_b32 v34, off, s32 offset:12
+; GFX11-DENORM-NEXT:    scratch_load_b32 v35, off, s32 offset:16
+; GFX11-DENORM-NEXT:    scratch_load_b32 v36, off, s32 offset:20
+; GFX11-DENORM-NEXT:    scratch_load_b32 v37, off, s32 offset:24
+; GFX11-DENORM-NEXT:    scratch_load_b32 v38, off, s32 offset:28
+; GFX11-DENORM-NEXT:    scratch_load_b32 v39, off, s32 offset:32
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(6)
+; GFX11-DENORM-NEXT:    v_fma_f64 v[16:17], v[16:17], v[24:25], v[32:33]
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(4)
+; GFX11-DENORM-NEXT:    v_fma_f64 v[18:19], v[18:19], v[26:27], v[34:35]
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(2)
+; GFX11-DENORM-NEXT:    v_fma_f64 v[20:21], v[20:21], v[28:29], v[36:37]
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-DENORM-NEXT:    v_fma_f64 v[22:23], v[22:23], v[30:31], v[38:39]
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-DENORM-NEXT:    v_fma_f64 v[0:1], v[0:1], v[8:9], v[16:17]
+; GFX11-DENORM-NEXT:    v_fma_f64 v[2:3], v[2:3], v[10:11], v[18:19]
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-DENORM-NEXT:    v_fma_f64 v[4:5], v[4:5], v[12:13], v[20:21]
+; GFX11-DENORM-NEXT:    v_fma_f64 v[6:7], v[6:7], v[14:15], v[22:23]
+; GFX11-DENORM-NEXT:    s_setpc_b64 s[30:31]
 .entry:
   %x = fmul fast <4 x double> %c, %d
   %y = call fast <4 x double> @llvm.fmuladd.v4f64(<4 x double> %a, <4 x double> %b, <4 x double> %x)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-sub-mul.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-sub-mul.ll
index 2e9a374caef07..11a3b96d8255a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-sub-mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-fma-sub-mul.ll
@@ -5,6 +5,8 @@
 ; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
 ; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 -fp-contract=fast < %s | FileCheck -check-prefix=GFX10-CONTRACT %s
 ; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1010 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX10-DENORM %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1100 -fp-contract=fast < %s | FileCheck -check-prefix=GFX11-CONTRACT %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1100 --denormal-fp-math=preserve-sign < %s | FileCheck -check-prefix=GFX11-DENORM %s
 
 ; fold (fsub (fmul x, y), z) -> (fma x, y, (fneg z))
 ; fold (fsub x, (fmul y, z)) -> (fma (fneg y), z, x)
@@ -50,6 +52,22 @@ define float @test_f32_sub_mul(float %x, float %y, float %z) {
 ; GFX10-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-DENORM-NEXT:    v_mad_f32 v0, v0, v1, -v2
 ; GFX10-DENORM-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-CONTRACT-LABEL: test_f32_sub_mul:
+; GFX11-CONTRACT:       ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-CONTRACT-NEXT:    v_fma_f32 v0, v0, v1, -v2
+; GFX11-CONTRACT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-DENORM-LABEL: test_f32_sub_mul:
+; GFX11-DENORM:       ; %bb.0: ; %.entry
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-DENORM-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-DENORM-NEXT:    v_sub_f32_e32 v0, v0, v2
+; GFX11-DENORM-NEXT:    s_setpc_b64 s[30:31]
 .entry:
   %a = fmul float %x, %y
   %b = fsub float %a, %z
@@ -97,6 +115,22 @@ define float @test_f32_sub_mul_rhs(float %x, float %y, float %z) {
 ; GFX10-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-DENORM-NEXT:    v_mad_f32 v0, -v0, v1, v2
 ; GFX10-DENORM-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-CONTRACT-LABEL: test_f32_sub_mul_rhs:
+; GFX11-CONTRACT:       ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-CONTRACT-NEXT:    v_fma_f32 v0, -v0, v1, v2
+; GFX11-CONTRACT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-DENORM-LABEL: test_f32_sub_mul_rhs:
+; GFX11-DENORM:       ; %bb.0: ; %.entry
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-DENORM-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-DENORM-NEXT:    v_sub_f32_e32 v0, v2, v0
+; GFX11-DENORM-NEXT:    s_setpc_b64 s[30:31]
 .entry:
   %a = fmul float %x, %y
   %b = fsub float %z, %a
@@ -145,6 +179,22 @@ define half @test_half_sub_mul(half %x, half %y, half %z) {
 ; GFX10-DENORM-NEXT:    v_mul_f16_e32 v0, v0, v1
 ; GFX10-DENORM-NEXT:    v_sub_f16_e32 v0, v0, v2
 ; GFX10-DENORM-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-CONTRACT-LABEL: test_half_sub_mul:
+; GFX11-CONTRACT:       ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-CONTRACT-NEXT:    v_fma_f16 v0, v0, v1, -v2
+; GFX11-CONTRACT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-DENORM-LABEL: test_half_sub_mul:
+; GFX11-DENORM:       ; %bb.0: ; %.entry
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-DENORM-NEXT:    v_mul_f16_e32 v0, v0, v1
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-DENORM-NEXT:    v_sub_f16_e32 v0, v0, v2
+; GFX11-DENORM-NEXT:    s_setpc_b64 s[30:31]
 .entry:
   %a = fmul half %x, %y
   %b = fsub half %a, %z
@@ -193,6 +243,22 @@ define half @test_half_sub_mul_rhs(half %x, half %y, half %z) {
 ; GFX10-DENORM-NEXT:    v_mul_f16_e32 v0, v0, v1
 ; GFX10-DENORM-NEXT:    v_sub_f16_e32 v0, v2, v0
 ; GFX10-DENORM-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-CONTRACT-LABEL: test_half_sub_mul_rhs:
+; GFX11-CONTRACT:       ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-CONTRACT-NEXT:    v_fma_f16 v0, -v0, v1, v2
+; GFX11-CONTRACT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-DENORM-LABEL: test_half_sub_mul_rhs:
+; GFX11-DENORM:       ; %bb.0: ; %.entry
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-DENORM-NEXT:    v_mul_f16_e32 v0, v0, v1
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-DENORM-NEXT:    v_sub_f16_e32 v0, v2, v0
+; GFX11-DENORM-NEXT:    s_setpc_b64 s[30:31]
 .entry:
   %a = fmul half %x, %y
   %b = fsub half %z, %a
@@ -242,6 +308,22 @@ define double @test_double_sub_mul(double %x, double %y, double %z) {
 ; GFX10-DENORM-NEXT:    v_mul_f64 v[0:1], v[0:1], v[2:3]
 ; GFX10-DENORM-NEXT:    v_add_f64 v[0:1], v[0:1], -v[4:5]
 ; GFX10-DENORM-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-CONTRACT-LABEL: test_double_sub_mul:
+; GFX11-CONTRACT:       ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[0:1], v[0:1], v[2:3], -v[4:5]
+; GFX11-CONTRACT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-DENORM-LABEL: test_double_sub_mul:
+; GFX11-DENORM:       ; %bb.0: ; %.entry
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-DENORM-NEXT:    v_mul_f64 v[0:1], v[0:1], v[2:3]
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-DENORM-NEXT:    v_add_f64 v[0:1], v[0:1], -v[4:5]
+; GFX11-DENORM-NEXT:    s_setpc_b64 s[30:31]
 .entry:
   %a = fmul double %x, %y
   %b = fsub double %a, %z
@@ -291,6 +373,22 @@ define double @test_double_sub_mul_rhs(double %x, double %y, double %z) {
 ; GFX10-DENORM-NEXT:    v_mul_f64 v[0:1], v[0:1], v[2:3]
 ; GFX10-DENORM-NEXT:    v_add_f64 v[0:1], v[4:5], -v[0:1]
 ; GFX10-DENORM-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-CONTRACT-LABEL: test_double_sub_mul_rhs:
+; GFX11-CONTRACT:       ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[0:1], -v[0:1], v[2:3], v[4:5]
+; GFX11-CONTRACT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-DENORM-LABEL: test_double_sub_mul_rhs:
+; GFX11-DENORM:       ; %bb.0: ; %.entry
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-DENORM-NEXT:    v_mul_f64 v[0:1], v[0:1], v[2:3]
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-DENORM-NEXT:    v_add_f64 v[0:1], v[4:5], -v[0:1]
+; GFX11-DENORM-NEXT:    s_setpc_b64 s[30:31]
 .entry:
   %a = fmul double %x, %y
   %b = fsub double %z, %a
@@ -362,6 +460,27 @@ define <4 x float> @test_v4f32_sub_mul(<4 x float> %x, <4 x float> %y, <4 x floa
 ; GFX10-DENORM-NEXT:    v_mad_f32 v2, v2, v6, -v10
 ; GFX10-DENORM-NEXT:    v_mad_f32 v3, v3, v7, -v11
 ; GFX10-DENORM-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-CONTRACT-LABEL: test_v4f32_sub_mul:
+; GFX11-CONTRACT:       ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-CONTRACT-NEXT:    v_fma_f32 v0, v0, v4, -v8
+; GFX11-CONTRACT-NEXT:    v_fma_f32 v1, v1, v5, -v9
+; GFX11-CONTRACT-NEXT:    v_fma_f32 v2, v2, v6, -v10
+; GFX11-CONTRACT-NEXT:    v_fma_f32 v3, v3, v7, -v11
+; GFX11-CONTRACT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-DENORM-LABEL: test_v4f32_sub_mul:
+; GFX11-DENORM:       ; %bb.0: ; %.entry
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-DENORM-NEXT:    v_dual_mul_f32 v0, v0, v4 :: v_dual_mul_f32 v1, v1, v5
+; GFX11-DENORM-NEXT:    v_dual_mul_f32 v2, v2, v6 :: v_dual_mul_f32 v3, v3, v7
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-DENORM-NEXT:    v_dual_sub_f32 v0, v0, v8 :: v_dual_sub_f32 v1, v1, v9
+; GFX11-DENORM-NEXT:    v_dual_sub_f32 v2, v2, v10 :: v_dual_sub_f32 v3, v3, v11
+; GFX11-DENORM-NEXT:    s_setpc_b64 s[30:31]
 .entry:
   %a = fmul <4 x float> %x, %y
   %b = fsub <4 x float> %a, %z
@@ -433,6 +552,27 @@ define <4 x float> @test_v4f32_sub_mul_rhs(<4 x float> %x, <4 x float> %y, <4 x
 ; GFX10-DENORM-NEXT:    v_mad_f32 v2, -v2, v6, v10
 ; GFX10-DENORM-NEXT:    v_mad_f32 v3, -v3, v7, v11
 ; GFX10-DENORM-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-CONTRACT-LABEL: test_v4f32_sub_mul_rhs:
+; GFX11-CONTRACT:       ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-CONTRACT-NEXT:    v_fma_f32 v0, -v0, v4, v8
+; GFX11-CONTRACT-NEXT:    v_fma_f32 v1, -v1, v5, v9
+; GFX11-CONTRACT-NEXT:    v_fma_f32 v2, -v2, v6, v10
+; GFX11-CONTRACT-NEXT:    v_fma_f32 v3, -v3, v7, v11
+; GFX11-CONTRACT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-DENORM-LABEL: test_v4f32_sub_mul_rhs:
+; GFX11-DENORM:       ; %bb.0: ; %.entry
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-DENORM-NEXT:    v_dual_mul_f32 v0, v0, v4 :: v_dual_mul_f32 v1, v1, v5
+; GFX11-DENORM-NEXT:    v_dual_mul_f32 v2, v2, v6 :: v_dual_mul_f32 v3, v3, v7
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-DENORM-NEXT:    v_dual_sub_f32 v0, v8, v0 :: v_dual_sub_f32 v1, v9, v1
+; GFX11-DENORM-NEXT:    v_dual_sub_f32 v2, v10, v2 :: v_dual_sub_f32 v3, v11, v3
+; GFX11-DENORM-NEXT:    s_setpc_b64 s[30:31]
 .entry:
   %a = fmul <4 x float> %x, %y
   %b = fsub <4 x float> %z, %a
@@ -508,6 +648,35 @@ define <4 x half> @test_v4f16_sub_mul(<4 x half> %x, <4 x half> %y, <4 x half> %
 ; GFX10-DENORM-NEXT:    v_pack_b32_f16 v0, v2, v0
 ; GFX10-DENORM-NEXT:    v_pack_b32_f16 v1, v3, v1
 ; GFX10-DENORM-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-CONTRACT-LABEL: test_v4f16_sub_mul:
+; GFX11-CONTRACT:       ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-CONTRACT-NEXT:    v_pk_fma_f16 v0, v0, v2, v4 neg_lo:[0,0,1] neg_hi:[0,0,1]
+; GFX11-CONTRACT-NEXT:    v_pk_fma_f16 v1, v1, v3, v5 neg_lo:[0,0,1] neg_hi:[0,0,1]
+; GFX11-CONTRACT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-DENORM-LABEL: test_v4f16_sub_mul:
+; GFX11-DENORM:       ; %bb.0: ; %.entry
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-DENORM-NEXT:    v_pk_mul_f16 v0, v0, v2
+; GFX11-DENORM-NEXT:    v_pk_mul_f16 v1, v1, v3
+; GFX11-DENORM-NEXT:    v_lshrrev_b32_e32 v2, 16, v4
+; GFX11-DENORM-NEXT:    v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-DENORM-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-DENORM-NEXT:    v_lshrrev_b32_e32 v7, 16, v1
+; GFX11-DENORM-NEXT:    v_sub_f16_e32 v0, v0, v4
+; GFX11-DENORM-NEXT:    v_sub_f16_e32 v1, v1, v5
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-DENORM-NEXT:    v_sub_f16_e32 v2, v6, v2
+; GFX11-DENORM-NEXT:    v_sub_f16_e32 v3, v7, v3
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-DENORM-NEXT:    v_pack_b32_f16 v0, v0, v2
+; GFX11-DENORM-NEXT:    v_pack_b32_f16 v1, v1, v3
+; GFX11-DENORM-NEXT:    s_setpc_b64 s[30:31]
 .entry:
   %a = fmul <4 x half> %x, %y
   %b = fsub <4 x half> %a, %z
@@ -583,6 +752,35 @@ define <4 x half> @test_v4f16_sub_mul_rhs(<4 x half> %x, <4 x half> %y, <4 x hal
 ; GFX10-DENORM-NEXT:    v_pack_b32_f16 v0, v2, v0
 ; GFX10-DENORM-NEXT:    v_pack_b32_f16 v1, v3, v1
 ; GFX10-DENORM-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-CONTRACT-LABEL: test_v4f16_sub_mul_rhs:
+; GFX11-CONTRACT:       ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-CONTRACT-NEXT:    v_pk_fma_f16 v0, v0, v2, v4 neg_lo:[1,0,0] neg_hi:[1,0,0]
+; GFX11-CONTRACT-NEXT:    v_pk_fma_f16 v1, v1, v3, v5 neg_lo:[1,0,0] neg_hi:[1,0,0]
+; GFX11-CONTRACT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-DENORM-LABEL: test_v4f16_sub_mul_rhs:
+; GFX11-DENORM:       ; %bb.0: ; %.entry
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-DENORM-NEXT:    v_pk_mul_f16 v0, v0, v2
+; GFX11-DENORM-NEXT:    v_pk_mul_f16 v1, v1, v3
+; GFX11-DENORM-NEXT:    v_lshrrev_b32_e32 v2, 16, v4
+; GFX11-DENORM-NEXT:    v_lshrrev_b32_e32 v3, 16, v5
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-DENORM-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
+; GFX11-DENORM-NEXT:    v_lshrrev_b32_e32 v7, 16, v1
+; GFX11-DENORM-NEXT:    v_sub_f16_e32 v0, v4, v0
+; GFX11-DENORM-NEXT:    v_sub_f16_e32 v1, v5, v1
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-DENORM-NEXT:    v_sub_f16_e32 v2, v2, v6
+; GFX11-DENORM-NEXT:    v_sub_f16_e32 v3, v3, v7
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-DENORM-NEXT:    v_pack_b32_f16 v0, v0, v2
+; GFX11-DENORM-NEXT:    v_pack_b32_f16 v1, v1, v3
+; GFX11-DENORM-NEXT:    s_setpc_b64 s[30:31]
 .entry:
   %a = fmul <4 x half> %x, %y
   %b = fsub <4 x half> %z, %a
@@ -662,6 +860,32 @@ define <4 x double> @test_v4f64_sub_mul(<4 x double> %x, <4 x double> %y, <4 x d
 ; GFX10-DENORM-NEXT:    v_add_f64 v[4:5], v[4:5], -v[20:21]
 ; GFX10-DENORM-NEXT:    v_add_f64 v[6:7], v[6:7], -v[22:23]
 ; GFX10-DENORM-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-CONTRACT-LABEL: test_v4f64_sub_mul:
+; GFX11-CONTRACT:       ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[0:1], v[0:1], v[8:9], -v[16:17]
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[2:3], v[2:3], v[10:11], -v[18:19]
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[4:5], v[4:5], v[12:13], -v[20:21]
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[6:7], v[6:7], v[14:15], -v[22:23]
+; GFX11-CONTRACT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-DENORM-LABEL: test_v4f64_sub_mul:
+; GFX11-DENORM:       ; %bb.0: ; %.entry
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-DENORM-NEXT:    v_mul_f64 v[0:1], v[0:1], v[8:9]
+; GFX11-DENORM-NEXT:    v_mul_f64 v[2:3], v[2:3], v[10:11]
+; GFX11-DENORM-NEXT:    v_mul_f64 v[4:5], v[4:5], v[12:13]
+; GFX11-DENORM-NEXT:    v_mul_f64 v[6:7], v[6:7], v[14:15]
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-DENORM-NEXT:    v_add_f64 v[0:1], v[0:1], -v[16:17]
+; GFX11-DENORM-NEXT:    v_add_f64 v[2:3], v[2:3], -v[18:19]
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-DENORM-NEXT:    v_add_f64 v[4:5], v[4:5], -v[20:21]
+; GFX11-DENORM-NEXT:    v_add_f64 v[6:7], v[6:7], -v[22:23]
+; GFX11-DENORM-NEXT:    s_setpc_b64 s[30:31]
 .entry:
   %a = fmul <4 x double> %x, %y
   %b = fsub <4 x double> %a, %z
@@ -741,6 +965,32 @@ define <4 x double> @test_v4f64_sub_mul_rhs(<4 x double> %x, <4 x double> %y, <4
 ; GFX10-DENORM-NEXT:    v_add_f64 v[4:5], v[20:21], -v[4:5]
 ; GFX10-DENORM-NEXT:    v_add_f64 v[6:7], v[22:23], -v[6:7]
 ; GFX10-DENORM-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-CONTRACT-LABEL: test_v4f64_sub_mul_rhs:
+; GFX11-CONTRACT:       ; %bb.0: ; %.entry
+; GFX11-CONTRACT-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-CONTRACT-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[0:1], -v[0:1], v[8:9], v[16:17]
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[2:3], -v[2:3], v[10:11], v[18:19]
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[4:5], -v[4:5], v[12:13], v[20:21]
+; GFX11-CONTRACT-NEXT:    v_fma_f64 v[6:7], -v[6:7], v[14:15], v[22:23]
+; GFX11-CONTRACT-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-DENORM-LABEL: test_v4f64_sub_mul_rhs:
+; GFX11-DENORM:       ; %bb.0: ; %.entry
+; GFX11-DENORM-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-DENORM-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-DENORM-NEXT:    v_mul_f64 v[0:1], v[0:1], v[8:9]
+; GFX11-DENORM-NEXT:    v_mul_f64 v[2:3], v[2:3], v[10:11]
+; GFX11-DENORM-NEXT:    v_mul_f64 v[4:5], v[4:5], v[12:13]
+; GFX11-DENORM-NEXT:    v_mul_f64 v[6:7], v[6:7], v[14:15]
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-DENORM-NEXT:    v_add_f64 v[0:1], v[16:17], -v[0:1]
+; GFX11-DENORM-NEXT:    v_add_f64 v[2:3], v[18:19], -v[2:3]
+; GFX11-DENORM-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-DENORM-NEXT:    v_add_f64 v[4:5], v[20:21], -v[4:5]
+; GFX11-DENORM-NEXT:    v_add_f64 v[6:7], v[22:23], -v[6:7]
+; GFX11-DENORM-NEXT:    s_setpc_b64 s[30:31]
 .entry:
   %a = fmul <4 x double> %x, %y
   %b = fsub <4 x double> %z, %a
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll
index eda1355a35e36..3cf0a82e9adfb 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll
@@ -1,71 +1,140 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX10 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX11 %s
 
 define amdgpu_ps float @_amdgpu_ps_main() #0 {
-; GCN-LABEL: _amdgpu_ps_main:
-; GCN:       ; %bb.0: ; %.entry
-; GCN-NEXT:    image_sample v[0:1], v[0:1], s[0:7], s[0:3] dmask:0x3 dim:SQ_RSRC_IMG_2D
-; GCN-NEXT:    v_mov_b32_e32 v4, 0
-; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    s_clause 0x1
-; GCN-NEXT:    image_sample v2, v[0:1], s[0:7], s[0:3] dmask:0x4 dim:SQ_RSRC_IMG_2D
-; GCN-NEXT:    image_sample v3, v[0:1], s[0:7], s[0:3] dmask:0x1 dim:SQ_RSRC_IMG_2D
-; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    image_load_mip v4, v[2:4], s[0:7] dmask:0x4 dim:SQ_RSRC_IMG_2D unorm
-; GCN-NEXT:    s_clause 0x3
-; GCN-NEXT:    s_buffer_load_dword s24, s[0:3], 0x5c
-; GCN-NEXT:    s_buffer_load_dword s28, s[0:3], 0x7c
-; GCN-NEXT:    s_buffer_load_dword s29, s[0:3], 0xc0
-; GCN-NEXT:    s_waitcnt_depctr 0xffe3
-; GCN-NEXT:    s_nop 0
-; GCN-NEXT:    s_buffer_load_dwordx4 s[0:3], s[0:3], 0x40
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_clause 0x1
-; GCN-NEXT:    s_buffer_load_dwordx4 s[4:7], s[0:3], 0x50
-; GCN-NEXT:    s_nop 0
-; GCN-NEXT:    s_buffer_load_dword s0, s[0:3], 0x2c
-; GCN-NEXT:    v_sub_f32_e64 v5, s24, s28
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    s_clause 0x4
-; GCN-NEXT:    s_buffer_load_dwordx4 s[8:11], s[0:3], 0x60
-; GCN-NEXT:    s_buffer_load_dwordx4 s[12:15], s[0:3], 0x20
-; GCN-NEXT:    s_buffer_load_dwordx4 s[16:19], s[0:3], 0x0
-; GCN-NEXT:    s_buffer_load_dwordx4 s[20:23], s[0:3], 0x70
-; GCN-NEXT:    s_buffer_load_dwordx4 s[24:27], s[0:3], 0x10
-; GCN-NEXT:    v_fma_f32 v1, v1, v5, s28
-; GCN-NEXT:    v_max_f32_e64 v6, s0, s0 clamp
-; GCN-NEXT:    v_add_f32_e64 v5, s29, -1.0
-; GCN-NEXT:    v_sub_f32_e32 v8, s0, v1
-; GCN-NEXT:    v_fma_f32 v7, -s2, v6, s6
-; GCN-NEXT:    v_fma_f32 v5, v6, v5, 1.0
-; GCN-NEXT:    v_mad_f32 v10, s2, v6, v2
-; GCN-NEXT:    s_mov_b32 s0, 0x3c23d70a
-; GCN-NEXT:    v_fmac_f32_e32 v1, v6, v8
-; GCN-NEXT:    v_fmac_f32_e32 v10, v7, v6
-; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    v_mul_f32_e32 v9, s10, v0
-; GCN-NEXT:    v_fma_f32 v0, -v0, s10, s14
-; GCN-NEXT:    v_mul_f32_e32 v8, s18, v2
-; GCN-NEXT:    v_mul_f32_e32 v3, s22, v3
-; GCN-NEXT:    v_fmac_f32_e32 v9, v0, v6
-; GCN-NEXT:    v_sub_f32_e32 v0, v1, v5
-; GCN-NEXT:    v_mul_f32_e32 v1, v8, v6
-; GCN-NEXT:    v_mul_f32_e32 v7, v6, v3
-; GCN-NEXT:    v_fma_f32 v3, -v6, v3, v9
-; GCN-NEXT:    v_fmac_f32_e32 v5, v0, v6
-; GCN-NEXT:    v_fma_f32 v0, v2, s26, -v1
-; GCN-NEXT:    v_fmac_f32_e32 v7, v3, v6
-; GCN-NEXT:    v_fmac_f32_e32 v1, v0, v6
-; GCN-NEXT:    v_mul_f32_e32 v0, v2, v6
-; GCN-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NEXT:    v_add_f32_e32 v4, v4, v10
-; GCN-NEXT:    v_mul_f32_e32 v3, v4, v6
-; GCN-NEXT:    v_fmaak_f32 v4, s0, v5, 0x3ca3d70a
-; GCN-NEXT:    v_mul_f32_e32 v1, v3, v1
-; GCN-NEXT:    v_mul_f32_e32 v2, v7, v4
-; GCN-NEXT:    v_fmac_f32_e32 v1, v2, v0
-; GCN-NEXT:    v_max_f32_e32 v0, 0, v1
-; GCN-NEXT:    ; return to shader part epilog
+; GFX10-LABEL: _amdgpu_ps_main:
+; GFX10:       ; %bb.0: ; %.entry
+; GFX10-NEXT:    image_sample v[0:1], v[0:1], s[0:7], s[0:3] dmask:0x3 dim:SQ_RSRC_IMG_2D
+; GFX10-NEXT:    v_mov_b32_e32 v4, 0
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    s_clause 0x1
+; GFX10-NEXT:    image_sample v2, v[0:1], s[0:7], s[0:3] dmask:0x4 dim:SQ_RSRC_IMG_2D
+; GFX10-NEXT:    image_sample v3, v[0:1], s[0:7], s[0:3] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    image_load_mip v4, v[2:4], s[0:7] dmask:0x4 dim:SQ_RSRC_IMG_2D unorm
+; GFX10-NEXT:    s_clause 0x3
+; GFX10-NEXT:    s_buffer_load_dword s24, s[0:3], 0x5c
+; GFX10-NEXT:    s_buffer_load_dword s28, s[0:3], 0x7c
+; GFX10-NEXT:    s_buffer_load_dword s29, s[0:3], 0xc0
+; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX10-NEXT:    s_nop 0
+; GFX10-NEXT:    s_buffer_load_dwordx4 s[0:3], s[0:3], 0x40
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_clause 0x1
+; GFX10-NEXT:    s_buffer_load_dwordx4 s[4:7], s[0:3], 0x50
+; GFX10-NEXT:    s_nop 0
+; GFX10-NEXT:    s_buffer_load_dword s0, s[0:3], 0x2c
+; GFX10-NEXT:    v_sub_f32_e64 v5, s24, s28
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_clause 0x4
+; GFX10-NEXT:    s_buffer_load_dwordx4 s[8:11], s[0:3], 0x60
+; GFX10-NEXT:    s_buffer_load_dwordx4 s[12:15], s[0:3], 0x20
+; GFX10-NEXT:    s_buffer_load_dwordx4 s[16:19], s[0:3], 0x0
+; GFX10-NEXT:    s_buffer_load_dwordx4 s[20:23], s[0:3], 0x70
+; GFX10-NEXT:    s_buffer_load_dwordx4 s[24:27], s[0:3], 0x10
+; GFX10-NEXT:    v_fma_f32 v1, v1, v5, s28
+; GFX10-NEXT:    v_max_f32_e64 v6, s0, s0 clamp
+; GFX10-NEXT:    v_add_f32_e64 v5, s29, -1.0
+; GFX10-NEXT:    v_sub_f32_e32 v8, s0, v1
+; GFX10-NEXT:    v_fma_f32 v7, -s2, v6, s6
+; GFX10-NEXT:    v_fma_f32 v5, v6, v5, 1.0
+; GFX10-NEXT:    v_mad_f32 v10, s2, v6, v2
+; GFX10-NEXT:    s_mov_b32 s0, 0x3c23d70a
+; GFX10-NEXT:    v_fmac_f32_e32 v1, v6, v8
+; GFX10-NEXT:    v_fmac_f32_e32 v10, v7, v6
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    v_mul_f32_e32 v9, s10, v0
+; GFX10-NEXT:    v_fma_f32 v0, -v0, s10, s14
+; GFX10-NEXT:    v_mul_f32_e32 v8, s18, v2
+; GFX10-NEXT:    v_mul_f32_e32 v3, s22, v3
+; GFX10-NEXT:    v_fmac_f32_e32 v9, v0, v6
+; GFX10-NEXT:    v_sub_f32_e32 v0, v1, v5
+; GFX10-NEXT:    v_mul_f32_e32 v1, v8, v6
+; GFX10-NEXT:    v_mul_f32_e32 v7, v6, v3
+; GFX10-NEXT:    v_fma_f32 v3, -v6, v3, v9
+; GFX10-NEXT:    v_fmac_f32_e32 v5, v0, v6
+; GFX10-NEXT:    v_fma_f32 v0, v2, s26, -v1
+; GFX10-NEXT:    v_fmac_f32_e32 v7, v3, v6
+; GFX10-NEXT:    v_fmac_f32_e32 v1, v0, v6
+; GFX10-NEXT:    v_mul_f32_e32 v0, v2, v6
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_add_f32_e32 v4, v4, v10
+; GFX10-NEXT:    v_mul_f32_e32 v3, v4, v6
+; GFX10-NEXT:    v_fmaak_f32 v4, s0, v5, 0x3ca3d70a
+; GFX10-NEXT:    v_mul_f32_e32 v1, v3, v1
+; GFX10-NEXT:    v_mul_f32_e32 v2, v7, v4
+; GFX10-NEXT:    v_fmac_f32_e32 v1, v2, v0
+; GFX10-NEXT:    v_max_f32_e32 v0, 0, v1
+; GFX10-NEXT:    ; return to shader part epilog
+;
+; GFX11-LABEL: _amdgpu_ps_main:
+; GFX11:       ; %bb.0: ; %.entry
+; GFX11-NEXT:    image_sample v[0:1], v[0:1], s[0:7], s[0:3] dmask:0x3 dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT:    v_mov_b32_e32 v4, 0
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    image_sample v2, v[0:1], s[0:7], s[0:3] dmask:0x4 dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT:    image_sample v3, v[0:1], s[0:7], s[0:3] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    image_load_mip v4, v[2:4], s[0:7] dmask:0x4 dim:SQ_RSRC_IMG_2D unorm
+; GFX11-NEXT:    s_clause 0x3
+; GFX11-NEXT:    s_buffer_load_b32 s24, s[0:3], 0x5c
+; GFX11-NEXT:    s_buffer_load_b32 s28, s[0:3], 0x7c
+; GFX11-NEXT:    s_buffer_load_b32 s29, s[0:3], 0xc0
+; GFX11-NEXT:    s_buffer_load_b128 s[0:3], s[0:3], 0x40
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_clause 0x1
+; GFX11-NEXT:    s_buffer_load_b128 s[4:7], s[0:3], 0x50
+; GFX11-NEXT:    s_buffer_load_b32 s0, s[0:3], 0x2c
+; GFX11-NEXT:    v_sub_f32_e64 v5, s24, s28
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_clause 0x3
+; GFX11-NEXT:    s_buffer_load_b128 s[8:11], s[0:3], 0x60
+; GFX11-NEXT:    s_buffer_load_b128 s[12:15], s[0:3], 0x20
+; GFX11-NEXT:    s_buffer_load_b128 s[16:19], s[0:3], 0x0
+; GFX11-NEXT:    s_buffer_load_b128 s[20:23], s[0:3], 0x70
+; GFX11-NEXT:    v_fma_f32 v1, v1, v5, s28
+; GFX11-NEXT:    v_max_f32_e64 v6, s0, s0 clamp
+; GFX11-NEXT:    s_buffer_load_b128 s[24:27], s[0:3], 0x10
+; GFX11-NEXT:    v_add_f32_e64 v5, s29, -1.0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_sub_f32_e32 v8, s0, v1
+; GFX11-NEXT:    v_fma_f32 v7, -s2, v6, s6
+; GFX11-NEXT:    v_fma_f32 v10, s2, v6, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT:    v_fma_f32 v5, v6, v5, 1.0
+; GFX11-NEXT:    s_mov_b32 s0, 0x3c23d70a
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    v_mul_f32_e32 v9, s10, v0
+; GFX11-NEXT:    v_fma_f32 v0, -v0, s10, s14
+; GFX11-NEXT:    v_mul_f32_e32 v3, s22, v3
+; GFX11-NEXT:    v_dual_fmac_f32 v1, v6, v8 :: v_dual_mul_f32 v8, s18, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_fmac_f32_e32 v9, v0, v6
+; GFX11-NEXT:    v_dual_fmac_f32 v10, v7, v6 :: v_dual_mul_f32 v7, v6, v3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_sub_f32_e32 v0, v1, v5
+; GFX11-NEXT:    v_fma_f32 v3, -v6, v3, v9
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_fmac_f32_e32 v7, v3, v6
+; GFX11-NEXT:    v_fmac_f32_e32 v5, v0, v6
+; GFX11-NEXT:    v_mul_f32_e32 v1, v8, v6
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_add_f32_e32 v4, v4, v10
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_dual_mul_f32 v3, v4, v6 :: v_dual_fmaak_f32 v4, s0, v5, 0x3ca3d70a
+; GFX11-NEXT:    v_fma_f32 v0, v2, s26, -v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-NEXT:    v_fmac_f32_e32 v1, v0, v6
+; GFX11-NEXT:    v_mul_f32_e32 v0, v2, v6
+; GFX11-NEXT:    v_mul_f32_e32 v2, v7, v4
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_mul_f32_e32 v1, v3, v1
+; GFX11-NEXT:    v_fmac_f32_e32 v1, v2, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_max_f32_e32 v0, 0, v1
+; GFX11-NEXT:    ; return to shader part epilog
 .entry:
   %0 = call <3 x float> @llvm.amdgcn.image.sample.2d.v3f32.f32(i32 7, float undef, float undef, <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0)
   %.i2243 = extractelement <3 x float> %0, i32 2
@@ -168,14 +237,24 @@ define amdgpu_ps float @_amdgpu_ps_main() #0 {
 }
 
 define float @fmac_sequence_simple(float %a, float %b, float %c, float %d, float %e) #0 {
-; GCN-LABEL: fmac_sequence_simple:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    s_waitcnt_vscnt null, 0x0
-; GCN-NEXT:    v_fma_f32 v2, v2, v3, v4
-; GCN-NEXT:    v_fmac_f32_e32 v2, v0, v1
-; GCN-NEXT:    v_mov_b32_e32 v0, v2
-; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: fmac_sequence_simple:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_fma_f32 v2, v2, v3, v4
+; GFX10-NEXT:    v_fmac_f32_e32 v2, v0, v1
+; GFX10-NEXT:    v_mov_b32_e32 v0, v2
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fmac_sequence_simple:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    v_fma_f32 v2, v2, v3, v4
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_fmac_f32_e32 v2, v0, v1
+; GFX11-NEXT:    v_mov_b32_e32 v0, v2
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %t0 = fmul fast float %a, %b
   %t1 = fmul fast float %c, %d
   %t2 = fadd fast float %t0, %t1
@@ -184,15 +263,27 @@ define float @fmac_sequence_simple(float %a, float %b, float %c, float %d, float
 }
 
 define float @fmac_sequence_innermost_fmul(float %a, float %b, float %c, float %d, float %e, float %f, float %g) #0 {
-; GCN-LABEL: fmac_sequence_innermost_fmul:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    s_waitcnt_vscnt null, 0x0
-; GCN-NEXT:    v_mad_f32 v2, v2, v3, v6
-; GCN-NEXT:    v_fmac_f32_e32 v2, v0, v1
-; GCN-NEXT:    v_fmac_f32_e32 v2, v4, v5
-; GCN-NEXT:    v_mov_b32_e32 v0, v2
-; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: fmac_sequence_innermost_fmul:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_mad_f32 v2, v2, v3, v6
+; GFX10-NEXT:    v_fmac_f32_e32 v2, v0, v1
+; GFX10-NEXT:    v_fmac_f32_e32 v2, v4, v5
+; GFX10-NEXT:    v_mov_b32_e32 v0, v2
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fmac_sequence_innermost_fmul:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    v_fma_f32 v2, v2, v3, v6
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_fmac_f32_e32 v2, v0, v1
+; GFX11-NEXT:    v_fmac_f32_e32 v2, v4, v5
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_mov_b32_e32 v0, v2
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %t0 = fmul fast float %a, %b
   %t1 = fmul fast float %c, %d
   %t2 = fadd fast float %t0, %t1
@@ -203,15 +294,27 @@ define float @fmac_sequence_innermost_fmul(float %a, float %b, float %c, float %
 }
 
 define float @fmac_sequence_innermost_fmul_swapped_operands(float %a, float %b, float %c, float %d, float %e, float %f, float %g) #0 {
-; GCN-LABEL: fmac_sequence_innermost_fmul_swapped_operands:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    s_waitcnt_vscnt null, 0x0
-; GCN-NEXT:    v_mad_f32 v2, v2, v3, v6
-; GCN-NEXT:    v_fmac_f32_e32 v2, v0, v1
-; GCN-NEXT:    v_fmac_f32_e32 v2, v4, v5
-; GCN-NEXT:    v_mov_b32_e32 v0, v2
-; GCN-NEXT:    s_setpc_b64 s[30:31]
+; GFX10-LABEL: fmac_sequence_innermost_fmul_swapped_operands:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX10-NEXT:    v_mad_f32 v2, v2, v3, v6
+; GFX10-NEXT:    v_fmac_f32_e32 v2, v0, v1
+; GFX10-NEXT:    v_fmac_f32_e32 v2, v4, v5
+; GFX10-NEXT:    v_mov_b32_e32 v0, v2
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: fmac_sequence_innermost_fmul_swapped_operands:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    v_fma_f32 v2, v2, v3, v6
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_fmac_f32_e32 v2, v0, v1
+; GFX11-NEXT:    v_fmac_f32_e32 v2, v4, v5
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_mov_b32_e32 v0, v2
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %t0 = fmul fast float %a, %b
   %t1 = fmul fast float %c, %d
   %t2 = fadd fast float %t0, %t1
@@ -222,12 +325,20 @@ define float @fmac_sequence_innermost_fmul_swapped_operands(float %a, float %b,
 }
 
 define amdgpu_ps float @fmac_sequence_innermost_fmul_sgpr(float inreg %a, float inreg %b, float inreg %c, float inreg %d, float inreg %e, float inreg %f, float %g) #0 {
-; GCN-LABEL: fmac_sequence_innermost_fmul_sgpr:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    v_mac_f32_e64 v0, s2, s3
-; GCN-NEXT:    v_fmac_f32_e64 v0, s0, s1
-; GCN-NEXT:    v_fmac_f32_e64 v0, s4, s5
-; GCN-NEXT:    ; return to shader part epilog
+; GFX10-LABEL: fmac_sequence_innermost_fmul_sgpr:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_mac_f32_e64 v0, s2, s3
+; GFX10-NEXT:    v_fmac_f32_e64 v0, s0, s1
+; GFX10-NEXT:    v_fmac_f32_e64 v0, s4, s5
+; GFX10-NEXT:    ; return to shader part epilog
+;
+; GFX11-LABEL: fmac_sequence_innermost_fmul_sgpr:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_fmac_f32_e64 v0, s2, s3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_fmac_f32_e64 v0, s0, s1
+; GFX11-NEXT:    v_fmac_f32_e64 v0, s4, s5
+; GFX11-NEXT:    ; return to shader part epilog
   %t0 = fmul fast float %a, %b
   %t1 = fmul fast float %c, %d
   %t2 = fadd fast float %t0, %t1
@@ -238,14 +349,25 @@ define amdgpu_ps float @fmac_sequence_innermost_fmul_sgpr(float inreg %a, float
 }
 
 define amdgpu_ps float @fmac_sequence_innermost_fmul_multiple_use(float inreg %a, float inreg %b, float inreg %c, float inreg %d, float inreg %e, float inreg %f, float %g) #0 {
-; GCN-LABEL: fmac_sequence_innermost_fmul_multiple_use:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    v_mul_f32_e64 v1, s2, s3
-; GCN-NEXT:    v_fmac_f32_e64 v1, s0, s1
-; GCN-NEXT:    v_fma_f32 v2, s5, s4, v1
-; GCN-NEXT:    v_fmac_f32_e32 v1, s5, v2
-; GCN-NEXT:    v_add_f32_e32 v0, v1, v0
-; GCN-NEXT:    ; return to shader part epilog
+; GFX10-LABEL: fmac_sequence_innermost_fmul_multiple_use:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_mul_f32_e64 v1, s2, s3
+; GFX10-NEXT:    v_fmac_f32_e64 v1, s0, s1
+; GFX10-NEXT:    v_fma_f32 v2, s5, s4, v1
+; GFX10-NEXT:    v_fmac_f32_e32 v1, s5, v2
+; GFX10-NEXT:    v_add_f32_e32 v0, v1, v0
+; GFX10-NEXT:    ; return to shader part epilog
+;
+; GFX11-LABEL: fmac_sequence_innermost_fmul_multiple_use:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_mul_f32_e64 v1, s2, s3
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_fmac_f32_e64 v1, s0, s1
+; GFX11-NEXT:    v_fma_f32 v2, s5, s4, v1
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_fmac_f32_e32 v1, s5, v2
+; GFX11-NEXT:    v_add_f32_e32 v0, v1, v0
+; GFX11-NEXT:    ; return to shader part epilog
   %t0 = fmul fast float %a, %b
   %t1 = fmul fast float %c, %d
   %t2 = fadd fast float %t0, %t1
@@ -261,12 +383,20 @@ define amdgpu_ps float @fmac_sequence_innermost_fmul_multiple_use(float inreg %a
 ; selecting it as a multiply. In some cases the multiply is better because
 ; SIFoldOperands can fold it into a previous instruction as an output modifier.
 define amdgpu_ps float @fma_vs_output_modifier(float %x, i32 %n) #0 {
-; GCN-LABEL: fma_vs_output_modifier:
-; GCN:       ; %bb.0:
-; GCN-NEXT:    v_cvt_f32_i32_e64 v1, v1 mul:2
-; GCN-NEXT:    v_mul_f32_e32 v0, v0, v0
-; GCN-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GCN-NEXT:    ; return to shader part epilog
+; GFX10-LABEL: fma_vs_output_modifier:
+; GFX10:       ; %bb.0:
+; GFX10-NEXT:    v_cvt_f32_i32_e64 v1, v1 mul:2
+; GFX10-NEXT:    v_mul_f32_e32 v0, v0, v0
+; GFX10-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX10-NEXT:    ; return to shader part epilog
+;
+; GFX11-LABEL: fma_vs_output_modifier:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    v_cvt_f32_i32_e64 v1, v1 mul:2
+; GFX11-NEXT:    v_mul_f32_e32 v0, v0, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GFX11-NEXT:    ; return to shader part epilog
   %s = sitofp i32 %n to float
   %m = fmul contract float %x, %x
   %a = fmul contract float %m, 2.0
diff --git a/llvm/test/CodeGen/AMDGPU/fma-combine.ll b/llvm/test/CodeGen/AMDGPU/fma-combine.ll
index 058b9784c9001..afffa5d6306c2 100644
--- a/llvm/test/CodeGen/AMDGPU/fma-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/fma-combine.ll
@@ -1,6 +1,9 @@
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -verify-machineinstrs -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefix=SI-NOFMA -check-prefix=SI-SAFE -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=verde -denormal-fp-math-f32=preserve-sign -verify-machineinstrs -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefix=SI-NOFMA -check-prefix=SI-SAFE -check-prefix=SI -check-prefix=FUNC %s
-; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -denormal-fp-math-f32=ieee -verify-machineinstrs -fp-contract=fast -enable-no-infs-fp-math -enable-unsafe-fp-math < %s | FileCheck -enable-var-scope -check-prefix=SI-FMA -check-prefix=SI-UNSAFE -check-prefix=SI -check-prefix=FUNC %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -denormal-fp-math-f32=preserve-sign -verify-machineinstrs -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefixes=SI,SI-NOFMA %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=verde -denormal-fp-math-f32=preserve-sign -verify-machineinstrs -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefixes=SI,SI-NOFMA %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -denormal-fp-math-f32=ieee -verify-machineinstrs -fp-contract=fast -enable-no-infs-fp-math -enable-unsafe-fp-math < %s | FileCheck -enable-var-scope -check-prefixes=SI,SI-FMA %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx1100 -denormal-fp-math-f32=preserve-sign -verify-machineinstrs -fp-contract=fast < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-NOFMA %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx1100 -denormal-fp-math-f32=ieee -verify-machineinstrs -fp-contract=fast -enable-no-infs-fp-math -enable-unsafe-fp-math < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-FMA %s
 
 ; FIXME: Remove enable-unsafe-fp-math in RUN line and add flags to IR instrs
 
@@ -15,13 +18,42 @@ declare float @llvm.fma.f32(float, float, float) #0
 declare <4 x float> @llvm.fma.v4f32(<4 x float>, <4 x float>, <4 x float>) #0
 
 ; (fadd (fmul x, y), z) -> (fma x, y, z)
-; FUNC-LABEL: {{^}}combine_to_fma_f64_0:
-; SI-DAG: buffer_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[B:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[C:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16 glc{{$}}
-; SI: v_fma_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[A]], [[B]], [[C]]
-; SI: buffer_store_dwordx2 [[RESULT]]
 define amdgpu_kernel void @combine_to_fma_f64_0(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-LABEL: combine_to_fma_f64_0:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    v_fma_f64 v[2:3], v[2:3], v[4:5], v[6:7]
+; SI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: combine_to_fma_f64_0:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-NEXT:    v_lshlrev_b32_e32 v6, 3, v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[0:1], v6, s[2:3] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[2:3], v6, s[2:3] offset:8 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[4:5], v6, s[2:3] offset:16 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_fma_f64 v[0:1], v[0:1], v[2:3], v[4:5]
+; GFX11-NEXT:    global_store_b64 v6, v[0:1], s[0:1]
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr double, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr double, ptr addrspace(1) %gep.0, i32 1
@@ -39,17 +71,54 @@ define amdgpu_kernel void @combine_to_fma_f64_0(ptr addrspace(1) noalias %out, p
 }
 
 ; (fadd (fmul x, y), z) -> (fma x, y, z)
-; FUNC-LABEL: {{^}}combine_to_fma_f64_0_2use:
-; SI-DAG: buffer_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[B:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[C:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[D:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:24 glc{{$}}
-; SI-DAG: v_fma_f64 [[RESULT0:v\[[0-9]+:[0-9]+\]]], [[A]], [[B]], [[C]]
-; SI-DAG: v_fma_f64 [[RESULT1:v\[[0-9]+:[0-9]+\]]], [[A]], [[B]], [[D]]
-; SI-DAG: buffer_store_dwordx2 [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-; SI-DAG: buffer_store_dwordx2 [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
-; SI: s_endpgm
 define amdgpu_kernel void @combine_to_fma_f64_0_2use(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-LABEL: combine_to_fma_f64_0_2use:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dwordx2 v[8:9], v[0:1], s[4:7], 0 addr64 offset:24 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    v_fma_f64 v[6:7], v[2:3], v[4:5], v[6:7]
+; SI-NEXT:    v_fma_f64 v[2:3], v[2:3], v[4:5], v[8:9]
+; SI-NEXT:    buffer_store_dwordx2 v[6:7], v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64 offset:8
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: combine_to_fma_f64_0_2use:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-NEXT:    v_lshlrev_b32_e32 v8, 3, v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[0:1], v8, s[2:3] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[2:3], v8, s[2:3] offset:8 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[4:5], v8, s[2:3] offset:16 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[6:7], v8, s[2:3] offset:24 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_fma_f64 v[4:5], v[0:1], v[2:3], v[4:5]
+; GFX11-NEXT:    v_fma_f64 v[0:1], v[0:1], v[2:3], v[6:7]
+; GFX11-NEXT:    global_store_b64 v8, v[4:5], s[0:1] dlc
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    global_store_b64 v8, v[0:1], s[0:1] offset:8 dlc
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr double, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr double, ptr addrspace(1) %gep.0, i32 1
@@ -72,13 +141,42 @@ define amdgpu_kernel void @combine_to_fma_f64_0_2use(ptr addrspace(1) noalias %o
 }
 
 ; (fadd x, (fmul y, z)) -> (fma y, z, x)
-; FUNC-LABEL: {{^}}combine_to_fma_f64_1:
-; SI-DAG: buffer_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[B:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[C:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16 glc{{$}}
-; SI: v_fma_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[A]], [[B]], [[C]]
-; SI: buffer_store_dwordx2 [[RESULT]]
 define amdgpu_kernel void @combine_to_fma_f64_1(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-LABEL: combine_to_fma_f64_1:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    v_fma_f64 v[2:3], v[2:3], v[4:5], v[6:7]
+; SI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: combine_to_fma_f64_1:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-NEXT:    v_lshlrev_b32_e32 v6, 3, v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[0:1], v6, s[2:3] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[2:3], v6, s[2:3] offset:8 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[4:5], v6, s[2:3] offset:16 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_fma_f64 v[0:1], v[0:1], v[2:3], v[4:5]
+; GFX11-NEXT:    global_store_b64 v6, v[0:1], s[0:1]
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr double, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr double, ptr addrspace(1) %gep.0, i32 1
@@ -96,13 +194,42 @@ define amdgpu_kernel void @combine_to_fma_f64_1(ptr addrspace(1) noalias %out, p
 }
 
 ; (fsub (fmul x, y), z) -> (fma x, y, (fneg z))
-; FUNC-LABEL: {{^}}combine_to_fma_fsub_0_f64:
-; SI-DAG: buffer_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[B:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[C:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16 glc{{$}}
-; SI: v_fma_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[A]], [[B]], -[[C]]
-; SI: buffer_store_dwordx2 [[RESULT]]
 define amdgpu_kernel void @combine_to_fma_fsub_0_f64(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-LABEL: combine_to_fma_fsub_0_f64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    v_fma_f64 v[2:3], v[2:3], v[4:5], -v[6:7]
+; SI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: combine_to_fma_fsub_0_f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-NEXT:    v_lshlrev_b32_e32 v6, 3, v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[0:1], v6, s[2:3] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[2:3], v6, s[2:3] offset:8 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[4:5], v6, s[2:3] offset:16 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_fma_f64 v[0:1], v[0:1], v[2:3], -v[4:5]
+; GFX11-NEXT:    global_store_b64 v6, v[0:1], s[0:1]
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr double, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr double, ptr addrspace(1) %gep.0, i32 1
@@ -120,17 +247,54 @@ define amdgpu_kernel void @combine_to_fma_fsub_0_f64(ptr addrspace(1) noalias %o
 }
 
 ; (fsub (fmul x, y), z) -> (fma x, y, (fneg z))
-; FUNC-LABEL: {{^}}combine_to_fma_fsub_f64_0_2use:
-; SI-DAG: buffer_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[B:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[C:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[D:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:24 glc{{$}}
-; SI-DAG: v_fma_f64 [[RESULT0:v\[[0-9]+:[0-9]+\]]], [[A]], [[B]], -[[C]]
-; SI-DAG: v_fma_f64 [[RESULT1:v\[[0-9]+:[0-9]+\]]], [[A]], [[B]], -[[D]]
-; SI-DAG: buffer_store_dwordx2 [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-; SI-DAG: buffer_store_dwordx2 [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
-; SI: s_endpgm
 define amdgpu_kernel void @combine_to_fma_fsub_f64_0_2use(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-LABEL: combine_to_fma_fsub_f64_0_2use:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dwordx2 v[8:9], v[0:1], s[4:7], 0 addr64 offset:24 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    v_fma_f64 v[6:7], v[2:3], v[4:5], -v[6:7]
+; SI-NEXT:    v_fma_f64 v[2:3], v[2:3], v[4:5], -v[8:9]
+; SI-NEXT:    buffer_store_dwordx2 v[6:7], v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64 offset:8
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: combine_to_fma_fsub_f64_0_2use:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-NEXT:    v_lshlrev_b32_e32 v8, 3, v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[0:1], v8, s[2:3] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[2:3], v8, s[2:3] offset:8 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[4:5], v8, s[2:3] offset:16 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[6:7], v8, s[2:3] offset:24 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_fma_f64 v[4:5], v[0:1], v[2:3], -v[4:5]
+; GFX11-NEXT:    v_fma_f64 v[0:1], v[0:1], v[2:3], -v[6:7]
+; GFX11-NEXT:    global_store_b64 v8, v[4:5], s[0:1] dlc
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    global_store_b64 v8, v[0:1], s[0:1] offset:8 dlc
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr double, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr double, ptr addrspace(1) %gep.0, i32 1
@@ -153,13 +317,42 @@ define amdgpu_kernel void @combine_to_fma_fsub_f64_0_2use(ptr addrspace(1) noali
 }
 
 ; (fsub x, (fmul y, z)) -> (fma (fneg y), z, x)
-; FUNC-LABEL: {{^}}combine_to_fma_fsub_1_f64:
-; SI-DAG: buffer_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[B:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[C:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16 glc{{$}}
-; SI: v_fma_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], -[[A]], [[B]], [[C]]
-; SI: buffer_store_dwordx2 [[RESULT]]
 define amdgpu_kernel void @combine_to_fma_fsub_1_f64(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-LABEL: combine_to_fma_fsub_1_f64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    v_fma_f64 v[2:3], -v[2:3], v[4:5], v[6:7]
+; SI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: combine_to_fma_fsub_1_f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-NEXT:    v_lshlrev_b32_e32 v6, 3, v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[0:1], v6, s[2:3] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[2:3], v6, s[2:3] offset:8 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[4:5], v6, s[2:3] offset:16 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_fma_f64 v[0:1], -v[0:1], v[2:3], v[4:5]
+; GFX11-NEXT:    global_store_b64 v6, v[0:1], s[0:1]
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr double, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr double, ptr addrspace(1) %gep.0, i32 1
@@ -177,17 +370,54 @@ define amdgpu_kernel void @combine_to_fma_fsub_1_f64(ptr addrspace(1) noalias %o
 }
 
 ; (fsub x, (fmul y, z)) -> (fma (fneg y), z, x)
-; FUNC-LABEL: {{^}}combine_to_fma_fsub_1_f64_2use:
-; SI-DAG: buffer_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[B:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[C:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[D:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:24 glc{{$}}
-; SI-DAG: v_fma_f64 [[RESULT0:v\[[0-9]+:[0-9]+\]]], -[[A]], [[B]], [[C]]
-; SI-DAG: v_fma_f64 [[RESULT1:v\[[0-9]+:[0-9]+\]]], -[[A]], [[B]], [[D]]
-; SI-DAG: buffer_store_dwordx2 [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-; SI-DAG: buffer_store_dwordx2 [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
-; SI: s_endpgm
 define amdgpu_kernel void @combine_to_fma_fsub_1_f64_2use(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-LABEL: combine_to_fma_fsub_1_f64_2use:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dwordx2 v[8:9], v[0:1], s[4:7], 0 addr64 offset:24 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], v[6:7]
+; SI-NEXT:    v_fma_f64 v[2:3], -v[2:3], v[4:5], v[8:9]
+; SI-NEXT:    buffer_store_dwordx2 v[6:7], v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64 offset:8
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: combine_to_fma_fsub_1_f64_2use:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-NEXT:    v_lshlrev_b32_e32 v8, 3, v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[0:1], v8, s[2:3] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[2:3], v8, s[2:3] offset:8 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[4:5], v8, s[2:3] offset:16 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[6:7], v8, s[2:3] offset:24 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_fma_f64 v[4:5], -v[0:1], v[2:3], v[4:5]
+; GFX11-NEXT:    v_fma_f64 v[0:1], -v[0:1], v[2:3], v[6:7]
+; GFX11-NEXT:    global_store_b64 v8, v[4:5], s[0:1] dlc
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    global_store_b64 v8, v[0:1], s[0:1] offset:8 dlc
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr double, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr double, ptr addrspace(1) %gep.0, i32 1
@@ -210,13 +440,42 @@ define amdgpu_kernel void @combine_to_fma_fsub_1_f64_2use(ptr addrspace(1) noali
 }
 
 ; (fsub (fneg (fmul x, y)), z) -> (fma (fneg x), y, (fneg z))
-; FUNC-LABEL: {{^}}combine_to_fma_fsub_2_f64:
-; SI-DAG: buffer_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[B:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[C:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16 glc{{$}}
-; SI: v_fma_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], -[[A]], [[B]], -[[C]]
-; SI: buffer_store_dwordx2 [[RESULT]]
 define amdgpu_kernel void @combine_to_fma_fsub_2_f64(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-LABEL: combine_to_fma_fsub_2_f64:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    v_fma_f64 v[2:3], -v[2:3], v[4:5], -v[6:7]
+; SI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: combine_to_fma_fsub_2_f64:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-NEXT:    v_lshlrev_b32_e32 v6, 3, v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[0:1], v6, s[2:3] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[2:3], v6, s[2:3] offset:8 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[4:5], v6, s[2:3] offset:16 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_fma_f64 v[0:1], -v[0:1], v[2:3], -v[4:5]
+; GFX11-NEXT:    global_store_b64 v6, v[0:1], s[0:1]
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr double, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr double, ptr addrspace(1) %gep.0, i32 1
@@ -236,17 +495,54 @@ define amdgpu_kernel void @combine_to_fma_fsub_2_f64(ptr addrspace(1) noalias %o
 }
 
 ; (fsub (fneg (fmul x, y)), z) -> (fma (fneg x), y, (fneg z))
-; FUNC-LABEL: {{^}}combine_to_fma_fsub_2_f64_2uses_neg:
-; SI-DAG: buffer_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[B:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[C:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[D:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:24 glc{{$}}
-; SI-DAG: v_fma_f64 [[RESULT0:v\[[0-9]+:[0-9]+\]]], -[[A]], [[B]], -[[C]]
-; SI-DAG: v_fma_f64 [[RESULT1:v\[[0-9]+:[0-9]+\]]], -[[A]], [[B]], -[[D]]
-; SI-DAG: buffer_store_dwordx2 [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-; SI-DAG: buffer_store_dwordx2 [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
-; SI: s_endpgm
 define amdgpu_kernel void @combine_to_fma_fsub_2_f64_2uses_neg(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-LABEL: combine_to_fma_fsub_2_f64_2uses_neg:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dwordx2 v[8:9], v[0:1], s[4:7], 0 addr64 offset:24 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], -v[6:7]
+; SI-NEXT:    v_fma_f64 v[2:3], -v[2:3], v[4:5], -v[8:9]
+; SI-NEXT:    buffer_store_dwordx2 v[6:7], v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64 offset:8
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: combine_to_fma_fsub_2_f64_2uses_neg:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-NEXT:    v_lshlrev_b32_e32 v8, 3, v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[0:1], v8, s[2:3] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[2:3], v8, s[2:3] offset:8 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[4:5], v8, s[2:3] offset:16 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[6:7], v8, s[2:3] offset:24 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_fma_f64 v[4:5], -v[0:1], v[2:3], -v[4:5]
+; GFX11-NEXT:    v_fma_f64 v[0:1], -v[0:1], v[2:3], -v[6:7]
+; GFX11-NEXT:    global_store_b64 v8, v[4:5], s[0:1] dlc
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    global_store_b64 v8, v[0:1], s[0:1] offset:8 dlc
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr double, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr double, ptr addrspace(1) %gep.0, i32 1
@@ -271,17 +567,54 @@ define amdgpu_kernel void @combine_to_fma_fsub_2_f64_2uses_neg(ptr addrspace(1)
 }
 
 ; (fsub (fneg (fmul x, y)), z) -> (fma (fneg x), y, (fneg z))
-; FUNC-LABEL: {{^}}combine_to_fma_fsub_2_f64_2uses_mul:
-; SI-DAG: buffer_load_dwordx2 [[A:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[B:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[C:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[D:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:24 glc{{$}}
-; SI-DAG: v_fma_f64 [[RESULT0:v\[[0-9]+:[0-9]+\]]], -[[A]], [[B]], -[[C]]
-; SI-DAG: v_fma_f64 [[RESULT1:v\[[0-9]+:[0-9]+\]]], [[A]], [[B]], -[[D]]
-; SI-DAG: buffer_store_dwordx2 [[RESULT0]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64{{$}}
-; SI-DAG: buffer_store_dwordx2 [[RESULT1]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8{{$}}
-; SI: s_endpgm
 define amdgpu_kernel void @combine_to_fma_fsub_2_f64_2uses_mul(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-LABEL: combine_to_fma_fsub_2_f64_2uses_mul:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dwordx2 v[8:9], v[0:1], s[4:7], 0 addr64 offset:24 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    v_fma_f64 v[6:7], -v[2:3], v[4:5], -v[6:7]
+; SI-NEXT:    v_fma_f64 v[2:3], v[2:3], v[4:5], -v[8:9]
+; SI-NEXT:    buffer_store_dwordx2 v[6:7], v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64 offset:8
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: combine_to_fma_fsub_2_f64_2uses_mul:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-NEXT:    v_lshlrev_b32_e32 v8, 3, v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[0:1], v8, s[2:3] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[2:3], v8, s[2:3] offset:8 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[4:5], v8, s[2:3] offset:16 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b64 v[6:7], v8, s[2:3] offset:24 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_fma_f64 v[4:5], -v[0:1], v[2:3], -v[4:5]
+; GFX11-NEXT:    v_fma_f64 v[0:1], v[0:1], v[2:3], -v[6:7]
+; GFX11-NEXT:    global_store_b64 v8, v[4:5], s[0:1] dlc
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    global_store_b64 v8, v[0:1], s[0:1] offset:8 dlc
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr double, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr double, ptr addrspace(1) %gep.0, i32 1
@@ -306,23 +639,102 @@ define amdgpu_kernel void @combine_to_fma_fsub_2_f64_2uses_mul(ptr addrspace(1)
 }
 
 ; fold (fsub (fma x, y, (fmul u, v)), z) -> (fma x, y (fma u, v, (fneg z)))
-
-; FUNC-LABEL: {{^}}aggressive_combine_to_fma_fsub_0_f64:
-; SI-DAG: buffer_load_dwordx2 [[X:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[Y:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[Z:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[U:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:24 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[V:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:32 glc{{$}}
-
-; SI-SAFE: v_mul_f64 [[TMP0:v\[[0-9]+:[0-9]+\]]], [[U]], [[V]]
-; SI-SAFE: v_fma_f64 [[TMP1:v\[[0-9]+:[0-9]+\]]], [[X]], [[Y]], [[TMP0]]
-; SI-SAFE: v_add_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[TMP1]], -[[Z]]
-
-; SI-UNSAFE: v_fma_f64 [[FMA0:v\[[0-9]+:[0-9]+\]]], [[U]], [[V]], -[[Z]]
-; SI-UNSAFE: v_fma_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[X]], [[Y]], [[FMA0]]
-
-; SI: buffer_store_dwordx2 [[RESULT]]
 define amdgpu_kernel void @aggressive_combine_to_fma_fsub_0_f64(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-NOFMA-LABEL: aggressive_combine_to_fma_fsub_0_f64:
+; SI-NOFMA:       ; %bb.0:
+; SI-NOFMA-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NOFMA-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NOFMA-NEXT:    s_mov_b32 s6, 0
+; SI-NOFMA-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; SI-NOFMA-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NOFMA-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[8:9], v[0:1], s[4:7], 0 addr64 offset:24 glc
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[10:11], v[0:1], s[4:7], 0 addr64 offset:32 glc
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NOFMA-NEXT:    v_mul_f64 v[8:9], v[8:9], v[10:11]
+; SI-NOFMA-NEXT:    v_fma_f64 v[2:3], v[2:3], v[4:5], v[8:9]
+; SI-NOFMA-NEXT:    v_add_f64 v[2:3], v[2:3], -v[6:7]
+; SI-NOFMA-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; SI-NOFMA-NEXT:    s_endpgm
+;
+; SI-FMA-LABEL: aggressive_combine_to_fma_fsub_0_f64:
+; SI-FMA:       ; %bb.0:
+; SI-FMA-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-FMA-NEXT:    s_mov_b32 s7, 0xf000
+; SI-FMA-NEXT:    s_mov_b32 s6, 0
+; SI-FMA-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; SI-FMA-NEXT:    v_mov_b32_e32 v1, 0
+; SI-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-FMA-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-FMA-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; SI-FMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-FMA-NEXT:    buffer_load_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-FMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-FMA-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-FMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-FMA-NEXT:    buffer_load_dwordx2 v[8:9], v[0:1], s[4:7], 0 addr64 offset:24 glc
+; SI-FMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-FMA-NEXT:    buffer_load_dwordx2 v[10:11], v[0:1], s[4:7], 0 addr64 offset:32 glc
+; SI-FMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-FMA-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-FMA-NEXT:    v_fma_f64 v[6:7], v[8:9], v[10:11], -v[6:7]
+; SI-FMA-NEXT:    v_fma_f64 v[2:3], v[2:3], v[4:5], v[6:7]
+; SI-FMA-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; SI-FMA-NEXT:    s_endpgm
+;
+; GFX11-NOFMA-LABEL: aggressive_combine_to_fma_fsub_0_f64:
+; GFX11-NOFMA:       ; %bb.0:
+; GFX11-NOFMA-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-NOFMA-NEXT:    v_lshlrev_b32_e32 v10, 3, v0
+; GFX11-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[0:1], v10, s[2:3] glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[2:3], v10, s[2:3] offset:8 glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[4:5], v10, s[2:3] offset:16 glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[6:7], v10, s[2:3] offset:24 glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[8:9], v10, s[2:3] offset:32 glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    v_mul_f64 v[6:7], v[6:7], v[8:9]
+; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NOFMA-NEXT:    v_fma_f64 v[0:1], v[0:1], v[2:3], v[6:7]
+; GFX11-NOFMA-NEXT:    v_add_f64 v[0:1], v[0:1], -v[4:5]
+; GFX11-NOFMA-NEXT:    global_store_b64 v10, v[0:1], s[0:1]
+; GFX11-NOFMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NOFMA-NEXT:    s_endpgm
+;
+; GFX11-FMA-LABEL: aggressive_combine_to_fma_fsub_0_f64:
+; GFX11-FMA:       ; %bb.0:
+; GFX11-FMA-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-FMA-NEXT:    v_lshlrev_b32_e32 v10, 3, v0
+; GFX11-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FMA-NEXT:    global_load_b64 v[0:1], v10, s[2:3] glc dlc
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FMA-NEXT:    global_load_b64 v[2:3], v10, s[2:3] offset:8 glc dlc
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FMA-NEXT:    global_load_b64 v[4:5], v10, s[2:3] offset:16 glc dlc
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FMA-NEXT:    global_load_b64 v[6:7], v10, s[2:3] offset:24 glc dlc
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FMA-NEXT:    global_load_b64 v[8:9], v10, s[2:3] offset:32 glc dlc
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FMA-NEXT:    v_fma_f64 v[4:5], v[6:7], v[8:9], -v[4:5]
+; GFX11-FMA-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FMA-NEXT:    v_fma_f64 v[0:1], v[0:1], v[2:3], v[4:5]
+; GFX11-FMA-NEXT:    global_store_b64 v10, v[0:1], s[0:1]
+; GFX11-FMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-FMA-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr double, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr double, ptr addrspace(1) %gep.0, i32 1
@@ -347,23 +759,102 @@ define amdgpu_kernel void @aggressive_combine_to_fma_fsub_0_f64(ptr addrspace(1)
 
 ; fold (fsub x, (fma y, z, (fmul u, v)))
 ;   -> (fma (fneg y), z, (fma (fneg u), v, x))
-
-; FUNC-LABEL: {{^}}aggressive_combine_to_fma_fsub_1_f64:
-; SI-DAG: buffer_load_dwordx2 [[X:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[Y:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:8 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[Z:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:16 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[U:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:24 glc{{$}}
-; SI-DAG: buffer_load_dwordx2 [[V:v\[[0-9]+:[0-9]+\]]], v{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0 addr64 offset:32 glc{{$}}
-
-; SI-SAFE: v_mul_f64 [[TMP0:v\[[0-9]+:[0-9]+\]]], [[U]], [[V]]
-; SI-SAFE: v_fma_f64 [[TMP1:v\[[0-9]+:[0-9]+\]]], [[Y]], [[Z]], [[TMP0]]
-; SI-SAFE: v_add_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[X]], -[[TMP1]]
-
-; SI-UNSAFE: v_fma_f64 [[FMA0:v\[[0-9]+:[0-9]+\]]], -[[U]], [[V]], [[X]]
-; SI-UNSAFE: v_fma_f64 [[RESULT:v\[[0-9]+:[0-9]+\]]], -[[Y]], [[Z]], [[FMA0]]
-
-; SI: buffer_store_dwordx2 [[RESULT]]
 define amdgpu_kernel void @aggressive_combine_to_fma_fsub_1_f64(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #1 {
+; SI-NOFMA-LABEL: aggressive_combine_to_fma_fsub_1_f64:
+; SI-NOFMA:       ; %bb.0:
+; SI-NOFMA-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NOFMA-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NOFMA-NEXT:    s_mov_b32 s6, 0
+; SI-NOFMA-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; SI-NOFMA-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NOFMA-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[8:9], v[0:1], s[4:7], 0 addr64 offset:24 glc
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    buffer_load_dwordx2 v[10:11], v[0:1], s[4:7], 0 addr64 offset:32 glc
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NOFMA-NEXT:    v_mul_f64 v[8:9], v[8:9], v[10:11]
+; SI-NOFMA-NEXT:    v_fma_f64 v[4:5], v[4:5], v[6:7], v[8:9]
+; SI-NOFMA-NEXT:    v_add_f64 v[2:3], v[2:3], -v[4:5]
+; SI-NOFMA-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; SI-NOFMA-NEXT:    s_endpgm
+;
+; SI-FMA-LABEL: aggressive_combine_to_fma_fsub_1_f64:
+; SI-FMA:       ; %bb.0:
+; SI-FMA-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-FMA-NEXT:    s_mov_b32 s7, 0xf000
+; SI-FMA-NEXT:    s_mov_b32 s6, 0
+; SI-FMA-NEXT:    v_lshlrev_b32_e32 v0, 3, v0
+; SI-FMA-NEXT:    v_mov_b32_e32 v1, 0
+; SI-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-FMA-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-FMA-NEXT:    buffer_load_dwordx2 v[2:3], v[0:1], s[4:7], 0 addr64 glc
+; SI-FMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-FMA-NEXT:    buffer_load_dwordx2 v[4:5], v[0:1], s[4:7], 0 addr64 offset:8 glc
+; SI-FMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-FMA-NEXT:    buffer_load_dwordx2 v[6:7], v[0:1], s[4:7], 0 addr64 offset:16 glc
+; SI-FMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-FMA-NEXT:    buffer_load_dwordx2 v[8:9], v[0:1], s[4:7], 0 addr64 offset:24 glc
+; SI-FMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-FMA-NEXT:    buffer_load_dwordx2 v[10:11], v[0:1], s[4:7], 0 addr64 offset:32 glc
+; SI-FMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-FMA-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-FMA-NEXT:    v_fma_f64 v[2:3], -v[8:9], v[10:11], v[2:3]
+; SI-FMA-NEXT:    v_fma_f64 v[2:3], -v[4:5], v[6:7], v[2:3]
+; SI-FMA-NEXT:    buffer_store_dwordx2 v[2:3], v[0:1], s[0:3], 0 addr64
+; SI-FMA-NEXT:    s_endpgm
+;
+; GFX11-NOFMA-LABEL: aggressive_combine_to_fma_fsub_1_f64:
+; GFX11-NOFMA:       ; %bb.0:
+; GFX11-NOFMA-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-NOFMA-NEXT:    v_lshlrev_b32_e32 v10, 3, v0
+; GFX11-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[0:1], v10, s[2:3] glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[2:3], v10, s[2:3] offset:8 glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[4:5], v10, s[2:3] offset:16 glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[6:7], v10, s[2:3] offset:24 glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b64 v[8:9], v10, s[2:3] offset:32 glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    v_mul_f64 v[6:7], v[6:7], v[8:9]
+; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NOFMA-NEXT:    v_fma_f64 v[2:3], v[2:3], v[4:5], v[6:7]
+; GFX11-NOFMA-NEXT:    v_add_f64 v[0:1], v[0:1], -v[2:3]
+; GFX11-NOFMA-NEXT:    global_store_b64 v10, v[0:1], s[0:1]
+; GFX11-NOFMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NOFMA-NEXT:    s_endpgm
+;
+; GFX11-FMA-LABEL: aggressive_combine_to_fma_fsub_1_f64:
+; GFX11-FMA:       ; %bb.0:
+; GFX11-FMA-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-FMA-NEXT:    v_lshlrev_b32_e32 v10, 3, v0
+; GFX11-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FMA-NEXT:    global_load_b64 v[0:1], v10, s[2:3] glc dlc
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FMA-NEXT:    global_load_b64 v[2:3], v10, s[2:3] offset:8 glc dlc
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FMA-NEXT:    global_load_b64 v[4:5], v10, s[2:3] offset:16 glc dlc
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FMA-NEXT:    global_load_b64 v[6:7], v10, s[2:3] offset:24 glc dlc
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FMA-NEXT:    global_load_b64 v[8:9], v10, s[2:3] offset:32 glc dlc
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FMA-NEXT:    v_fma_f64 v[0:1], -v[6:7], v[8:9], v[0:1]
+; GFX11-FMA-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FMA-NEXT:    v_fma_f64 v[0:1], -v[2:3], v[4:5], v[0:1]
+; GFX11-FMA-NEXT:    global_store_b64 v10, v[0:1], s[0:1]
+; GFX11-FMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-FMA-NEXT:    s_endpgm
   %tid = tail call i32 @llvm.amdgcn.workitem.id.x() #0
   %gep.0 = getelementptr double, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr double, ptr addrspace(1) %gep.0, i32 1
@@ -391,12 +882,87 @@ define amdgpu_kernel void @aggressive_combine_to_fma_fsub_1_f64(ptr addrspace(1)
 ; Patterns (+ fneg variants): mul(add(1.0,x),y), mul(sub(1.0,x),y), mul(sub(x,1.0),y)
 ;
 
-; FUNC-LABEL: {{^}}test_f32_mul_add_x_one_y:
-; SI-NOFMA: v_add_f32_e32 [[VS:v[0-9]]], 1.0, [[VX:v[0-9]]]
-; SI-NOFMA: v_mul_f32_e32 {{v[0-9]}}, [[VS]], [[VY:v[0-9]]]
-;
-; SI-FMA: v_fma_f32 {{v[0-9]}}, [[VX:v[0-9]]], [[VY:v[0-9]]], [[VY:v[0-9]]]
 define amdgpu_kernel void @test_f32_mul_add_x_one_y(ptr addrspace(1) %out,
+; SI-NOFMA-LABEL: test_f32_mul_add_x_one_y:
+; SI-NOFMA:       ; %bb.0:
+; SI-NOFMA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NOFMA-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-NOFMA-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NOFMA-NEXT:    s_mov_b32 s2, -1
+; SI-NOFMA-NEXT:    s_mov_b32 s14, s2
+; SI-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NOFMA-NEXT:    s_mov_b32 s12, s6
+; SI-NOFMA-NEXT:    s_mov_b32 s13, s7
+; SI-NOFMA-NEXT:    s_mov_b32 s15, s3
+; SI-NOFMA-NEXT:    s_mov_b32 s10, s2
+; SI-NOFMA-NEXT:    s_mov_b32 s11, s3
+; SI-NOFMA-NEXT:    buffer_load_dword v0, off, s[12:15], 0 glc
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    buffer_load_dword v1, off, s[8:11], 0 glc
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    s_mov_b32 s0, s4
+; SI-NOFMA-NEXT:    s_mov_b32 s1, s5
+; SI-NOFMA-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; SI-NOFMA-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-NOFMA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NOFMA-NEXT:    s_endpgm
+;
+; SI-FMA-LABEL: test_f32_mul_add_x_one_y:
+; SI-FMA:       ; %bb.0:
+; SI-FMA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-FMA-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-FMA-NEXT:    s_mov_b32 s3, 0xf000
+; SI-FMA-NEXT:    s_mov_b32 s2, -1
+; SI-FMA-NEXT:    s_mov_b32 s14, s2
+; SI-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-FMA-NEXT:    s_mov_b32 s12, s6
+; SI-FMA-NEXT:    s_mov_b32 s13, s7
+; SI-FMA-NEXT:    s_mov_b32 s15, s3
+; SI-FMA-NEXT:    s_mov_b32 s10, s2
+; SI-FMA-NEXT:    s_mov_b32 s11, s3
+; SI-FMA-NEXT:    buffer_load_dword v0, off, s[12:15], 0 glc
+; SI-FMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-FMA-NEXT:    buffer_load_dword v1, off, s[8:11], 0 glc
+; SI-FMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-FMA-NEXT:    s_mov_b32 s0, s4
+; SI-FMA-NEXT:    s_mov_b32 s1, s5
+; SI-FMA-NEXT:    v_fma_f32 v0, v0, v1, v1
+; SI-FMA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-FMA-NEXT:    s_endpgm
+;
+; GFX11-NOFMA-LABEL: test_f32_mul_add_x_one_y:
+; GFX11-NOFMA:       ; %bb.0:
+; GFX11-NOFMA-NEXT:    s_clause 0x1
+; GFX11-NOFMA-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GFX11-NOFMA-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GFX11-NOFMA-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b32 v1, v0, s[6:7] glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b32 v2, v0, s[0:1] glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NOFMA-NEXT:    v_mul_f32_e32 v1, v1, v2
+; GFX11-NOFMA-NEXT:    global_store_b32 v0, v1, s[4:5]
+; GFX11-NOFMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NOFMA-NEXT:    s_endpgm
+;
+; GFX11-FMA-LABEL: test_f32_mul_add_x_one_y:
+; GFX11-FMA:       ; %bb.0:
+; GFX11-FMA-NEXT:    s_clause 0x1
+; GFX11-FMA-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GFX11-FMA-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GFX11-FMA-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FMA-NEXT:    global_load_b32 v1, v0, s[6:7] glc dlc
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FMA-NEXT:    global_load_b32 v2, v0, s[0:1] glc dlc
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FMA-NEXT:    v_fmac_f32_e32 v2, v1, v2
+; GFX11-FMA-NEXT:    global_store_b32 v0, v2, s[4:5]
+; GFX11-FMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-FMA-NEXT:    s_endpgm
                                         ptr addrspace(1) %in1,
                                         ptr addrspace(1) %in2) {
   %x = load volatile float, ptr addrspace(1) %in1
@@ -407,12 +973,87 @@ define amdgpu_kernel void @test_f32_mul_add_x_one_y(ptr addrspace(1) %out,
   ret void
 }
 
-; FUNC-LABEL: {{^}}test_f32_mul_y_add_x_one:
-; SI-NOFMA: v_add_f32_e32 [[VS:v[0-9]]], 1.0, [[VX:v[0-9]]]
-; SI-NOFMA: v_mul_f32_e32 {{v[0-9]}}, [[VY:v[0-9]]], [[VS]]
-;
-; SI-FMA: v_fma_f32 {{v[0-9]}}, [[VX:v[0-9]]], [[VY:v[0-9]]], [[VY:v[0-9]]]
 define amdgpu_kernel void @test_f32_mul_y_add_x_one(ptr addrspace(1) %out,
+; SI-NOFMA-LABEL: test_f32_mul_y_add_x_one:
+; SI-NOFMA:       ; %bb.0:
+; SI-NOFMA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NOFMA-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-NOFMA-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NOFMA-NEXT:    s_mov_b32 s2, -1
+; SI-NOFMA-NEXT:    s_mov_b32 s14, s2
+; SI-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NOFMA-NEXT:    s_mov_b32 s12, s6
+; SI-NOFMA-NEXT:    s_mov_b32 s13, s7
+; SI-NOFMA-NEXT:    s_mov_b32 s15, s3
+; SI-NOFMA-NEXT:    s_mov_b32 s10, s2
+; SI-NOFMA-NEXT:    s_mov_b32 s11, s3
+; SI-NOFMA-NEXT:    buffer_load_dword v0, off, s[12:15], 0 glc
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    buffer_load_dword v1, off, s[8:11], 0 glc
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    s_mov_b32 s0, s4
+; SI-NOFMA-NEXT:    s_mov_b32 s1, s5
+; SI-NOFMA-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; SI-NOFMA-NEXT:    v_mul_f32_e32 v0, v1, v0
+; SI-NOFMA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NOFMA-NEXT:    s_endpgm
+;
+; SI-FMA-LABEL: test_f32_mul_y_add_x_one:
+; SI-FMA:       ; %bb.0:
+; SI-FMA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-FMA-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-FMA-NEXT:    s_mov_b32 s3, 0xf000
+; SI-FMA-NEXT:    s_mov_b32 s2, -1
+; SI-FMA-NEXT:    s_mov_b32 s14, s2
+; SI-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-FMA-NEXT:    s_mov_b32 s12, s6
+; SI-FMA-NEXT:    s_mov_b32 s13, s7
+; SI-FMA-NEXT:    s_mov_b32 s15, s3
+; SI-FMA-NEXT:    s_mov_b32 s10, s2
+; SI-FMA-NEXT:    s_mov_b32 s11, s3
+; SI-FMA-NEXT:    buffer_load_dword v0, off, s[12:15], 0 glc
+; SI-FMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-FMA-NEXT:    buffer_load_dword v1, off, s[8:11], 0 glc
+; SI-FMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-FMA-NEXT:    s_mov_b32 s0, s4
+; SI-FMA-NEXT:    s_mov_b32 s1, s5
+; SI-FMA-NEXT:    v_fma_f32 v0, v0, v1, v1
+; SI-FMA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-FMA-NEXT:    s_endpgm
+;
+; GFX11-NOFMA-LABEL: test_f32_mul_y_add_x_one:
+; GFX11-NOFMA:       ; %bb.0:
+; GFX11-NOFMA-NEXT:    s_clause 0x1
+; GFX11-NOFMA-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GFX11-NOFMA-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GFX11-NOFMA-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b32 v1, v0, s[6:7] glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    global_load_b32 v2, v0, s[0:1] glc dlc
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NOFMA-NEXT:    v_mul_f32_e32 v1, v2, v1
+; GFX11-NOFMA-NEXT:    global_store_b32 v0, v1, s[4:5]
+; GFX11-NOFMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NOFMA-NEXT:    s_endpgm
+;
+; GFX11-FMA-LABEL: test_f32_mul_y_add_x_one:
+; GFX11-FMA:       ; %bb.0:
+; GFX11-FMA-NEXT:    s_clause 0x1
+; GFX11-FMA-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GFX11-FMA-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GFX11-FMA-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FMA-NEXT:    global_load_b32 v1, v0, s[6:7] glc dlc
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FMA-NEXT:    global_load_b32 v2, v0, s[0:1] glc dlc
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FMA-NEXT:    v_fmac_f32_e32 v2, v1, v2
+; GFX11-FMA-NEXT:    global_store_b32 v0, v2, s[4:5]
+; GFX11-FMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-FMA-NEXT:    s_endpgm
                                         ptr addrspace(1) %in1,
                                         ptr addrspace(1) %in2) {
   %x = load volatile float, ptr addrspace(1) %in1
@@ -423,12 +1064,87 @@ define amdgpu_kernel void @test_f32_mul_y_add_x_one(ptr addrspace(1) %out,
   ret void
 }
 
-; FUNC-LABEL: {{^}}test_f32_mul_add_x_negone_y:
-; SI-NOFMA: v_add_f32_e32 [[VS:v[0-9]]], -1.0, [[VX:v[0-9]]]
-; SI-NOFMA: v_mul_f32_e32 {{v[0-9]}}, [[VS]], [[VY:v[0-9]]]
-;
-; SI-FMA: v_fma_f32 {{v[0-9]}}, [[VX:v[0-9]]], [[VY:v[0-9]]], -[[VY:v[0-9]]]
 define amdgpu_kernel void @test_f32_mul_add_x_negone_y(ptr addrspace(1) %out,
+; SI-NOFMA-LABEL: test_f32_mul_add_x_negone_y:
+; SI-NOFMA:       ; %bb.0:
+; SI-NOFMA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NOFMA-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-NOFMA-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NOFMA-NEXT:    s_mov_b32 s2, -1
+; SI-NOFMA-NEXT:    s_mov_b32 s14, s2
+; SI-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NOFMA-NEXT:    s_mov_b32 s12, s6
+; SI-NOFMA-NEXT:    s_mov_b32 s13, s7
+; SI-NOFMA-NEXT:    s_mov_b32 s15, s3
+; SI-NOFMA-NEXT:    s_mov_b32 s10, s2
+; SI-NOFMA-NEXT:    s_mov_b32 s11, s3
+; SI-NOFMA-NEXT:    buffer_load_dword v0, off, s[12:15], 0
+; SI-NOFMA-NEXT:    buffer_load_dword v1, off, s[8:11], 0
+; SI-NOFMA-NEXT:    s_mov_b32 s0, s4
+; SI-NOFMA-NEXT:    s_mov_b32 s1, s5
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(1)
+; SI-NOFMA-NEXT:    v_add_f32_e32 v0, -1.0, v0
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-NOFMA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NOFMA-NEXT:    s_endpgm
+;
+; SI-FMA-LABEL: test_f32_mul_add_x_negone_y:
+; SI-FMA:       ; %bb.0:
+; SI-FMA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-FMA-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-FMA-NEXT:    s_mov_b32 s3, 0xf000
+; SI-FMA-NEXT:    s_mov_b32 s2, -1
+; SI-FMA-NEXT:    s_mov_b32 s14, s2
+; SI-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-FMA-NEXT:    s_mov_b32 s12, s6
+; SI-FMA-NEXT:    s_mov_b32 s13, s7
+; SI-FMA-NEXT:    s_mov_b32 s15, s3
+; SI-FMA-NEXT:    s_mov_b32 s10, s2
+; SI-FMA-NEXT:    s_mov_b32 s11, s3
+; SI-FMA-NEXT:    buffer_load_dword v0, off, s[12:15], 0
+; SI-FMA-NEXT:    buffer_load_dword v1, off, s[8:11], 0
+; SI-FMA-NEXT:    s_mov_b32 s0, s4
+; SI-FMA-NEXT:    s_mov_b32 s1, s5
+; SI-FMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-FMA-NEXT:    v_fma_f32 v0, v0, v1, -v1
+; SI-FMA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-FMA-NEXT:    s_endpgm
+;
+; GFX11-NOFMA-LABEL: test_f32_mul_add_x_negone_y:
+; GFX11-NOFMA:       ; %bb.0:
+; GFX11-NOFMA-NEXT:    s_clause 0x1
+; GFX11-NOFMA-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GFX11-NOFMA-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GFX11-NOFMA-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NOFMA-NEXT:    s_clause 0x1
+; GFX11-NOFMA-NEXT:    global_load_b32 v1, v0, s[6:7]
+; GFX11-NOFMA-NEXT:    global_load_b32 v2, v0, s[0:1]
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-NOFMA-NEXT:    v_add_f32_e32 v1, -1.0, v1
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NOFMA-NEXT:    v_mul_f32_e32 v1, v1, v2
+; GFX11-NOFMA-NEXT:    global_store_b32 v0, v1, s[4:5]
+; GFX11-NOFMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NOFMA-NEXT:    s_endpgm
+;
+; GFX11-FMA-LABEL: test_f32_mul_add_x_negone_y:
+; GFX11-FMA:       ; %bb.0:
+; GFX11-FMA-NEXT:    s_clause 0x1
+; GFX11-FMA-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GFX11-FMA-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GFX11-FMA-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FMA-NEXT:    s_clause 0x1
+; GFX11-FMA-NEXT:    global_load_b32 v1, v0, s[6:7]
+; GFX11-FMA-NEXT:    global_load_b32 v2, v0, s[0:1]
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FMA-NEXT:    v_fma_f32 v1, v1, v2, -v2
+; GFX11-FMA-NEXT:    global_store_b32 v0, v1, s[4:5]
+; GFX11-FMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-FMA-NEXT:    s_endpgm
                                            ptr addrspace(1) %in1,
                                            ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
@@ -439,12 +1155,87 @@ define amdgpu_kernel void @test_f32_mul_add_x_negone_y(ptr addrspace(1) %out,
   ret void
 }
 
-; FUNC-LABEL: {{^}}test_f32_mul_y_add_x_negone:
-; SI-NOFMA: v_add_f32_e32 [[VS:v[0-9]]], -1.0, [[VX:v[0-9]]]
-; SI-NOFMA: v_mul_f32_e32 {{v[0-9]}}, [[VY:v[0-9]]], [[VS]]
-;
-; SI-FMA: v_fma_f32 {{v[0-9]}}, [[VX:v[0-9]]], [[VY:v[0-9]]], -[[VY:v[0-9]]]
 define amdgpu_kernel void @test_f32_mul_y_add_x_negone(ptr addrspace(1) %out,
+; SI-NOFMA-LABEL: test_f32_mul_y_add_x_negone:
+; SI-NOFMA:       ; %bb.0:
+; SI-NOFMA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NOFMA-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-NOFMA-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NOFMA-NEXT:    s_mov_b32 s2, -1
+; SI-NOFMA-NEXT:    s_mov_b32 s14, s2
+; SI-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NOFMA-NEXT:    s_mov_b32 s12, s6
+; SI-NOFMA-NEXT:    s_mov_b32 s13, s7
+; SI-NOFMA-NEXT:    s_mov_b32 s15, s3
+; SI-NOFMA-NEXT:    s_mov_b32 s10, s2
+; SI-NOFMA-NEXT:    s_mov_b32 s11, s3
+; SI-NOFMA-NEXT:    buffer_load_dword v0, off, s[12:15], 0
+; SI-NOFMA-NEXT:    buffer_load_dword v1, off, s[8:11], 0
+; SI-NOFMA-NEXT:    s_mov_b32 s0, s4
+; SI-NOFMA-NEXT:    s_mov_b32 s1, s5
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(1)
+; SI-NOFMA-NEXT:    v_add_f32_e32 v0, -1.0, v0
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    v_mul_f32_e32 v0, v1, v0
+; SI-NOFMA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NOFMA-NEXT:    s_endpgm
+;
+; SI-FMA-LABEL: test_f32_mul_y_add_x_negone:
+; SI-FMA:       ; %bb.0:
+; SI-FMA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-FMA-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-FMA-NEXT:    s_mov_b32 s3, 0xf000
+; SI-FMA-NEXT:    s_mov_b32 s2, -1
+; SI-FMA-NEXT:    s_mov_b32 s14, s2
+; SI-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-FMA-NEXT:    s_mov_b32 s12, s6
+; SI-FMA-NEXT:    s_mov_b32 s13, s7
+; SI-FMA-NEXT:    s_mov_b32 s15, s3
+; SI-FMA-NEXT:    s_mov_b32 s10, s2
+; SI-FMA-NEXT:    s_mov_b32 s11, s3
+; SI-FMA-NEXT:    buffer_load_dword v0, off, s[12:15], 0
+; SI-FMA-NEXT:    buffer_load_dword v1, off, s[8:11], 0
+; SI-FMA-NEXT:    s_mov_b32 s0, s4
+; SI-FMA-NEXT:    s_mov_b32 s1, s5
+; SI-FMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-FMA-NEXT:    v_fma_f32 v0, v0, v1, -v1
+; SI-FMA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-FMA-NEXT:    s_endpgm
+;
+; GFX11-NOFMA-LABEL: test_f32_mul_y_add_x_negone:
+; GFX11-NOFMA:       ; %bb.0:
+; GFX11-NOFMA-NEXT:    s_clause 0x1
+; GFX11-NOFMA-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GFX11-NOFMA-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GFX11-NOFMA-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NOFMA-NEXT:    s_clause 0x1
+; GFX11-NOFMA-NEXT:    global_load_b32 v1, v0, s[6:7]
+; GFX11-NOFMA-NEXT:    global_load_b32 v2, v0, s[0:1]
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-NOFMA-NEXT:    v_add_f32_e32 v1, -1.0, v1
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NOFMA-NEXT:    v_mul_f32_e32 v1, v2, v1
+; GFX11-NOFMA-NEXT:    global_store_b32 v0, v1, s[4:5]
+; GFX11-NOFMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NOFMA-NEXT:    s_endpgm
+;
+; GFX11-FMA-LABEL: test_f32_mul_y_add_x_negone:
+; GFX11-FMA:       ; %bb.0:
+; GFX11-FMA-NEXT:    s_clause 0x1
+; GFX11-FMA-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GFX11-FMA-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GFX11-FMA-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FMA-NEXT:    s_clause 0x1
+; GFX11-FMA-NEXT:    global_load_b32 v1, v0, s[6:7]
+; GFX11-FMA-NEXT:    global_load_b32 v2, v0, s[0:1]
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FMA-NEXT:    v_fma_f32 v1, v1, v2, -v2
+; GFX11-FMA-NEXT:    global_store_b32 v0, v1, s[4:5]
+; GFX11-FMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-FMA-NEXT:    s_endpgm
                                            ptr addrspace(1) %in1,
                                            ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
@@ -455,12 +1246,87 @@ define amdgpu_kernel void @test_f32_mul_y_add_x_negone(ptr addrspace(1) %out,
   ret void
 }
 
-; FUNC-LABEL: {{^}}test_f32_mul_sub_one_x_y:
-; SI-NOFMA: v_sub_f32_e32 [[VS:v[0-9]]], 1.0, [[VX:v[0-9]]]
-; SI-NOFMA: v_mul_f32_e32 {{v[0-9]}}, [[VS]], [[VY:v[0-9]]]
-;
-; SI-FMA: v_fma_f32 {{v[0-9]}}, -[[VX:v[0-9]]], [[VY:v[0-9]]], [[VY:v[0-9]]]
 define amdgpu_kernel void @test_f32_mul_sub_one_x_y(ptr addrspace(1) %out,
+; SI-NOFMA-LABEL: test_f32_mul_sub_one_x_y:
+; SI-NOFMA:       ; %bb.0:
+; SI-NOFMA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NOFMA-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-NOFMA-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NOFMA-NEXT:    s_mov_b32 s2, -1
+; SI-NOFMA-NEXT:    s_mov_b32 s14, s2
+; SI-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NOFMA-NEXT:    s_mov_b32 s12, s6
+; SI-NOFMA-NEXT:    s_mov_b32 s13, s7
+; SI-NOFMA-NEXT:    s_mov_b32 s15, s3
+; SI-NOFMA-NEXT:    s_mov_b32 s10, s2
+; SI-NOFMA-NEXT:    s_mov_b32 s11, s3
+; SI-NOFMA-NEXT:    buffer_load_dword v0, off, s[12:15], 0
+; SI-NOFMA-NEXT:    buffer_load_dword v1, off, s[8:11], 0
+; SI-NOFMA-NEXT:    s_mov_b32 s0, s4
+; SI-NOFMA-NEXT:    s_mov_b32 s1, s5
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(1)
+; SI-NOFMA-NEXT:    v_sub_f32_e32 v0, 1.0, v0
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-NOFMA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NOFMA-NEXT:    s_endpgm
+;
+; SI-FMA-LABEL: test_f32_mul_sub_one_x_y:
+; SI-FMA:       ; %bb.0:
+; SI-FMA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-FMA-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-FMA-NEXT:    s_mov_b32 s3, 0xf000
+; SI-FMA-NEXT:    s_mov_b32 s2, -1
+; SI-FMA-NEXT:    s_mov_b32 s14, s2
+; SI-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-FMA-NEXT:    s_mov_b32 s12, s6
+; SI-FMA-NEXT:    s_mov_b32 s13, s7
+; SI-FMA-NEXT:    s_mov_b32 s15, s3
+; SI-FMA-NEXT:    s_mov_b32 s10, s2
+; SI-FMA-NEXT:    s_mov_b32 s11, s3
+; SI-FMA-NEXT:    buffer_load_dword v0, off, s[12:15], 0
+; SI-FMA-NEXT:    buffer_load_dword v1, off, s[8:11], 0
+; SI-FMA-NEXT:    s_mov_b32 s0, s4
+; SI-FMA-NEXT:    s_mov_b32 s1, s5
+; SI-FMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-FMA-NEXT:    v_fma_f32 v0, -v0, v1, v1
+; SI-FMA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-FMA-NEXT:    s_endpgm
+;
+; GFX11-NOFMA-LABEL: test_f32_mul_sub_one_x_y:
+; GFX11-NOFMA:       ; %bb.0:
+; GFX11-NOFMA-NEXT:    s_clause 0x1
+; GFX11-NOFMA-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GFX11-NOFMA-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GFX11-NOFMA-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NOFMA-NEXT:    s_clause 0x1
+; GFX11-NOFMA-NEXT:    global_load_b32 v1, v0, s[6:7]
+; GFX11-NOFMA-NEXT:    global_load_b32 v2, v0, s[0:1]
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-NOFMA-NEXT:    v_sub_f32_e32 v1, 1.0, v1
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NOFMA-NEXT:    v_mul_f32_e32 v1, v1, v2
+; GFX11-NOFMA-NEXT:    global_store_b32 v0, v1, s[4:5]
+; GFX11-NOFMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NOFMA-NEXT:    s_endpgm
+;
+; GFX11-FMA-LABEL: test_f32_mul_sub_one_x_y:
+; GFX11-FMA:       ; %bb.0:
+; GFX11-FMA-NEXT:    s_clause 0x1
+; GFX11-FMA-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GFX11-FMA-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GFX11-FMA-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FMA-NEXT:    s_clause 0x1
+; GFX11-FMA-NEXT:    global_load_b32 v1, v0, s[6:7]
+; GFX11-FMA-NEXT:    global_load_b32 v2, v0, s[0:1]
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FMA-NEXT:    v_fma_f32 v1, -v1, v2, v2
+; GFX11-FMA-NEXT:    global_store_b32 v0, v1, s[4:5]
+; GFX11-FMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-FMA-NEXT:    s_endpgm
                                         ptr addrspace(1) %in1,
                                         ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
@@ -471,12 +1337,87 @@ define amdgpu_kernel void @test_f32_mul_sub_one_x_y(ptr addrspace(1) %out,
   ret void
 }
 
-; FUNC-LABEL: {{^}}test_f32_mul_y_sub_one_x:
-; SI-NOFMA: v_sub_f32_e32 [[VS:v[0-9]]], 1.0, [[VX:v[0-9]]]
-; SI-NOFMA: v_mul_f32_e32 {{v[0-9]}}, [[VY:v[0-9]]], [[VS]]
-;
-; SI-FMA: v_fma_f32 {{v[0-9]}}, -[[VX:v[0-9]]], [[VY:v[0-9]]], [[VY:v[0-9]]]
 define amdgpu_kernel void @test_f32_mul_y_sub_one_x(ptr addrspace(1) %out,
+; SI-NOFMA-LABEL: test_f32_mul_y_sub_one_x:
+; SI-NOFMA:       ; %bb.0:
+; SI-NOFMA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NOFMA-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-NOFMA-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NOFMA-NEXT:    s_mov_b32 s2, -1
+; SI-NOFMA-NEXT:    s_mov_b32 s14, s2
+; SI-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NOFMA-NEXT:    s_mov_b32 s12, s6
+; SI-NOFMA-NEXT:    s_mov_b32 s13, s7
+; SI-NOFMA-NEXT:    s_mov_b32 s15, s3
+; SI-NOFMA-NEXT:    s_mov_b32 s10, s2
+; SI-NOFMA-NEXT:    s_mov_b32 s11, s3
+; SI-NOFMA-NEXT:    buffer_load_dword v0, off, s[12:15], 0
+; SI-NOFMA-NEXT:    buffer_load_dword v1, off, s[8:11], 0
+; SI-NOFMA-NEXT:    s_mov_b32 s0, s4
+; SI-NOFMA-NEXT:    s_mov_b32 s1, s5
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(1)
+; SI-NOFMA-NEXT:    v_sub_f32_e32 v0, 1.0, v0
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    v_mul_f32_e32 v0, v1, v0
+; SI-NOFMA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NOFMA-NEXT:    s_endpgm
+;
+; SI-FMA-LABEL: test_f32_mul_y_sub_one_x:
+; SI-FMA:       ; %bb.0:
+; SI-FMA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-FMA-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-FMA-NEXT:    s_mov_b32 s3, 0xf000
+; SI-FMA-NEXT:    s_mov_b32 s2, -1
+; SI-FMA-NEXT:    s_mov_b32 s14, s2
+; SI-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-FMA-NEXT:    s_mov_b32 s12, s6
+; SI-FMA-NEXT:    s_mov_b32 s13, s7
+; SI-FMA-NEXT:    s_mov_b32 s15, s3
+; SI-FMA-NEXT:    s_mov_b32 s10, s2
+; SI-FMA-NEXT:    s_mov_b32 s11, s3
+; SI-FMA-NEXT:    buffer_load_dword v0, off, s[12:15], 0
+; SI-FMA-NEXT:    buffer_load_dword v1, off, s[8:11], 0
+; SI-FMA-NEXT:    s_mov_b32 s0, s4
+; SI-FMA-NEXT:    s_mov_b32 s1, s5
+; SI-FMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-FMA-NEXT:    v_fma_f32 v0, -v0, v1, v1
+; SI-FMA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-FMA-NEXT:    s_endpgm
+;
+; GFX11-NOFMA-LABEL: test_f32_mul_y_sub_one_x:
+; GFX11-NOFMA:       ; %bb.0:
+; GFX11-NOFMA-NEXT:    s_clause 0x1
+; GFX11-NOFMA-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GFX11-NOFMA-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GFX11-NOFMA-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NOFMA-NEXT:    s_clause 0x1
+; GFX11-NOFMA-NEXT:    global_load_b32 v1, v0, s[6:7]
+; GFX11-NOFMA-NEXT:    global_load_b32 v2, v0, s[0:1]
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-NOFMA-NEXT:    v_sub_f32_e32 v1, 1.0, v1
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NOFMA-NEXT:    v_mul_f32_e32 v1, v2, v1
+; GFX11-NOFMA-NEXT:    global_store_b32 v0, v1, s[4:5]
+; GFX11-NOFMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NOFMA-NEXT:    s_endpgm
+;
+; GFX11-FMA-LABEL: test_f32_mul_y_sub_one_x:
+; GFX11-FMA:       ; %bb.0:
+; GFX11-FMA-NEXT:    s_clause 0x1
+; GFX11-FMA-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GFX11-FMA-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GFX11-FMA-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FMA-NEXT:    s_clause 0x1
+; GFX11-FMA-NEXT:    global_load_b32 v1, v0, s[6:7]
+; GFX11-FMA-NEXT:    global_load_b32 v2, v0, s[0:1]
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FMA-NEXT:    v_fma_f32 v1, -v1, v2, v2
+; GFX11-FMA-NEXT:    global_store_b32 v0, v1, s[4:5]
+; GFX11-FMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-FMA-NEXT:    s_endpgm
                                         ptr addrspace(1) %in1,
                                         ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
@@ -487,12 +1428,87 @@ define amdgpu_kernel void @test_f32_mul_y_sub_one_x(ptr addrspace(1) %out,
   ret void
 }
 
-; FUNC-LABEL: {{^}}test_f32_mul_sub_negone_x_y:
-; SI-NOFMA: v_sub_f32_e32 [[VS:v[0-9]]], -1.0, [[VX:v[0-9]]]
-; SI-NOFMA: v_mul_f32_e32 {{v[0-9]}}, [[VS]], [[VY:v[0-9]]]
-;
-; SI-FMA: v_fma_f32 {{v[0-9]}}, -[[VX:v[0-9]]], [[VY:v[0-9]]], -[[VY:v[0-9]]]
 define amdgpu_kernel void @test_f32_mul_sub_negone_x_y(ptr addrspace(1) %out,
+; SI-NOFMA-LABEL: test_f32_mul_sub_negone_x_y:
+; SI-NOFMA:       ; %bb.0:
+; SI-NOFMA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NOFMA-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-NOFMA-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NOFMA-NEXT:    s_mov_b32 s2, -1
+; SI-NOFMA-NEXT:    s_mov_b32 s14, s2
+; SI-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NOFMA-NEXT:    s_mov_b32 s12, s6
+; SI-NOFMA-NEXT:    s_mov_b32 s13, s7
+; SI-NOFMA-NEXT:    s_mov_b32 s15, s3
+; SI-NOFMA-NEXT:    s_mov_b32 s10, s2
+; SI-NOFMA-NEXT:    s_mov_b32 s11, s3
+; SI-NOFMA-NEXT:    buffer_load_dword v0, off, s[12:15], 0
+; SI-NOFMA-NEXT:    buffer_load_dword v1, off, s[8:11], 0
+; SI-NOFMA-NEXT:    s_mov_b32 s0, s4
+; SI-NOFMA-NEXT:    s_mov_b32 s1, s5
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(1)
+; SI-NOFMA-NEXT:    v_sub_f32_e32 v0, -1.0, v0
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-NOFMA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NOFMA-NEXT:    s_endpgm
+;
+; SI-FMA-LABEL: test_f32_mul_sub_negone_x_y:
+; SI-FMA:       ; %bb.0:
+; SI-FMA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-FMA-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-FMA-NEXT:    s_mov_b32 s3, 0xf000
+; SI-FMA-NEXT:    s_mov_b32 s2, -1
+; SI-FMA-NEXT:    s_mov_b32 s14, s2
+; SI-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-FMA-NEXT:    s_mov_b32 s12, s6
+; SI-FMA-NEXT:    s_mov_b32 s13, s7
+; SI-FMA-NEXT:    s_mov_b32 s15, s3
+; SI-FMA-NEXT:    s_mov_b32 s10, s2
+; SI-FMA-NEXT:    s_mov_b32 s11, s3
+; SI-FMA-NEXT:    buffer_load_dword v0, off, s[12:15], 0
+; SI-FMA-NEXT:    buffer_load_dword v1, off, s[8:11], 0
+; SI-FMA-NEXT:    s_mov_b32 s0, s4
+; SI-FMA-NEXT:    s_mov_b32 s1, s5
+; SI-FMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-FMA-NEXT:    v_fma_f32 v0, -v0, v1, -v1
+; SI-FMA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-FMA-NEXT:    s_endpgm
+;
+; GFX11-NOFMA-LABEL: test_f32_mul_sub_negone_x_y:
+; GFX11-NOFMA:       ; %bb.0:
+; GFX11-NOFMA-NEXT:    s_clause 0x1
+; GFX11-NOFMA-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GFX11-NOFMA-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GFX11-NOFMA-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NOFMA-NEXT:    s_clause 0x1
+; GFX11-NOFMA-NEXT:    global_load_b32 v1, v0, s[6:7]
+; GFX11-NOFMA-NEXT:    global_load_b32 v2, v0, s[0:1]
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-NOFMA-NEXT:    v_sub_f32_e32 v1, -1.0, v1
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NOFMA-NEXT:    v_mul_f32_e32 v1, v1, v2
+; GFX11-NOFMA-NEXT:    global_store_b32 v0, v1, s[4:5]
+; GFX11-NOFMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NOFMA-NEXT:    s_endpgm
+;
+; GFX11-FMA-LABEL: test_f32_mul_sub_negone_x_y:
+; GFX11-FMA:       ; %bb.0:
+; GFX11-FMA-NEXT:    s_clause 0x1
+; GFX11-FMA-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GFX11-FMA-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GFX11-FMA-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FMA-NEXT:    s_clause 0x1
+; GFX11-FMA-NEXT:    global_load_b32 v1, v0, s[6:7]
+; GFX11-FMA-NEXT:    global_load_b32 v2, v0, s[0:1]
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FMA-NEXT:    v_fma_f32 v1, -v1, v2, -v2
+; GFX11-FMA-NEXT:    global_store_b32 v0, v1, s[4:5]
+; GFX11-FMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-FMA-NEXT:    s_endpgm
                                            ptr addrspace(1) %in1,
                                            ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
@@ -503,12 +1519,87 @@ define amdgpu_kernel void @test_f32_mul_sub_negone_x_y(ptr addrspace(1) %out,
   ret void
 }
 
-; FUNC-LABEL: {{^}}test_f32_mul_y_sub_negone_x:
-; SI-NOFMA: v_sub_f32_e32 [[VS:v[0-9]]], -1.0, [[VX:v[0-9]]]
-; SI-NOFMA: v_mul_f32_e32 {{v[0-9]}}, [[VY:v[0-9]]], [[VS]]
-;
-; SI-FMA: v_fma_f32 {{v[0-9]}}, -[[VX:v[0-9]]], [[VY:v[0-9]]], -[[VY:v[0-9]]]
 define amdgpu_kernel void @test_f32_mul_y_sub_negone_x(ptr addrspace(1) %out,
+; SI-NOFMA-LABEL: test_f32_mul_y_sub_negone_x:
+; SI-NOFMA:       ; %bb.0:
+; SI-NOFMA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NOFMA-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-NOFMA-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NOFMA-NEXT:    s_mov_b32 s2, -1
+; SI-NOFMA-NEXT:    s_mov_b32 s14, s2
+; SI-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NOFMA-NEXT:    s_mov_b32 s12, s6
+; SI-NOFMA-NEXT:    s_mov_b32 s13, s7
+; SI-NOFMA-NEXT:    s_mov_b32 s15, s3
+; SI-NOFMA-NEXT:    s_mov_b32 s10, s2
+; SI-NOFMA-NEXT:    s_mov_b32 s11, s3
+; SI-NOFMA-NEXT:    buffer_load_dword v0, off, s[12:15], 0
+; SI-NOFMA-NEXT:    buffer_load_dword v1, off, s[8:11], 0
+; SI-NOFMA-NEXT:    s_mov_b32 s0, s4
+; SI-NOFMA-NEXT:    s_mov_b32 s1, s5
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(1)
+; SI-NOFMA-NEXT:    v_sub_f32_e32 v0, -1.0, v0
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    v_mul_f32_e32 v0, v1, v0
+; SI-NOFMA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NOFMA-NEXT:    s_endpgm
+;
+; SI-FMA-LABEL: test_f32_mul_y_sub_negone_x:
+; SI-FMA:       ; %bb.0:
+; SI-FMA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-FMA-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-FMA-NEXT:    s_mov_b32 s3, 0xf000
+; SI-FMA-NEXT:    s_mov_b32 s2, -1
+; SI-FMA-NEXT:    s_mov_b32 s14, s2
+; SI-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-FMA-NEXT:    s_mov_b32 s12, s6
+; SI-FMA-NEXT:    s_mov_b32 s13, s7
+; SI-FMA-NEXT:    s_mov_b32 s15, s3
+; SI-FMA-NEXT:    s_mov_b32 s10, s2
+; SI-FMA-NEXT:    s_mov_b32 s11, s3
+; SI-FMA-NEXT:    buffer_load_dword v0, off, s[12:15], 0
+; SI-FMA-NEXT:    buffer_load_dword v1, off, s[8:11], 0
+; SI-FMA-NEXT:    s_mov_b32 s0, s4
+; SI-FMA-NEXT:    s_mov_b32 s1, s5
+; SI-FMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-FMA-NEXT:    v_fma_f32 v0, -v0, v1, -v1
+; SI-FMA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-FMA-NEXT:    s_endpgm
+;
+; GFX11-NOFMA-LABEL: test_f32_mul_y_sub_negone_x:
+; GFX11-NOFMA:       ; %bb.0:
+; GFX11-NOFMA-NEXT:    s_clause 0x1
+; GFX11-NOFMA-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GFX11-NOFMA-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GFX11-NOFMA-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NOFMA-NEXT:    s_clause 0x1
+; GFX11-NOFMA-NEXT:    global_load_b32 v1, v0, s[6:7]
+; GFX11-NOFMA-NEXT:    global_load_b32 v2, v0, s[0:1]
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-NOFMA-NEXT:    v_sub_f32_e32 v1, -1.0, v1
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NOFMA-NEXT:    v_mul_f32_e32 v1, v2, v1
+; GFX11-NOFMA-NEXT:    global_store_b32 v0, v1, s[4:5]
+; GFX11-NOFMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NOFMA-NEXT:    s_endpgm
+;
+; GFX11-FMA-LABEL: test_f32_mul_y_sub_negone_x:
+; GFX11-FMA:       ; %bb.0:
+; GFX11-FMA-NEXT:    s_clause 0x1
+; GFX11-FMA-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GFX11-FMA-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GFX11-FMA-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FMA-NEXT:    s_clause 0x1
+; GFX11-FMA-NEXT:    global_load_b32 v1, v0, s[6:7]
+; GFX11-FMA-NEXT:    global_load_b32 v2, v0, s[0:1]
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FMA-NEXT:    v_fma_f32 v1, -v1, v2, -v2
+; GFX11-FMA-NEXT:    global_store_b32 v0, v1, s[4:5]
+; GFX11-FMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-FMA-NEXT:    s_endpgm
                                          ptr addrspace(1) %in1,
                                          ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
@@ -519,12 +1610,87 @@ define amdgpu_kernel void @test_f32_mul_y_sub_negone_x(ptr addrspace(1) %out,
   ret void
 }
 
-; FUNC-LABEL: {{^}}test_f32_mul_sub_x_one_y:
-; SI-NOFMA: v_add_f32_e32 [[VS:v[0-9]]], -1.0, [[VX:v[0-9]]]
-; SI-NOFMA: v_mul_f32_e32 {{v[0-9]}}, [[VS]], [[VY:v[0-9]]]
-;
-; SI-FMA: v_fma_f32 {{v[0-9]}}, [[VX:v[0-9]]], [[VY:v[0-9]]], -[[VY:v[0-9]]]
 define amdgpu_kernel void @test_f32_mul_sub_x_one_y(ptr addrspace(1) %out,
+; SI-NOFMA-LABEL: test_f32_mul_sub_x_one_y:
+; SI-NOFMA:       ; %bb.0:
+; SI-NOFMA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NOFMA-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-NOFMA-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NOFMA-NEXT:    s_mov_b32 s2, -1
+; SI-NOFMA-NEXT:    s_mov_b32 s14, s2
+; SI-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NOFMA-NEXT:    s_mov_b32 s12, s6
+; SI-NOFMA-NEXT:    s_mov_b32 s13, s7
+; SI-NOFMA-NEXT:    s_mov_b32 s15, s3
+; SI-NOFMA-NEXT:    s_mov_b32 s10, s2
+; SI-NOFMA-NEXT:    s_mov_b32 s11, s3
+; SI-NOFMA-NEXT:    buffer_load_dword v0, off, s[12:15], 0
+; SI-NOFMA-NEXT:    buffer_load_dword v1, off, s[8:11], 0
+; SI-NOFMA-NEXT:    s_mov_b32 s0, s4
+; SI-NOFMA-NEXT:    s_mov_b32 s1, s5
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(1)
+; SI-NOFMA-NEXT:    v_add_f32_e32 v0, -1.0, v0
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-NOFMA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NOFMA-NEXT:    s_endpgm
+;
+; SI-FMA-LABEL: test_f32_mul_sub_x_one_y:
+; SI-FMA:       ; %bb.0:
+; SI-FMA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-FMA-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-FMA-NEXT:    s_mov_b32 s3, 0xf000
+; SI-FMA-NEXT:    s_mov_b32 s2, -1
+; SI-FMA-NEXT:    s_mov_b32 s14, s2
+; SI-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-FMA-NEXT:    s_mov_b32 s12, s6
+; SI-FMA-NEXT:    s_mov_b32 s13, s7
+; SI-FMA-NEXT:    s_mov_b32 s15, s3
+; SI-FMA-NEXT:    s_mov_b32 s10, s2
+; SI-FMA-NEXT:    s_mov_b32 s11, s3
+; SI-FMA-NEXT:    buffer_load_dword v0, off, s[12:15], 0
+; SI-FMA-NEXT:    buffer_load_dword v1, off, s[8:11], 0
+; SI-FMA-NEXT:    s_mov_b32 s0, s4
+; SI-FMA-NEXT:    s_mov_b32 s1, s5
+; SI-FMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-FMA-NEXT:    v_fma_f32 v0, v0, v1, -v1
+; SI-FMA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-FMA-NEXT:    s_endpgm
+;
+; GFX11-NOFMA-LABEL: test_f32_mul_sub_x_one_y:
+; GFX11-NOFMA:       ; %bb.0:
+; GFX11-NOFMA-NEXT:    s_clause 0x1
+; GFX11-NOFMA-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GFX11-NOFMA-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GFX11-NOFMA-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NOFMA-NEXT:    s_clause 0x1
+; GFX11-NOFMA-NEXT:    global_load_b32 v1, v0, s[6:7]
+; GFX11-NOFMA-NEXT:    global_load_b32 v2, v0, s[0:1]
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-NOFMA-NEXT:    v_add_f32_e32 v1, -1.0, v1
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NOFMA-NEXT:    v_mul_f32_e32 v1, v1, v2
+; GFX11-NOFMA-NEXT:    global_store_b32 v0, v1, s[4:5]
+; GFX11-NOFMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NOFMA-NEXT:    s_endpgm
+;
+; GFX11-FMA-LABEL: test_f32_mul_sub_x_one_y:
+; GFX11-FMA:       ; %bb.0:
+; GFX11-FMA-NEXT:    s_clause 0x1
+; GFX11-FMA-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GFX11-FMA-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GFX11-FMA-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FMA-NEXT:    s_clause 0x1
+; GFX11-FMA-NEXT:    global_load_b32 v1, v0, s[6:7]
+; GFX11-FMA-NEXT:    global_load_b32 v2, v0, s[0:1]
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FMA-NEXT:    v_fma_f32 v1, v1, v2, -v2
+; GFX11-FMA-NEXT:    global_store_b32 v0, v1, s[4:5]
+; GFX11-FMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-FMA-NEXT:    s_endpgm
                                         ptr addrspace(1) %in1,
                                         ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
@@ -535,12 +1701,87 @@ define amdgpu_kernel void @test_f32_mul_sub_x_one_y(ptr addrspace(1) %out,
   ret void
 }
 
-; FUNC-LABEL: {{^}}test_f32_mul_y_sub_x_one:
-; SI-NOFMA: v_add_f32_e32 [[VS:v[0-9]]], -1.0, [[VX:v[0-9]]]
-; SI-NOFMA: v_mul_f32_e32 {{v[0-9]}}, [[VY:v[0-9]]], [[VS]]
-;
-; SI-FMA: v_fma_f32 {{v[0-9]}}, [[VX:v[0-9]]], [[VY:v[0-9]]], -[[VY:v[0-9]]]
 define amdgpu_kernel void @test_f32_mul_y_sub_x_one(ptr addrspace(1) %out,
+; SI-NOFMA-LABEL: test_f32_mul_y_sub_x_one:
+; SI-NOFMA:       ; %bb.0:
+; SI-NOFMA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NOFMA-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-NOFMA-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NOFMA-NEXT:    s_mov_b32 s2, -1
+; SI-NOFMA-NEXT:    s_mov_b32 s14, s2
+; SI-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NOFMA-NEXT:    s_mov_b32 s12, s6
+; SI-NOFMA-NEXT:    s_mov_b32 s13, s7
+; SI-NOFMA-NEXT:    s_mov_b32 s15, s3
+; SI-NOFMA-NEXT:    s_mov_b32 s10, s2
+; SI-NOFMA-NEXT:    s_mov_b32 s11, s3
+; SI-NOFMA-NEXT:    buffer_load_dword v0, off, s[12:15], 0
+; SI-NOFMA-NEXT:    buffer_load_dword v1, off, s[8:11], 0
+; SI-NOFMA-NEXT:    s_mov_b32 s0, s4
+; SI-NOFMA-NEXT:    s_mov_b32 s1, s5
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(1)
+; SI-NOFMA-NEXT:    v_add_f32_e32 v0, -1.0, v0
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    v_mul_f32_e32 v0, v1, v0
+; SI-NOFMA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NOFMA-NEXT:    s_endpgm
+;
+; SI-FMA-LABEL: test_f32_mul_y_sub_x_one:
+; SI-FMA:       ; %bb.0:
+; SI-FMA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-FMA-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-FMA-NEXT:    s_mov_b32 s3, 0xf000
+; SI-FMA-NEXT:    s_mov_b32 s2, -1
+; SI-FMA-NEXT:    s_mov_b32 s14, s2
+; SI-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-FMA-NEXT:    s_mov_b32 s12, s6
+; SI-FMA-NEXT:    s_mov_b32 s13, s7
+; SI-FMA-NEXT:    s_mov_b32 s15, s3
+; SI-FMA-NEXT:    s_mov_b32 s10, s2
+; SI-FMA-NEXT:    s_mov_b32 s11, s3
+; SI-FMA-NEXT:    buffer_load_dword v0, off, s[12:15], 0
+; SI-FMA-NEXT:    buffer_load_dword v1, off, s[8:11], 0
+; SI-FMA-NEXT:    s_mov_b32 s0, s4
+; SI-FMA-NEXT:    s_mov_b32 s1, s5
+; SI-FMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-FMA-NEXT:    v_fma_f32 v0, v0, v1, -v1
+; SI-FMA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-FMA-NEXT:    s_endpgm
+;
+; GFX11-NOFMA-LABEL: test_f32_mul_y_sub_x_one:
+; GFX11-NOFMA:       ; %bb.0:
+; GFX11-NOFMA-NEXT:    s_clause 0x1
+; GFX11-NOFMA-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GFX11-NOFMA-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GFX11-NOFMA-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NOFMA-NEXT:    s_clause 0x1
+; GFX11-NOFMA-NEXT:    global_load_b32 v1, v0, s[6:7]
+; GFX11-NOFMA-NEXT:    global_load_b32 v2, v0, s[0:1]
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-NOFMA-NEXT:    v_add_f32_e32 v1, -1.0, v1
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NOFMA-NEXT:    v_mul_f32_e32 v1, v2, v1
+; GFX11-NOFMA-NEXT:    global_store_b32 v0, v1, s[4:5]
+; GFX11-NOFMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NOFMA-NEXT:    s_endpgm
+;
+; GFX11-FMA-LABEL: test_f32_mul_y_sub_x_one:
+; GFX11-FMA:       ; %bb.0:
+; GFX11-FMA-NEXT:    s_clause 0x1
+; GFX11-FMA-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GFX11-FMA-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GFX11-FMA-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FMA-NEXT:    s_clause 0x1
+; GFX11-FMA-NEXT:    global_load_b32 v1, v0, s[6:7]
+; GFX11-FMA-NEXT:    global_load_b32 v2, v0, s[0:1]
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FMA-NEXT:    v_fma_f32 v1, v1, v2, -v2
+; GFX11-FMA-NEXT:    global_store_b32 v0, v1, s[4:5]
+; GFX11-FMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-FMA-NEXT:    s_endpgm
                                       ptr addrspace(1) %in1,
                                       ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
@@ -551,12 +1792,87 @@ define amdgpu_kernel void @test_f32_mul_y_sub_x_one(ptr addrspace(1) %out,
   ret void
 }
 
-; FUNC-LABEL: {{^}}test_f32_mul_sub_x_negone_y:
-; SI-NOFMA: v_add_f32_e32 [[VS:v[0-9]]], 1.0, [[VX:v[0-9]]]
-; SI-NOFMA: v_mul_f32_e32 {{v[0-9]}}, [[VS]], [[VY:v[0-9]]]
-;
-; SI-FMA: v_fma_f32 {{v[0-9]}}, [[VX:v[0-9]]], [[VY:v[0-9]]], [[VY:v[0-9]]]
 define amdgpu_kernel void @test_f32_mul_sub_x_negone_y(ptr addrspace(1) %out,
+; SI-NOFMA-LABEL: test_f32_mul_sub_x_negone_y:
+; SI-NOFMA:       ; %bb.0:
+; SI-NOFMA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NOFMA-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-NOFMA-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NOFMA-NEXT:    s_mov_b32 s2, -1
+; SI-NOFMA-NEXT:    s_mov_b32 s14, s2
+; SI-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NOFMA-NEXT:    s_mov_b32 s12, s6
+; SI-NOFMA-NEXT:    s_mov_b32 s13, s7
+; SI-NOFMA-NEXT:    s_mov_b32 s15, s3
+; SI-NOFMA-NEXT:    s_mov_b32 s10, s2
+; SI-NOFMA-NEXT:    s_mov_b32 s11, s3
+; SI-NOFMA-NEXT:    buffer_load_dword v0, off, s[12:15], 0
+; SI-NOFMA-NEXT:    buffer_load_dword v1, off, s[8:11], 0
+; SI-NOFMA-NEXT:    s_mov_b32 s0, s4
+; SI-NOFMA-NEXT:    s_mov_b32 s1, s5
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(1)
+; SI-NOFMA-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    v_mul_f32_e32 v0, v0, v1
+; SI-NOFMA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NOFMA-NEXT:    s_endpgm
+;
+; SI-FMA-LABEL: test_f32_mul_sub_x_negone_y:
+; SI-FMA:       ; %bb.0:
+; SI-FMA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-FMA-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-FMA-NEXT:    s_mov_b32 s3, 0xf000
+; SI-FMA-NEXT:    s_mov_b32 s2, -1
+; SI-FMA-NEXT:    s_mov_b32 s14, s2
+; SI-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-FMA-NEXT:    s_mov_b32 s12, s6
+; SI-FMA-NEXT:    s_mov_b32 s13, s7
+; SI-FMA-NEXT:    s_mov_b32 s15, s3
+; SI-FMA-NEXT:    s_mov_b32 s10, s2
+; SI-FMA-NEXT:    s_mov_b32 s11, s3
+; SI-FMA-NEXT:    buffer_load_dword v0, off, s[12:15], 0
+; SI-FMA-NEXT:    buffer_load_dword v1, off, s[8:11], 0
+; SI-FMA-NEXT:    s_mov_b32 s0, s4
+; SI-FMA-NEXT:    s_mov_b32 s1, s5
+; SI-FMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-FMA-NEXT:    v_fma_f32 v0, v0, v1, v1
+; SI-FMA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-FMA-NEXT:    s_endpgm
+;
+; GFX11-NOFMA-LABEL: test_f32_mul_sub_x_negone_y:
+; GFX11-NOFMA:       ; %bb.0:
+; GFX11-NOFMA-NEXT:    s_clause 0x1
+; GFX11-NOFMA-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GFX11-NOFMA-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GFX11-NOFMA-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NOFMA-NEXT:    s_clause 0x1
+; GFX11-NOFMA-NEXT:    global_load_b32 v1, v0, s[6:7]
+; GFX11-NOFMA-NEXT:    global_load_b32 v2, v0, s[0:1]
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-NOFMA-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NOFMA-NEXT:    v_mul_f32_e32 v1, v1, v2
+; GFX11-NOFMA-NEXT:    global_store_b32 v0, v1, s[4:5]
+; GFX11-NOFMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NOFMA-NEXT:    s_endpgm
+;
+; GFX11-FMA-LABEL: test_f32_mul_sub_x_negone_y:
+; GFX11-FMA:       ; %bb.0:
+; GFX11-FMA-NEXT:    s_clause 0x1
+; GFX11-FMA-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GFX11-FMA-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GFX11-FMA-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FMA-NEXT:    s_clause 0x1
+; GFX11-FMA-NEXT:    global_load_b32 v1, v0, s[6:7]
+; GFX11-FMA-NEXT:    global_load_b32 v2, v0, s[0:1]
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FMA-NEXT:    v_fmac_f32_e32 v2, v1, v2
+; GFX11-FMA-NEXT:    global_store_b32 v0, v2, s[4:5]
+; GFX11-FMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-FMA-NEXT:    s_endpgm
                                          ptr addrspace(1) %in1,
                                          ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
@@ -567,12 +1883,87 @@ define amdgpu_kernel void @test_f32_mul_sub_x_negone_y(ptr addrspace(1) %out,
   ret void
 }
 
-; FUNC-LABEL: {{^}}test_f32_mul_y_sub_x_negone:
-; SI-NOFMA: v_add_f32_e32 [[VS:v[0-9]]], 1.0, [[VX:v[0-9]]]
-; SI-NOFMA: v_mul_f32_e32 {{v[0-9]}}, [[VY:v[0-9]]], [[VS]]
-;
-; SI-FMA: v_fma_f32 {{v[0-9]}}, [[VX:v[0-9]]], [[VY:v[0-9]]], [[VY:v[0-9]]]
 define amdgpu_kernel void @test_f32_mul_y_sub_x_negone(ptr addrspace(1) %out,
+; SI-NOFMA-LABEL: test_f32_mul_y_sub_x_negone:
+; SI-NOFMA:       ; %bb.0:
+; SI-NOFMA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-NOFMA-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-NOFMA-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NOFMA-NEXT:    s_mov_b32 s2, -1
+; SI-NOFMA-NEXT:    s_mov_b32 s14, s2
+; SI-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NOFMA-NEXT:    s_mov_b32 s12, s6
+; SI-NOFMA-NEXT:    s_mov_b32 s13, s7
+; SI-NOFMA-NEXT:    s_mov_b32 s15, s3
+; SI-NOFMA-NEXT:    s_mov_b32 s10, s2
+; SI-NOFMA-NEXT:    s_mov_b32 s11, s3
+; SI-NOFMA-NEXT:    buffer_load_dword v0, off, s[12:15], 0
+; SI-NOFMA-NEXT:    buffer_load_dword v1, off, s[8:11], 0
+; SI-NOFMA-NEXT:    s_mov_b32 s0, s4
+; SI-NOFMA-NEXT:    s_mov_b32 s1, s5
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(1)
+; SI-NOFMA-NEXT:    v_add_f32_e32 v0, 1.0, v0
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    v_mul_f32_e32 v0, v1, v0
+; SI-NOFMA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-NOFMA-NEXT:    s_endpgm
+;
+; SI-FMA-LABEL: test_f32_mul_y_sub_x_negone:
+; SI-FMA:       ; %bb.0:
+; SI-FMA-NEXT:    s_load_dwordx4 s[4:7], s[0:1], 0x9
+; SI-FMA-NEXT:    s_load_dwordx2 s[8:9], s[0:1], 0xd
+; SI-FMA-NEXT:    s_mov_b32 s3, 0xf000
+; SI-FMA-NEXT:    s_mov_b32 s2, -1
+; SI-FMA-NEXT:    s_mov_b32 s14, s2
+; SI-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-FMA-NEXT:    s_mov_b32 s12, s6
+; SI-FMA-NEXT:    s_mov_b32 s13, s7
+; SI-FMA-NEXT:    s_mov_b32 s15, s3
+; SI-FMA-NEXT:    s_mov_b32 s10, s2
+; SI-FMA-NEXT:    s_mov_b32 s11, s3
+; SI-FMA-NEXT:    buffer_load_dword v0, off, s[12:15], 0
+; SI-FMA-NEXT:    buffer_load_dword v1, off, s[8:11], 0
+; SI-FMA-NEXT:    s_mov_b32 s0, s4
+; SI-FMA-NEXT:    s_mov_b32 s1, s5
+; SI-FMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-FMA-NEXT:    v_fma_f32 v0, v0, v1, v1
+; SI-FMA-NEXT:    buffer_store_dword v0, off, s[0:3], 0
+; SI-FMA-NEXT:    s_endpgm
+;
+; GFX11-NOFMA-LABEL: test_f32_mul_y_sub_x_negone:
+; GFX11-NOFMA:       ; %bb.0:
+; GFX11-NOFMA-NEXT:    s_clause 0x1
+; GFX11-NOFMA-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GFX11-NOFMA-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GFX11-NOFMA-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NOFMA-NEXT:    s_clause 0x1
+; GFX11-NOFMA-NEXT:    global_load_b32 v1, v0, s[6:7]
+; GFX11-NOFMA-NEXT:    global_load_b32 v2, v0, s[0:1]
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-NOFMA-NEXT:    v_add_f32_e32 v1, 1.0, v1
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-NOFMA-NEXT:    v_mul_f32_e32 v1, v2, v1
+; GFX11-NOFMA-NEXT:    global_store_b32 v0, v1, s[4:5]
+; GFX11-NOFMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NOFMA-NEXT:    s_endpgm
+;
+; GFX11-FMA-LABEL: test_f32_mul_y_sub_x_negone:
+; GFX11-FMA:       ; %bb.0:
+; GFX11-FMA-NEXT:    s_clause 0x1
+; GFX11-FMA-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GFX11-FMA-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GFX11-FMA-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FMA-NEXT:    s_clause 0x1
+; GFX11-FMA-NEXT:    global_load_b32 v1, v0, s[6:7]
+; GFX11-FMA-NEXT:    global_load_b32 v2, v0, s[0:1]
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FMA-NEXT:    v_fmac_f32_e32 v2, v1, v2
+; GFX11-FMA-NEXT:    global_store_b32 v0, v2, s[4:5]
+; GFX11-FMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-FMA-NEXT:    s_endpgm
                                          ptr addrspace(1) %in1,
                                          ptr addrspace(1) %in2) {
   %x = load float, ptr addrspace(1) %in1
@@ -587,14 +1978,106 @@ define amdgpu_kernel void @test_f32_mul_y_sub_x_negone(ptr addrspace(1) %out,
 ; Interpolation Patterns: add(mul(x,t),mul(sub(1.0,t),y))
 ;
 
-; FUNC-LABEL: {{^}}test_f32_interp:
-; SI-NOFMA: v_sub_f32_e32 [[VT1:v[0-9]]], 1.0, [[VT:v[0-9]]]
-; SI-NOFMA: v_mul_f32_e32 [[VTY:v[0-9]]], [[VY:v[0-9]]], [[VT1]]
-; SI-NOFMA: v_mac_f32_e32 [[VTY]], [[VX:v[0-9]]], [[VT]]
-;
-; SI-FMA: v_fma_f32 [[VR:v[0-9]]], -[[VT:v[0-9]]], [[VY:v[0-9]]], [[VY]]
-; SI-FMA: v_fma_f32 {{v[0-9]}}, [[VX:v[0-9]]], [[VT]], [[VR]]
 define amdgpu_kernel void @test_f32_interp(ptr addrspace(1) %out,
+; SI-NOFMA-LABEL: test_f32_interp:
+; SI-NOFMA:       ; %bb.0:
+; SI-NOFMA-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x9
+; SI-NOFMA-NEXT:    s_mov_b32 s11, 0xf000
+; SI-NOFMA-NEXT:    s_mov_b32 s10, -1
+; SI-NOFMA-NEXT:    s_mov_b32 s14, s10
+; SI-NOFMA-NEXT:    s_mov_b32 s15, s11
+; SI-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NOFMA-NEXT:    s_mov_b32 s16, s4
+; SI-NOFMA-NEXT:    s_mov_b32 s17, s5
+; SI-NOFMA-NEXT:    s_mov_b32 s4, s6
+; SI-NOFMA-NEXT:    s_mov_b32 s5, s7
+; SI-NOFMA-NEXT:    s_mov_b32 s6, s10
+; SI-NOFMA-NEXT:    s_mov_b32 s7, s11
+; SI-NOFMA-NEXT:    s_mov_b32 s12, s2
+; SI-NOFMA-NEXT:    s_mov_b32 s13, s3
+; SI-NOFMA-NEXT:    s_mov_b32 s18, s10
+; SI-NOFMA-NEXT:    s_mov_b32 s19, s11
+; SI-NOFMA-NEXT:    buffer_load_dword v0, off, s[4:7], 0
+; SI-NOFMA-NEXT:    buffer_load_dword v1, off, s[16:19], 0
+; SI-NOFMA-NEXT:    buffer_load_dword v2, off, s[12:15], 0
+; SI-NOFMA-NEXT:    s_mov_b32 s8, s0
+; SI-NOFMA-NEXT:    s_mov_b32 s9, s1
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(2)
+; SI-NOFMA-NEXT:    v_sub_f32_e32 v3, 1.0, v0
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(1)
+; SI-NOFMA-NEXT:    v_mul_f32_e32 v1, v1, v3
+; SI-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-NOFMA-NEXT:    v_mac_f32_e32 v1, v2, v0
+; SI-NOFMA-NEXT:    buffer_store_dword v1, off, s[8:11], 0
+; SI-NOFMA-NEXT:    s_endpgm
+;
+; SI-FMA-LABEL: test_f32_interp:
+; SI-FMA:       ; %bb.0:
+; SI-FMA-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x9
+; SI-FMA-NEXT:    s_mov_b32 s11, 0xf000
+; SI-FMA-NEXT:    s_mov_b32 s10, -1
+; SI-FMA-NEXT:    s_mov_b32 s18, s10
+; SI-FMA-NEXT:    s_mov_b32 s19, s11
+; SI-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-FMA-NEXT:    s_mov_b32 s16, s4
+; SI-FMA-NEXT:    s_mov_b32 s17, s5
+; SI-FMA-NEXT:    s_mov_b32 s14, s10
+; SI-FMA-NEXT:    s_mov_b32 s12, s2
+; SI-FMA-NEXT:    s_mov_b32 s13, s3
+; SI-FMA-NEXT:    s_mov_b32 s15, s11
+; SI-FMA-NEXT:    s_mov_b32 s4, s6
+; SI-FMA-NEXT:    s_mov_b32 s5, s7
+; SI-FMA-NEXT:    s_mov_b32 s6, s10
+; SI-FMA-NEXT:    s_mov_b32 s7, s11
+; SI-FMA-NEXT:    buffer_load_dword v0, off, s[16:19], 0
+; SI-FMA-NEXT:    buffer_load_dword v1, off, s[4:7], 0
+; SI-FMA-NEXT:    buffer_load_dword v2, off, s[12:15], 0
+; SI-FMA-NEXT:    s_mov_b32 s8, s0
+; SI-FMA-NEXT:    s_mov_b32 s9, s1
+; SI-FMA-NEXT:    s_waitcnt vmcnt(1)
+; SI-FMA-NEXT:    v_fma_f32 v0, -v1, v0, v0
+; SI-FMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-FMA-NEXT:    v_fma_f32 v0, v2, v1, v0
+; SI-FMA-NEXT:    buffer_store_dword v0, off, s[8:11], 0
+; SI-FMA-NEXT:    s_endpgm
+;
+; GFX11-NOFMA-LABEL: test_f32_interp:
+; GFX11-NOFMA:       ; %bb.0:
+; GFX11-NOFMA-NEXT:    s_load_b256 s[0:7], s[0:1], 0x24
+; GFX11-NOFMA-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NOFMA-NEXT:    s_clause 0x2
+; GFX11-NOFMA-NEXT:    global_load_b32 v1, v0, s[6:7]
+; GFX11-NOFMA-NEXT:    global_load_b32 v2, v0, s[4:5]
+; GFX11-NOFMA-NEXT:    global_load_b32 v3, v0, s[2:3]
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(2)
+; GFX11-NOFMA-NEXT:    v_sub_f32_e32 v4, 1.0, v1
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NOFMA-NEXT:    v_mul_f32_e32 v2, v2, v4
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    v_fmac_f32_e32 v2, v3, v1
+; GFX11-NOFMA-NEXT:    global_store_b32 v0, v2, s[0:1]
+; GFX11-NOFMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NOFMA-NEXT:    s_endpgm
+;
+; GFX11-FMA-LABEL: test_f32_interp:
+; GFX11-FMA:       ; %bb.0:
+; GFX11-FMA-NEXT:    s_load_b256 s[0:7], s[0:1], 0x24
+; GFX11-FMA-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FMA-NEXT:    s_clause 0x2
+; GFX11-FMA-NEXT:    global_load_b32 v1, v0, s[4:5]
+; GFX11-FMA-NEXT:    global_load_b32 v2, v0, s[6:7]
+; GFX11-FMA-NEXT:    global_load_b32 v3, v0, s[2:3]
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-FMA-NEXT:    v_fma_f32 v1, -v2, v1, v1
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FMA-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FMA-NEXT:    v_fmac_f32_e32 v1, v3, v2
+; GFX11-FMA-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX11-FMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-FMA-NEXT:    s_endpgm
                              ptr addrspace(1) %in1,
                              ptr addrspace(1) %in2,
                              ptr addrspace(1) %in3) {
@@ -609,14 +2092,74 @@ define amdgpu_kernel void @test_f32_interp(ptr addrspace(1) %out,
   ret void
 }
 
-; FUNC-LABEL: {{^}}test_f64_interp:
-; SI-NOFMA: v_add_f64 [[VT1:v\[[0-9]+:[0-9]+\]]], -[[VT:v\[[0-9]+:[0-9]+\]]], 1.0
-; SI-NOFMA: v_mul_f64 [[VTY:v\[[0-9]+:[0-9]+\]]], [[VY:v\[[0-9]+:[0-9]+\]]], [[VT1]]
-; SI-NOFMA: v_fma_f64 v{{\[[0-9]+:[0-9]+\]}}, [[VX:v\[[0-9]+:[0-9]+\]]], [[VT]], [[VTY]]
-;
-; SI-FMA: v_fma_f64 [[VR:v\[[0-9]+:[0-9]+\]]], -[[VT:v\[[0-9]+:[0-9]+\]]], [[VY:v\[[0-9]+:[0-9]+\]]], [[VY]]
-; SI-FMA: v_fma_f64 v{{\[[0-9]+:[0-9]+\]}}, [[VX:v\[[0-9]+:[0-9]+\]]], [[VT]], [[VR]]
 define amdgpu_kernel void @test_f64_interp(ptr addrspace(1) %out,
+; SI-FMA-LABEL: test_f64_interp:
+; SI-FMA:       ; %bb.0:
+; SI-FMA-NEXT:    s_load_dwordx8 s[0:7], s[0:1], 0x9
+; SI-FMA-NEXT:    s_mov_b32 s11, 0xf000
+; SI-FMA-NEXT:    s_mov_b32 s10, -1
+; SI-FMA-NEXT:    s_mov_b32 s18, s10
+; SI-FMA-NEXT:    s_mov_b32 s19, s11
+; SI-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-FMA-NEXT:    s_mov_b32 s16, s4
+; SI-FMA-NEXT:    s_mov_b32 s17, s5
+; SI-FMA-NEXT:    s_mov_b32 s4, s6
+; SI-FMA-NEXT:    s_mov_b32 s5, s7
+; SI-FMA-NEXT:    s_mov_b32 s6, s10
+; SI-FMA-NEXT:    s_mov_b32 s7, s11
+; SI-FMA-NEXT:    buffer_load_dwordx2 v[0:1], off, s[16:19], 0
+; SI-FMA-NEXT:    buffer_load_dwordx2 v[2:3], off, s[4:7], 0
+; SI-FMA-NEXT:    s_mov_b32 s14, s10
+; SI-FMA-NEXT:    s_mov_b32 s12, s2
+; SI-FMA-NEXT:    s_mov_b32 s13, s3
+; SI-FMA-NEXT:    s_mov_b32 s15, s11
+; SI-FMA-NEXT:    buffer_load_dwordx2 v[4:5], off, s[12:15], 0
+; SI-FMA-NEXT:    s_mov_b32 s8, s0
+; SI-FMA-NEXT:    s_mov_b32 s9, s1
+; SI-FMA-NEXT:    s_waitcnt vmcnt(1)
+; SI-FMA-NEXT:    v_fma_f64 v[0:1], -v[2:3], v[0:1], v[0:1]
+; SI-FMA-NEXT:    s_waitcnt vmcnt(0)
+; SI-FMA-NEXT:    v_fma_f64 v[0:1], v[4:5], v[2:3], v[0:1]
+; SI-FMA-NEXT:    buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; SI-FMA-NEXT:    s_endpgm
+;
+; GFX11-NOFMA-LABEL: test_f64_interp:
+; GFX11-NOFMA:       ; %bb.0:
+; GFX11-NOFMA-NEXT:    s_load_b256 s[0:7], s[0:1], 0x24
+; GFX11-NOFMA-NEXT:    v_mov_b32_e32 v8, 0
+; GFX11-NOFMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NOFMA-NEXT:    s_clause 0x2
+; GFX11-NOFMA-NEXT:    global_load_b64 v[0:1], v8, s[6:7]
+; GFX11-NOFMA-NEXT:    global_load_b64 v[2:3], v8, s[4:5]
+; GFX11-NOFMA-NEXT:    global_load_b64 v[4:5], v8, s[2:3]
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(2)
+; GFX11-NOFMA-NEXT:    v_add_f64 v[6:7], -v[0:1], 1.0
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-NOFMA-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-NOFMA-NEXT:    v_mul_f64 v[2:3], v[2:3], v[6:7]
+; GFX11-NOFMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NOFMA-NEXT:    v_fma_f64 v[0:1], v[4:5], v[0:1], v[2:3]
+; GFX11-NOFMA-NEXT:    global_store_b64 v8, v[0:1], s[0:1]
+; GFX11-NOFMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NOFMA-NEXT:    s_endpgm
+;
+; GFX11-FMA-LABEL: test_f64_interp:
+; GFX11-FMA:       ; %bb.0:
+; GFX11-FMA-NEXT:    s_load_b256 s[0:7], s[0:1], 0x24
+; GFX11-FMA-NEXT:    v_mov_b32_e32 v6, 0
+; GFX11-FMA-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-FMA-NEXT:    s_clause 0x2
+; GFX11-FMA-NEXT:    global_load_b64 v[0:1], v6, s[4:5]
+; GFX11-FMA-NEXT:    global_load_b64 v[2:3], v6, s[6:7]
+; GFX11-FMA-NEXT:    global_load_b64 v[4:5], v6, s[2:3]
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(1)
+; GFX11-FMA-NEXT:    v_fma_f64 v[0:1], -v[2:3], v[0:1], v[0:1]
+; GFX11-FMA-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-FMA-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FMA-NEXT:    v_fma_f64 v[0:1], v[4:5], v[2:3], v[0:1]
+; GFX11-FMA-NEXT:    global_store_b64 v6, v[0:1], s[0:1]
+; GFX11-FMA-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-FMA-NEXT:    s_endpgm
                              ptr addrspace(1) %in1,
                              ptr addrspace(1) %in2,
                              ptr addrspace(1) %in3) {
@@ -632,13 +2175,36 @@ define amdgpu_kernel void @test_f64_interp(ptr addrspace(1) %out,
 }
 
 ; Make sure negative constant cancels out fneg
-; SI-LABEL: {{^}}fma_neg_2.0_neg_a_b_f32:
-; SI: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
-; SI: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
-; SI-NOT: [[A]]
-; SI-NOT: [[B]]
-; SI: v_fma_f32 v{{[0-9]+}}, [[A]], 2.0, [[B]]
 define amdgpu_kernel void @fma_neg_2.0_neg_a_b_f32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; SI-LABEL: fma_neg_2.0_neg_a_b_f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    buffer_load_dword v2, v[0:1], s[0:3], 0 addr64 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dword v3, v[0:1], s[0:3], 0 addr64 offset:4 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_fma_f32 v2, v2, 2.0, v3
+; SI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: fma_neg_2.0_neg_a_b_f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_b32 v1, v0, s[0:1] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b32 v2, v0, s[0:1] offset:4 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_fmac_f32_e32 v2, 2.0, v1
+; GFX11-NEXT:    global_store_b32 v0, v2, s[0:1]
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.0 = getelementptr float, ptr addrspace(1) %out, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -654,13 +2220,36 @@ define amdgpu_kernel void @fma_neg_2.0_neg_a_b_f32(ptr addrspace(1) %out, ptr ad
   ret void
 }
 
-; SI-LABEL: {{^}}fma_2.0_neg_a_b_f32:
-; SI: {{buffer|flat|global}}_load_dword [[A:v[0-9]+]]
-; SI: {{buffer|flat|global}}_load_dword [[B:v[0-9]+]]
-; SI-NOT: [[A]]
-; SI-NOT: [[B]]
-; SI: v_fma_f32 v{{[0-9]+}}, [[A]], -2.0, [[B]]
 define amdgpu_kernel void @fma_2.0_neg_a_b_f32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
+; SI-LABEL: fma_2.0_neg_a_b_f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s3, 0xf000
+; SI-NEXT:    s_mov_b32 s2, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; SI-NEXT:    v_mov_b32_e32 v1, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    buffer_load_dword v2, v[0:1], s[0:3], 0 addr64 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    buffer_load_dword v3, v[0:1], s[0:3], 0 addr64 offset:4 glc
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_fma_f32 v2, v2, -2.0, v3
+; SI-NEXT:    buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: fma_2.0_neg_a_b_f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX11-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    global_load_b32 v1, v0, s[0:1] glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    global_load_b32 v2, v0, s[0:1] offset:4 glc dlc
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_fmac_f32_e32 v2, -2.0, v1
+; GFX11-NEXT:    global_store_b32 v0, v2, s[0:1]
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.0 = getelementptr float, ptr addrspace(1) %out, i32 %tid
   %gep.1 = getelementptr float, ptr addrspace(1) %gep.0, i32 1
@@ -676,12 +2265,45 @@ define amdgpu_kernel void @fma_2.0_neg_a_b_f32(ptr addrspace(1) %out, ptr addrsp
   ret void
 }
 
-; SI-LABEL: {{^}}fma_neg_b_c_v4f32:
-; SI: v_fma_f32 v{{[0-9]+}}, v{{[0-9]+}}, -v{{[0-9]+}}, -v{{[0-9]+}}
-; SI: v_fma_f32 v{{[0-9]+}}, v{{[0-9]+}}, -v{{[0-9]+}}, -v{{[0-9]+}}
-; SI: v_fma_f32 v{{[0-9]+}}, v{{[0-9]+}}, -v{{[0-9]+}}, -v{{[0-9]+}}
-; SI: v_fma_f32 v{{[0-9]+}}, v{{[0-9]+}}, -v{{[0-9]+}}, -v{{[0-9]+}}
 define amdgpu_kernel void @fma_neg_b_c_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %in) #2 {
+; SI-LABEL: fma_neg_b_c_v4f32:
+; SI:       ; %bb.0:
+; SI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x9
+; SI-NEXT:    s_mov_b32 s7, 0xf000
+; SI-NEXT:    s_mov_b32 s6, 0
+; SI-NEXT:    v_lshlrev_b32_e32 v12, 4, v0
+; SI-NEXT:    v_mov_b32_e32 v13, 0
+; SI-NEXT:    s_waitcnt lgkmcnt(0)
+; SI-NEXT:    s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT:    buffer_load_dwordx4 v[0:3], v[12:13], s[4:7], 0 addr64
+; SI-NEXT:    buffer_load_dwordx4 v[4:7], v[12:13], s[4:7], 0 addr64 offset:16
+; SI-NEXT:    buffer_load_dwordx4 v[8:11], v[12:13], s[4:7], 0 addr64 offset:48
+; SI-NEXT:    s_mov_b64 s[2:3], s[6:7]
+; SI-NEXT:    s_waitcnt vmcnt(0)
+; SI-NEXT:    v_fma_f32 v3, v11, -v3, -v7
+; SI-NEXT:    v_fma_f32 v2, v10, -v2, -v6
+; SI-NEXT:    v_fma_f32 v1, v9, -v1, -v5
+; SI-NEXT:    v_fma_f32 v0, v8, -v0, -v4
+; SI-NEXT:    buffer_store_dwordx4 v[0:3], v[12:13], s[0:3], 0 addr64
+; SI-NEXT:    s_endpgm
+;
+; GFX11-LABEL: fma_neg_b_c_v4f32:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX11-NEXT:    v_lshlrev_b32_e32 v12, 4, v0
+; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-NEXT:    s_clause 0x2
+; GFX11-NEXT:    global_load_b128 v[0:3], v12, s[2:3] offset:16
+; GFX11-NEXT:    global_load_b128 v[4:7], v12, s[2:3]
+; GFX11-NEXT:    global_load_b128 v[8:11], v12, s[2:3] offset:48
+; GFX11-NEXT:    s_waitcnt vmcnt(0)
+; GFX11-NEXT:    v_fma_f32 v3, v11, -v7, -v3
+; GFX11-NEXT:    v_fma_f32 v2, v10, -v6, -v2
+; GFX11-NEXT:    v_fma_f32 v1, v9, -v5, -v1
+; GFX11-NEXT:    v_fma_f32 v0, v8, -v4, -v0
+; GFX11-NEXT:    global_store_b128 v12, v[0:3], s[0:1]
+; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep.0 = getelementptr <4 x float>, ptr addrspace(1) %in, i32 %tid
   %gep.1 = getelementptr <4 x float>, ptr addrspace(1) %gep.0, i32 1
diff --git a/llvm/test/CodeGen/AMDGPU/fma.f16.ll b/llvm/test/CodeGen/AMDGPU/fma.f16.ll
index 25937773eedac..a8a26103c59c4 100644
--- a/llvm/test/CodeGen/AMDGPU/fma.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fma.f16.ll
@@ -3,6 +3,8 @@
 ; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx900 < %s | FileCheck %s -check-prefixes=GFX9,GFX9-GISEL
 ; RUN: llc -global-isel=0 -march=amdgcn -mcpu=gfx1010 < %s | FileCheck %s -check-prefixes=GFX10,GFX10-SDAG
 ; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx1010 < %s | FileCheck %s -check-prefixes=GFX10,GFX10-GISEL
+; RUN: llc -global-isel=0 -march=amdgcn -mcpu=gfx1100 < %s | FileCheck %s -check-prefixes=GFX11,GFX11-SDAG
+; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx1100 < %s | FileCheck %s -check-prefixes=GFX11,GFX11-GISEL
 
 declare half @llvm.fma.f16(half, half, half)
 declare half @llvm.maxnum.f16(half, half)
@@ -20,6 +22,13 @@ define half @test_fma(half %x, half %y, half %z) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_fma_f16 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: test_fma:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    v_fma_f16 v0, v0, v1, v2
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %r = call half @llvm.fma.f16(half %x, half %y, half %z)
   ret half %r
 }
@@ -38,6 +47,13 @@ define half @test_fmac(half %x, half %y, half %z) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_fmac_f16_e32 v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: test_fmac:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    v_fmac_f16_e32 v0, v1, v2
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %r = call half @llvm.fma.f16(half %y, half %z, half %x)
   ret half %r
 }
@@ -64,6 +80,13 @@ define half @test_fmaak(half %x, half %y, half %z) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_fmaak_f16 v0, v0, v1, 0x4200
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: test_fmaak:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    v_fmaak_f16 v0, v0, v1, 0x4200
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %r = call half @llvm.fma.f16(half %x, half %y, half 0xH4200)
   ret half %r
 }
@@ -90,6 +113,13 @@ define half @test_fmamk(half %x, half %y, half %z) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_fmamk_f16 v0, v0, 0x4200, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-LABEL: test_fmamk:
+; GFX11:       ; %bb.0:
+; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-NEXT:    v_fmamk_f16 v0, v0, 0x4200, v2
+; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %r = call half @llvm.fma.f16(half %x, half 0xH4200, half %z)
   ret half %r
 }
@@ -134,6 +164,37 @@ define i32 @test_D139469_f16(half %arg) {
 ; GFX10-GISEL-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s4
 ; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: test_D139469_f16:
+; GFX11-SDAG:       ; %bb.0: ; %bb
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-SDAG-NEXT:    v_mov_b32_e32 v1, 0x211e
+; GFX11-SDAG-NEXT:    v_mul_f16_e32 v2, 0x291e, v0
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT:    v_fmac_f16_e32 v1, 0x291e, v0
+; GFX11-SDAG-NEXT:    v_cmp_gt_f16_e32 vcc_lo, 0, v2
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT:    v_cmp_gt_f16_e64 s0, 0, v1
+; GFX11-SDAG-NEXT:    s_or_b32 s0, vcc_lo, s0
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: test_D139469_f16:
+; GFX11-GISEL:       ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-GISEL-NEXT:    s_movk_i32 s0, 0x291e
+; GFX11-GISEL-NEXT:    v_mul_f16_e32 v1, 0x291e, v0
+; GFX11-GISEL-NEXT:    v_fmaak_f16 v0, s0, v0, 0x211e
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT:    v_cmp_gt_f16_e32 vcc_lo, 0, v1
+; GFX11-GISEL-NEXT:    v_cmp_gt_f16_e64 s0, 0, v0
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT:    s_or_b32 s0, vcc_lo, s0
+; GFX11-GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %i = fmul contract half %arg, 0xH291E
   %i1 = fcmp olt half %i, 0xH0000
@@ -216,6 +277,52 @@ define <2 x i32> @test_D139469_v2f16(<2 x half> %arg) {
 ; GFX10-GISEL-NEXT:    s_or_b32 s4, s6, s5
 ; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s4
 ; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-SDAG-LABEL: test_D139469_v2f16:
+; GFX11-SDAG:       ; %bb.0: ; %bb
+; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-SDAG-NEXT:    s_movk_i32 s0, 0x211e
+; GFX11-SDAG-NEXT:    v_pk_mul_f16 v1, 0x291e, v0 op_sel_hi:[0,1]
+; GFX11-SDAG-NEXT:    v_pk_fma_f16 v0, 0x291e, v0, s0 op_sel_hi:[0,1,0]
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
+; GFX11-SDAG-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GFX11-SDAG-NEXT:    v_cmp_gt_f16_e32 vcc_lo, 0, v1
+; GFX11-SDAG-NEXT:    v_cmp_gt_f16_e64 s0, 0, v0
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-SDAG-NEXT:    v_cmp_gt_f16_e64 s1, 0, v2
+; GFX11-SDAG-NEXT:    v_cmp_gt_f16_e64 s2, 0, v3
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT:    s_or_b32 s0, vcc_lo, s0
+; GFX11-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-SDAG-NEXT:    s_or_b32 s0, s1, s2
+; GFX11-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX11-GISEL-LABEL: test_D139469_v2f16:
+; GFX11-GISEL:       ; %bb.0: ; %bb
+; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-GISEL-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX11-GISEL-NEXT:    s_mov_b32 s0, 0x291e291e
+; GFX11-GISEL-NEXT:    v_pk_mul_f16 v1, v0, 0x291e op_sel_hi:[1,0]
+; GFX11-GISEL-NEXT:    v_pk_fma_f16 v0, v0, s0, 0x211e op_sel_hi:[1,1,0]
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
+; GFX11-GISEL-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
+; GFX11-GISEL-NEXT:    v_cmp_gt_f16_e32 vcc_lo, 0, v1
+; GFX11-GISEL-NEXT:    v_cmp_gt_f16_e64 s0, 0, v0
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-GISEL-NEXT:    v_cmp_gt_f16_e64 s1, 0, v2
+; GFX11-GISEL-NEXT:    v_cmp_gt_f16_e64 s2, 0, v3
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT:    s_or_b32 s0, vcc_lo, s0
+; GFX11-GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX11-GISEL-NEXT:    s_or_b32 s0, s1, s2
+; GFX11-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %i = fmul contract <2 x half> %arg, <half 0xH291E, half 0xH291E>
   %i1 = fcmp olt <2 x half> %i, <half 0xH0000, half 0xH0000>
diff --git a/llvm/test/CodeGen/AMDGPU/fma.f64.ll b/llvm/test/CodeGen/AMDGPU/fma.f64.ll
index 75bc89ffc39c0..f623dd8afe7f7 100644
--- a/llvm/test/CodeGen/AMDGPU/fma.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/fma.f64.ll
@@ -1,6 +1,7 @@
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,SI %s
-; RUN:  llc -amdgpu-scalarize-global-loads=false  -march=amdgcn -mcpu=gfx90a -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GFX90A %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,SIGFX11 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tonga -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,SIGFX11 %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx90a -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,GFX90A %s
+; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx1100 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -check-prefixes=FUNC,GCN,SIGFX11 %s
 
 declare double @llvm.fma.f64(double, double, double) nounwind readnone
 declare <2 x double> @llvm.fma.v2f64(<2 x double>, <2 x double>, <2 x double>) nounwind readnone
@@ -8,7 +9,7 @@ declare <4 x double> @llvm.fma.v4f64(<4 x double>, <4 x double>, <4 x double>) n
 declare double @llvm.fabs.f64(double) nounwind readnone
 
 ; FUNC-LABEL: {{^}}fma_f64:
-; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
+; SIGFX11: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
 ; GFX90A: v_fmac_f64_e32 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
 define amdgpu_kernel void @fma_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1,
                      ptr addrspace(1) %in2, ptr addrspace(1) %in3) {
@@ -21,8 +22,8 @@ define amdgpu_kernel void @fma_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1,
 }
 
 ; FUNC-LABEL: {{^}}fma_v2f64:
-; SI: v_fma_f64
-; SI: v_fma_f64
+; SIGFX11: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
+; SIGFX11: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
 ; GFX90A: v_fmac_f64_e32 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
 ; GFX90A: v_fmac_f64_e32 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
 define amdgpu_kernel void @fma_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %in1,
@@ -36,10 +37,10 @@ define amdgpu_kernel void @fma_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %in
 }
 
 ; FUNC-LABEL: {{^}}fma_v4f64:
-; SI: v_fma_f64
-; SI: v_fma_f64
-; SI: v_fma_f64
-; SI: v_fma_f64
+; SIGFX11: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
+; SIGFX11: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
+; SIGFX11: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
+; SIGFX11: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
 ; GFX90A: v_fmac_f64_e32 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
 ; GFX90A: v_fmac_f64_e32 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
 ; GFX90A: v_fmac_f64_e32 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\]}}
@@ -81,8 +82,7 @@ define amdgpu_kernel void @fma_f64_abs_src1(ptr addrspace(1) %out, ptr addrspace
 }
 
 ; FUNC-LABEL: {{^}}fma_f64_abs_src2:
-; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], \|v\[[0-9]+:[0-9]+\]\|}}
-; GFX90A: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], \|v\[[0-9]+:[0-9]+\]\|}}
+; GCN: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], \|v\[[0-9]+:[0-9]+\]\|}}
 define amdgpu_kernel void @fma_f64_abs_src2(ptr addrspace(1) %out, ptr addrspace(1) %in1,
                      ptr addrspace(1) %in2, ptr addrspace(1) %in3) {
    %r0 = load double, ptr addrspace(1) %in1
@@ -121,8 +121,7 @@ define amdgpu_kernel void @fma_f64_neg_src1(ptr addrspace(1) %out, ptr addrspace
 }
 
 ; FUNC-LABEL: {{^}}fma_f64_neg_src2:
-; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], -v\[[0-9]+:[0-9]+\]}}
-; GFX90A: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], -v\[[0-9]+:[0-9]+\]}}
+; GCN: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], -v\[[0-9]+:[0-9]+\]}}
 define amdgpu_kernel void @fma_f64_neg_src2(ptr addrspace(1) %out, ptr addrspace(1) %in1,
                      ptr addrspace(1) %in2, ptr addrspace(1) %in3) {
    %r0 = load double, ptr addrspace(1) %in1
@@ -163,8 +162,7 @@ define amdgpu_kernel void @fma_f64_abs_neg_src1(ptr addrspace(1) %out, ptr addrs
 }
 
 ; FUNC-LABEL: {{^}}fma_f64_abs_neg_src2:
-; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], -\|v\[[0-9]+:[0-9]+\]\|}}
-; GFX90A: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], -\|v\[[0-9]+:[0-9]+\]\|}}
+; GCN: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], -\|v\[[0-9]+:[0-9]+\]\|}}
 define amdgpu_kernel void @fma_f64_abs_neg_src2(ptr addrspace(1) %out, ptr addrspace(1) %in1,
                      ptr addrspace(1) %in2, ptr addrspace(1) %in3) {
    %r0 = load double, ptr addrspace(1) %in1
@@ -178,7 +176,7 @@ define amdgpu_kernel void @fma_f64_abs_neg_src2(ptr addrspace(1) %out, ptr addrs
 }
 
 ; FUNC-LABEL: {{^}}fma_f64_lit_src0:
-; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], 2.0, v\[[0-9]+:[0-9]+\]}}
+; SIGFX11: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], 2.0, v\[[0-9]+:[0-9]+\]}}
 ; GFX90A: v_fmac_f64_e32 {{v\[[0-9]+:[0-9]+\], 2.0, v\[[0-9]+:[0-9]+\]}}
 define amdgpu_kernel void @fma_f64_lit_src0(ptr addrspace(1) %out,
                      ptr addrspace(1) %in2, ptr addrspace(1) %in3) {
@@ -190,7 +188,7 @@ define amdgpu_kernel void @fma_f64_lit_src0(ptr addrspace(1) %out,
 }
 
 ; FUNC-LABEL: {{^}}fma_f64_lit_src1:
-; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], 2.0, v\[[0-9]+:[0-9]+\]}}
+; SIGFX11: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], 2.0, v\[[0-9]+:[0-9]+\]}}
 ; GFX90A: v_fmac_f64_e32 {{v\[[0-9]+:[0-9]+\], 2.0, v\[[0-9]+:[0-9]+\]}}
 define amdgpu_kernel void @fma_f64_lit_src1(ptr addrspace(1) %out, ptr addrspace(1) %in1,
                      ptr addrspace(1) %in3) {
@@ -202,8 +200,7 @@ define amdgpu_kernel void @fma_f64_lit_src1(ptr addrspace(1) %out, ptr addrspace
 }
 
 ; FUNC-LABEL: {{^}}fma_f64_lit_src2:
-; SI: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], 2.0}}
-; GFX90A: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], 2.0}}
+; GCN: v_fma_f64 {{v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], v\[[0-9]+:[0-9]+\], 2.0}}
 define amdgpu_kernel void @fma_f64_lit_src2(ptr addrspace(1) %out, ptr addrspace(1) %in1,
                      ptr addrspace(1) %in2) {
    %r0 = load double, ptr addrspace(1) %in1

From c7430ff9bfd3ff24f24cde1a5f6a5249713a2ff8 Mon Sep 17 00:00:00 2001
From: Francesco Petrogalli <francesco.petrogalli@apple.com>
Date: Tue, 20 Jun 2023 11:56:03 +0200
Subject: [PATCH 108/130] [CodeGen][test] Add missing `REQUIRES`.

Differential Revision: https://reviews.llvm.org/D153325
---
 .../test/CodeGen/AArch64/misched-detail-resource-booking-01.mir | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-01.mir b/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-01.mir
index a012fc5608a3f..bc34023c02ca4 100644
--- a/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-01.mir
+++ b/llvm/test/CodeGen/AArch64/misched-detail-resource-booking-01.mir
@@ -6,6 +6,8 @@
 # RUN:   -misched-dump-schedule-trace=true -misched-dump-schedule-trace-col-header-width=21 \
 # RUN: | FileCheck  %s
 
+# REQUIRES: asserts, aarch64-registered-target
+
 --- |
   ; ModuleID = '../llvm-project/llvm/test/CodeGen/AArch64/aarch64-smull.failing.ll'
   source_filename = "../llvm-project/llvm/test/CodeGen/AArch64/aarch64-smull.failing.ll"

From 1c64c414ed6a7ba2973f881a380d322adb5767a2 Mon Sep 17 00:00:00 2001
From: Jan Svoboda <jan_svoboda@apple.com>
Date: Tue, 20 Jun 2023 12:21:59 +0200
Subject: [PATCH 109/130] [clang][index] Fix cast warning

This is a follow-up to D151938 that should fix GCC's -Wcast-qual warning.
---
 clang/tools/libclang/CXFile.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/clang/tools/libclang/CXFile.h b/clang/tools/libclang/CXFile.h
index 8cb28a7ad0051..888593b7bc751 100644
--- a/clang/tools/libclang/CXFile.h
+++ b/clang/tools/libclang/CXFile.h
@@ -15,7 +15,8 @@
 namespace clang {
 namespace cxfile {
 inline CXFile makeCXFile(OptionalFileEntryRef FE) {
-  return CXFile(FE ? &FE->getMapEntry() : nullptr);
+  return CXFile(FE ? const_cast<FileEntryRef::MapEntry *>(&FE->getMapEntry())
+                   : nullptr);
 }
 
 inline OptionalFileEntryRef getFileEntryRef(CXFile File) {

From 2d3e6c440244ad94777aa13566b0376eb3c088f1 Mon Sep 17 00:00:00 2001
From: Ivan Kosarev <ivan.kosarev@amd.com>
Date: Tue, 20 Jun 2023 11:29:40 +0100
Subject: [PATCH 110/130] [AMDGPU] Drop GFX11 runs for dagcombine-fma-fmad.ll
 and fma.f16.ll.

They cause failures on the llvm-clang-x86_64-expensive-checks-debian
buildbot.

This partially reverts
D153269 [AMDGPU][GFX11] Add test coverage for FMA instructions.
---
 .../CodeGen/AMDGPU/dagcombine-fma-fmad.ll     | 352 ++++++------------
 llvm/test/CodeGen/AMDGPU/fma.f16.ll           | 107 ------
 2 files changed, 111 insertions(+), 348 deletions(-)

diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll
index 3cf0a82e9adfb..eda1355a35e36 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll
@@ -1,140 +1,71 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX10 %s
-; RUN: llc -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX11 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN %s
 
 define amdgpu_ps float @_amdgpu_ps_main() #0 {
-; GFX10-LABEL: _amdgpu_ps_main:
-; GFX10:       ; %bb.0: ; %.entry
-; GFX10-NEXT:    image_sample v[0:1], v[0:1], s[0:7], s[0:3] dmask:0x3 dim:SQ_RSRC_IMG_2D
-; GFX10-NEXT:    v_mov_b32_e32 v4, 0
-; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    image_sample v2, v[0:1], s[0:7], s[0:3] dmask:0x4 dim:SQ_RSRC_IMG_2D
-; GFX10-NEXT:    image_sample v3, v[0:1], s[0:7], s[0:3] dmask:0x1 dim:SQ_RSRC_IMG_2D
-; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    image_load_mip v4, v[2:4], s[0:7] dmask:0x4 dim:SQ_RSRC_IMG_2D unorm
-; GFX10-NEXT:    s_clause 0x3
-; GFX10-NEXT:    s_buffer_load_dword s24, s[0:3], 0x5c
-; GFX10-NEXT:    s_buffer_load_dword s28, s[0:3], 0x7c
-; GFX10-NEXT:    s_buffer_load_dword s29, s[0:3], 0xc0
-; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-NEXT:    s_nop 0
-; GFX10-NEXT:    s_buffer_load_dwordx4 s[0:3], s[0:3], 0x40
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    s_buffer_load_dwordx4 s[4:7], s[0:3], 0x50
-; GFX10-NEXT:    s_nop 0
-; GFX10-NEXT:    s_buffer_load_dword s0, s[0:3], 0x2c
-; GFX10-NEXT:    v_sub_f32_e64 v5, s24, s28
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_clause 0x4
-; GFX10-NEXT:    s_buffer_load_dwordx4 s[8:11], s[0:3], 0x60
-; GFX10-NEXT:    s_buffer_load_dwordx4 s[12:15], s[0:3], 0x20
-; GFX10-NEXT:    s_buffer_load_dwordx4 s[16:19], s[0:3], 0x0
-; GFX10-NEXT:    s_buffer_load_dwordx4 s[20:23], s[0:3], 0x70
-; GFX10-NEXT:    s_buffer_load_dwordx4 s[24:27], s[0:3], 0x10
-; GFX10-NEXT:    v_fma_f32 v1, v1, v5, s28
-; GFX10-NEXT:    v_max_f32_e64 v6, s0, s0 clamp
-; GFX10-NEXT:    v_add_f32_e64 v5, s29, -1.0
-; GFX10-NEXT:    v_sub_f32_e32 v8, s0, v1
-; GFX10-NEXT:    v_fma_f32 v7, -s2, v6, s6
-; GFX10-NEXT:    v_fma_f32 v5, v6, v5, 1.0
-; GFX10-NEXT:    v_mad_f32 v10, s2, v6, v2
-; GFX10-NEXT:    s_mov_b32 s0, 0x3c23d70a
-; GFX10-NEXT:    v_fmac_f32_e32 v1, v6, v8
-; GFX10-NEXT:    v_fmac_f32_e32 v10, v7, v6
-; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_mul_f32_e32 v9, s10, v0
-; GFX10-NEXT:    v_fma_f32 v0, -v0, s10, s14
-; GFX10-NEXT:    v_mul_f32_e32 v8, s18, v2
-; GFX10-NEXT:    v_mul_f32_e32 v3, s22, v3
-; GFX10-NEXT:    v_fmac_f32_e32 v9, v0, v6
-; GFX10-NEXT:    v_sub_f32_e32 v0, v1, v5
-; GFX10-NEXT:    v_mul_f32_e32 v1, v8, v6
-; GFX10-NEXT:    v_mul_f32_e32 v7, v6, v3
-; GFX10-NEXT:    v_fma_f32 v3, -v6, v3, v9
-; GFX10-NEXT:    v_fmac_f32_e32 v5, v0, v6
-; GFX10-NEXT:    v_fma_f32 v0, v2, s26, -v1
-; GFX10-NEXT:    v_fmac_f32_e32 v7, v3, v6
-; GFX10-NEXT:    v_fmac_f32_e32 v1, v0, v6
-; GFX10-NEXT:    v_mul_f32_e32 v0, v2, v6
-; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_add_f32_e32 v4, v4, v10
-; GFX10-NEXT:    v_mul_f32_e32 v3, v4, v6
-; GFX10-NEXT:    v_fmaak_f32 v4, s0, v5, 0x3ca3d70a
-; GFX10-NEXT:    v_mul_f32_e32 v1, v3, v1
-; GFX10-NEXT:    v_mul_f32_e32 v2, v7, v4
-; GFX10-NEXT:    v_fmac_f32_e32 v1, v2, v0
-; GFX10-NEXT:    v_max_f32_e32 v0, 0, v1
-; GFX10-NEXT:    ; return to shader part epilog
-;
-; GFX11-LABEL: _amdgpu_ps_main:
-; GFX11:       ; %bb.0: ; %.entry
-; GFX11-NEXT:    image_sample v[0:1], v[0:1], s[0:7], s[0:3] dmask:0x3 dim:SQ_RSRC_IMG_2D
-; GFX11-NEXT:    v_mov_b32_e32 v4, 0
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    image_sample v2, v[0:1], s[0:7], s[0:3] dmask:0x4 dim:SQ_RSRC_IMG_2D
-; GFX11-NEXT:    image_sample v3, v[0:1], s[0:7], s[0:3] dmask:0x1 dim:SQ_RSRC_IMG_2D
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    image_load_mip v4, v[2:4], s[0:7] dmask:0x4 dim:SQ_RSRC_IMG_2D unorm
-; GFX11-NEXT:    s_clause 0x3
-; GFX11-NEXT:    s_buffer_load_b32 s24, s[0:3], 0x5c
-; GFX11-NEXT:    s_buffer_load_b32 s28, s[0:3], 0x7c
-; GFX11-NEXT:    s_buffer_load_b32 s29, s[0:3], 0xc0
-; GFX11-NEXT:    s_buffer_load_b128 s[0:3], s[0:3], 0x40
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_buffer_load_b128 s[4:7], s[0:3], 0x50
-; GFX11-NEXT:    s_buffer_load_b32 s0, s[0:3], 0x2c
-; GFX11-NEXT:    v_sub_f32_e64 v5, s24, s28
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_clause 0x3
-; GFX11-NEXT:    s_buffer_load_b128 s[8:11], s[0:3], 0x60
-; GFX11-NEXT:    s_buffer_load_b128 s[12:15], s[0:3], 0x20
-; GFX11-NEXT:    s_buffer_load_b128 s[16:19], s[0:3], 0x0
-; GFX11-NEXT:    s_buffer_load_b128 s[20:23], s[0:3], 0x70
-; GFX11-NEXT:    v_fma_f32 v1, v1, v5, s28
-; GFX11-NEXT:    v_max_f32_e64 v6, s0, s0 clamp
-; GFX11-NEXT:    s_buffer_load_b128 s[24:27], s[0:3], 0x10
-; GFX11-NEXT:    v_add_f32_e64 v5, s29, -1.0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_sub_f32_e32 v8, s0, v1
-; GFX11-NEXT:    v_fma_f32 v7, -s2, v6, s6
-; GFX11-NEXT:    v_fma_f32 v10, s2, v6, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
-; GFX11-NEXT:    v_fma_f32 v5, v6, v5, 1.0
-; GFX11-NEXT:    s_mov_b32 s0, 0x3c23d70a
-; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    v_mul_f32_e32 v9, s10, v0
-; GFX11-NEXT:    v_fma_f32 v0, -v0, s10, s14
-; GFX11-NEXT:    v_mul_f32_e32 v3, s22, v3
-; GFX11-NEXT:    v_dual_fmac_f32 v1, v6, v8 :: v_dual_mul_f32 v8, s18, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_fmac_f32_e32 v9, v0, v6
-; GFX11-NEXT:    v_dual_fmac_f32 v10, v7, v6 :: v_dual_mul_f32 v7, v6, v3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_sub_f32_e32 v0, v1, v5
-; GFX11-NEXT:    v_fma_f32 v3, -v6, v3, v9
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_fmac_f32_e32 v7, v3, v6
-; GFX11-NEXT:    v_fmac_f32_e32 v5, v0, v6
-; GFX11-NEXT:    v_mul_f32_e32 v1, v8, v6
-; GFX11-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-NEXT:    v_add_f32_e32 v4, v4, v10
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_dual_mul_f32 v3, v4, v6 :: v_dual_fmaak_f32 v4, s0, v5, 0x3ca3d70a
-; GFX11-NEXT:    v_fma_f32 v0, v2, s26, -v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_fmac_f32_e32 v1, v0, v6
-; GFX11-NEXT:    v_mul_f32_e32 v0, v2, v6
-; GFX11-NEXT:    v_mul_f32_e32 v2, v7, v4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_mul_f32_e32 v1, v3, v1
-; GFX11-NEXT:    v_fmac_f32_e32 v1, v2, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_max_f32_e32 v0, 0, v1
-; GFX11-NEXT:    ; return to shader part epilog
+; GCN-LABEL: _amdgpu_ps_main:
+; GCN:       ; %bb.0: ; %.entry
+; GCN-NEXT:    image_sample v[0:1], v[0:1], s[0:7], s[0:3] dmask:0x3 dim:SQ_RSRC_IMG_2D
+; GCN-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    s_clause 0x1
+; GCN-NEXT:    image_sample v2, v[0:1], s[0:7], s[0:3] dmask:0x4 dim:SQ_RSRC_IMG_2D
+; GCN-NEXT:    image_sample v3, v[0:1], s[0:7], s[0:3] dmask:0x1 dim:SQ_RSRC_IMG_2D
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    image_load_mip v4, v[2:4], s[0:7] dmask:0x4 dim:SQ_RSRC_IMG_2D unorm
+; GCN-NEXT:    s_clause 0x3
+; GCN-NEXT:    s_buffer_load_dword s24, s[0:3], 0x5c
+; GCN-NEXT:    s_buffer_load_dword s28, s[0:3], 0x7c
+; GCN-NEXT:    s_buffer_load_dword s29, s[0:3], 0xc0
+; GCN-NEXT:    s_waitcnt_depctr 0xffe3
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_buffer_load_dwordx4 s[0:3], s[0:3], 0x40
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_clause 0x1
+; GCN-NEXT:    s_buffer_load_dwordx4 s[4:7], s[0:3], 0x50
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_buffer_load_dword s0, s[0:3], 0x2c
+; GCN-NEXT:    v_sub_f32_e64 v5, s24, s28
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_clause 0x4
+; GCN-NEXT:    s_buffer_load_dwordx4 s[8:11], s[0:3], 0x60
+; GCN-NEXT:    s_buffer_load_dwordx4 s[12:15], s[0:3], 0x20
+; GCN-NEXT:    s_buffer_load_dwordx4 s[16:19], s[0:3], 0x0
+; GCN-NEXT:    s_buffer_load_dwordx4 s[20:23], s[0:3], 0x70
+; GCN-NEXT:    s_buffer_load_dwordx4 s[24:27], s[0:3], 0x10
+; GCN-NEXT:    v_fma_f32 v1, v1, v5, s28
+; GCN-NEXT:    v_max_f32_e64 v6, s0, s0 clamp
+; GCN-NEXT:    v_add_f32_e64 v5, s29, -1.0
+; GCN-NEXT:    v_sub_f32_e32 v8, s0, v1
+; GCN-NEXT:    v_fma_f32 v7, -s2, v6, s6
+; GCN-NEXT:    v_fma_f32 v5, v6, v5, 1.0
+; GCN-NEXT:    v_mad_f32 v10, s2, v6, v2
+; GCN-NEXT:    s_mov_b32 s0, 0x3c23d70a
+; GCN-NEXT:    v_fmac_f32_e32 v1, v6, v8
+; GCN-NEXT:    v_fmac_f32_e32 v10, v7, v6
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mul_f32_e32 v9, s10, v0
+; GCN-NEXT:    v_fma_f32 v0, -v0, s10, s14
+; GCN-NEXT:    v_mul_f32_e32 v8, s18, v2
+; GCN-NEXT:    v_mul_f32_e32 v3, s22, v3
+; GCN-NEXT:    v_fmac_f32_e32 v9, v0, v6
+; GCN-NEXT:    v_sub_f32_e32 v0, v1, v5
+; GCN-NEXT:    v_mul_f32_e32 v1, v8, v6
+; GCN-NEXT:    v_mul_f32_e32 v7, v6, v3
+; GCN-NEXT:    v_fma_f32 v3, -v6, v3, v9
+; GCN-NEXT:    v_fmac_f32_e32 v5, v0, v6
+; GCN-NEXT:    v_fma_f32 v0, v2, s26, -v1
+; GCN-NEXT:    v_fmac_f32_e32 v7, v3, v6
+; GCN-NEXT:    v_fmac_f32_e32 v1, v0, v6
+; GCN-NEXT:    v_mul_f32_e32 v0, v2, v6
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_add_f32_e32 v4, v4, v10
+; GCN-NEXT:    v_mul_f32_e32 v3, v4, v6
+; GCN-NEXT:    v_fmaak_f32 v4, s0, v5, 0x3ca3d70a
+; GCN-NEXT:    v_mul_f32_e32 v1, v3, v1
+; GCN-NEXT:    v_mul_f32_e32 v2, v7, v4
+; GCN-NEXT:    v_fmac_f32_e32 v1, v2, v0
+; GCN-NEXT:    v_max_f32_e32 v0, 0, v1
+; GCN-NEXT:    ; return to shader part epilog
 .entry:
   %0 = call <3 x float> @llvm.amdgcn.image.sample.2d.v3f32.f32(i32 7, float undef, float undef, <8 x i32> undef, <4 x i32> undef, i1 false, i32 0, i32 0)
   %.i2243 = extractelement <3 x float> %0, i32 2
@@ -237,24 +168,14 @@ define amdgpu_ps float @_amdgpu_ps_main() #0 {
 }
 
 define float @fmac_sequence_simple(float %a, float %b, float %c, float %d, float %e) #0 {
-; GFX10-LABEL: fmac_sequence_simple:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_fma_f32 v2, v2, v3, v4
-; GFX10-NEXT:    v_fmac_f32_e32 v2, v0, v1
-; GFX10-NEXT:    v_mov_b32_e32 v0, v2
-; GFX10-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: fmac_sequence_simple:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT:    v_fma_f32 v2, v2, v3, v4
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_fmac_f32_e32 v2, v0, v1
-; GFX11-NEXT:    v_mov_b32_e32 v0, v2
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GCN-LABEL: fmac_sequence_simple:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_waitcnt_vscnt null, 0x0
+; GCN-NEXT:    v_fma_f32 v2, v2, v3, v4
+; GCN-NEXT:    v_fmac_f32_e32 v2, v0, v1
+; GCN-NEXT:    v_mov_b32_e32 v0, v2
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %t0 = fmul fast float %a, %b
   %t1 = fmul fast float %c, %d
   %t2 = fadd fast float %t0, %t1
@@ -263,27 +184,15 @@ define float @fmac_sequence_simple(float %a, float %b, float %c, float %d, float
 }
 
 define float @fmac_sequence_innermost_fmul(float %a, float %b, float %c, float %d, float %e, float %f, float %g) #0 {
-; GFX10-LABEL: fmac_sequence_innermost_fmul:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_mad_f32 v2, v2, v3, v6
-; GFX10-NEXT:    v_fmac_f32_e32 v2, v0, v1
-; GFX10-NEXT:    v_fmac_f32_e32 v2, v4, v5
-; GFX10-NEXT:    v_mov_b32_e32 v0, v2
-; GFX10-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: fmac_sequence_innermost_fmul:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT:    v_fma_f32 v2, v2, v3, v6
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_fmac_f32_e32 v2, v0, v1
-; GFX11-NEXT:    v_fmac_f32_e32 v2, v4, v5
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_mov_b32_e32 v0, v2
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GCN-LABEL: fmac_sequence_innermost_fmul:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_waitcnt_vscnt null, 0x0
+; GCN-NEXT:    v_mad_f32 v2, v2, v3, v6
+; GCN-NEXT:    v_fmac_f32_e32 v2, v0, v1
+; GCN-NEXT:    v_fmac_f32_e32 v2, v4, v5
+; GCN-NEXT:    v_mov_b32_e32 v0, v2
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %t0 = fmul fast float %a, %b
   %t1 = fmul fast float %c, %d
   %t2 = fadd fast float %t0, %t1
@@ -294,27 +203,15 @@ define float @fmac_sequence_innermost_fmul(float %a, float %b, float %c, float %
 }
 
 define float @fmac_sequence_innermost_fmul_swapped_operands(float %a, float %b, float %c, float %d, float %e, float %f, float %g) #0 {
-; GFX10-LABEL: fmac_sequence_innermost_fmul_swapped_operands:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT:    v_mad_f32 v2, v2, v3, v6
-; GFX10-NEXT:    v_fmac_f32_e32 v2, v0, v1
-; GFX10-NEXT:    v_fmac_f32_e32 v2, v4, v5
-; GFX10-NEXT:    v_mov_b32_e32 v0, v2
-; GFX10-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: fmac_sequence_innermost_fmul_swapped_operands:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT:    v_fma_f32 v2, v2, v3, v6
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_fmac_f32_e32 v2, v0, v1
-; GFX11-NEXT:    v_fmac_f32_e32 v2, v4, v5
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_mov_b32_e32 v0, v2
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
+; GCN-LABEL: fmac_sequence_innermost_fmul_swapped_operands:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    s_waitcnt_vscnt null, 0x0
+; GCN-NEXT:    v_mad_f32 v2, v2, v3, v6
+; GCN-NEXT:    v_fmac_f32_e32 v2, v0, v1
+; GCN-NEXT:    v_fmac_f32_e32 v2, v4, v5
+; GCN-NEXT:    v_mov_b32_e32 v0, v2
+; GCN-NEXT:    s_setpc_b64 s[30:31]
   %t0 = fmul fast float %a, %b
   %t1 = fmul fast float %c, %d
   %t2 = fadd fast float %t0, %t1
@@ -325,20 +222,12 @@ define float @fmac_sequence_innermost_fmul_swapped_operands(float %a, float %b,
 }
 
 define amdgpu_ps float @fmac_sequence_innermost_fmul_sgpr(float inreg %a, float inreg %b, float inreg %c, float inreg %d, float inreg %e, float inreg %f, float %g) #0 {
-; GFX10-LABEL: fmac_sequence_innermost_fmul_sgpr:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_mac_f32_e64 v0, s2, s3
-; GFX10-NEXT:    v_fmac_f32_e64 v0, s0, s1
-; GFX10-NEXT:    v_fmac_f32_e64 v0, s4, s5
-; GFX10-NEXT:    ; return to shader part epilog
-;
-; GFX11-LABEL: fmac_sequence_innermost_fmul_sgpr:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_fmac_f32_e64 v0, s2, s3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_fmac_f32_e64 v0, s0, s1
-; GFX11-NEXT:    v_fmac_f32_e64 v0, s4, s5
-; GFX11-NEXT:    ; return to shader part epilog
+; GCN-LABEL: fmac_sequence_innermost_fmul_sgpr:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_mac_f32_e64 v0, s2, s3
+; GCN-NEXT:    v_fmac_f32_e64 v0, s0, s1
+; GCN-NEXT:    v_fmac_f32_e64 v0, s4, s5
+; GCN-NEXT:    ; return to shader part epilog
   %t0 = fmul fast float %a, %b
   %t1 = fmul fast float %c, %d
   %t2 = fadd fast float %t0, %t1
@@ -349,25 +238,14 @@ define amdgpu_ps float @fmac_sequence_innermost_fmul_sgpr(float inreg %a, float
 }
 
 define amdgpu_ps float @fmac_sequence_innermost_fmul_multiple_use(float inreg %a, float inreg %b, float inreg %c, float inreg %d, float inreg %e, float inreg %f, float %g) #0 {
-; GFX10-LABEL: fmac_sequence_innermost_fmul_multiple_use:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_mul_f32_e64 v1, s2, s3
-; GFX10-NEXT:    v_fmac_f32_e64 v1, s0, s1
-; GFX10-NEXT:    v_fma_f32 v2, s5, s4, v1
-; GFX10-NEXT:    v_fmac_f32_e32 v1, s5, v2
-; GFX10-NEXT:    v_add_f32_e32 v0, v1, v0
-; GFX10-NEXT:    ; return to shader part epilog
-;
-; GFX11-LABEL: fmac_sequence_innermost_fmul_multiple_use:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_mul_f32_e64 v1, s2, s3
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_fmac_f32_e64 v1, s0, s1
-; GFX11-NEXT:    v_fma_f32 v2, s5, s4, v1
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_fmac_f32_e32 v1, s5, v2
-; GFX11-NEXT:    v_add_f32_e32 v0, v1, v0
-; GFX11-NEXT:    ; return to shader part epilog
+; GCN-LABEL: fmac_sequence_innermost_fmul_multiple_use:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_mul_f32_e64 v1, s2, s3
+; GCN-NEXT:    v_fmac_f32_e64 v1, s0, s1
+; GCN-NEXT:    v_fma_f32 v2, s5, s4, v1
+; GCN-NEXT:    v_fmac_f32_e32 v1, s5, v2
+; GCN-NEXT:    v_add_f32_e32 v0, v1, v0
+; GCN-NEXT:    ; return to shader part epilog
   %t0 = fmul fast float %a, %b
   %t1 = fmul fast float %c, %d
   %t2 = fadd fast float %t0, %t1
@@ -383,20 +261,12 @@ define amdgpu_ps float @fmac_sequence_innermost_fmul_multiple_use(float inreg %a
 ; selecting it as a multiply. In some cases the multiply is better because
 ; SIFoldOperands can fold it into a previous instruction as an output modifier.
 define amdgpu_ps float @fma_vs_output_modifier(float %x, i32 %n) #0 {
-; GFX10-LABEL: fma_vs_output_modifier:
-; GFX10:       ; %bb.0:
-; GFX10-NEXT:    v_cvt_f32_i32_e64 v1, v1 mul:2
-; GFX10-NEXT:    v_mul_f32_e32 v0, v0, v0
-; GFX10-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX10-NEXT:    ; return to shader part epilog
-;
-; GFX11-LABEL: fma_vs_output_modifier:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    v_cvt_f32_i32_e64 v1, v1 mul:2
-; GFX11-NEXT:    v_mul_f32_e32 v0, v0, v0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_mul_f32_e32 v0, v0, v1
-; GFX11-NEXT:    ; return to shader part epilog
+; GCN-LABEL: fma_vs_output_modifier:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_cvt_f32_i32_e64 v1, v1 mul:2
+; GCN-NEXT:    v_mul_f32_e32 v0, v0, v0
+; GCN-NEXT:    v_mul_f32_e32 v0, v0, v1
+; GCN-NEXT:    ; return to shader part epilog
   %s = sitofp i32 %n to float
   %m = fmul contract float %x, %x
   %a = fmul contract float %m, 2.0
diff --git a/llvm/test/CodeGen/AMDGPU/fma.f16.ll b/llvm/test/CodeGen/AMDGPU/fma.f16.ll
index a8a26103c59c4..25937773eedac 100644
--- a/llvm/test/CodeGen/AMDGPU/fma.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fma.f16.ll
@@ -3,8 +3,6 @@
 ; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx900 < %s | FileCheck %s -check-prefixes=GFX9,GFX9-GISEL
 ; RUN: llc -global-isel=0 -march=amdgcn -mcpu=gfx1010 < %s | FileCheck %s -check-prefixes=GFX10,GFX10-SDAG
 ; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx1010 < %s | FileCheck %s -check-prefixes=GFX10,GFX10-GISEL
-; RUN: llc -global-isel=0 -march=amdgcn -mcpu=gfx1100 < %s | FileCheck %s -check-prefixes=GFX11,GFX11-SDAG
-; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx1100 < %s | FileCheck %s -check-prefixes=GFX11,GFX11-GISEL
 
 declare half @llvm.fma.f16(half, half, half)
 declare half @llvm.maxnum.f16(half, half)
@@ -22,13 +20,6 @@ define half @test_fma(half %x, half %y, half %z) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_fma_f16 v0, v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: test_fma:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT:    v_fma_f16 v0, v0, v1, v2
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %r = call half @llvm.fma.f16(half %x, half %y, half %z)
   ret half %r
 }
@@ -47,13 +38,6 @@ define half @test_fmac(half %x, half %y, half %z) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_fmac_f16_e32 v0, v1, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: test_fmac:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT:    v_fmac_f16_e32 v0, v1, v2
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %r = call half @llvm.fma.f16(half %y, half %z, half %x)
   ret half %r
 }
@@ -80,13 +64,6 @@ define half @test_fmaak(half %x, half %y, half %z) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_fmaak_f16 v0, v0, v1, 0x4200
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: test_fmaak:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT:    v_fmaak_f16 v0, v0, v1, 0x4200
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %r = call half @llvm.fma.f16(half %x, half %y, half 0xH4200)
   ret half %r
 }
@@ -113,13 +90,6 @@ define half @test_fmamk(half %x, half %y, half %z) {
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    v_fmamk_f16 v0, v0, 0x4200, v2
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX11-LABEL: test_fmamk:
-; GFX11:       ; %bb.0:
-; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT:    v_fmamk_f16 v0, v0, 0x4200, v2
-; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %r = call half @llvm.fma.f16(half %x, half 0xH4200, half %z)
   ret half %r
 }
@@ -164,37 +134,6 @@ define i32 @test_D139469_f16(half %arg) {
 ; GFX10-GISEL-NEXT:    s_or_b32 s4, vcc_lo, s4
 ; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s4
 ; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX11-SDAG-LABEL: test_D139469_f16:
-; GFX11-SDAG:       ; %bb.0: ; %bb
-; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT:    v_mov_b32_e32 v1, 0x211e
-; GFX11-SDAG-NEXT:    v_mul_f16_e32 v2, 0x291e, v0
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT:    v_fmac_f16_e32 v1, 0x291e, v0
-; GFX11-SDAG-NEXT:    v_cmp_gt_f16_e32 vcc_lo, 0, v2
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT:    v_cmp_gt_f16_e64 s0, 0, v1
-; GFX11-SDAG-NEXT:    s_or_b32 s0, vcc_lo, s0
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX11-GISEL-LABEL: test_D139469_f16:
-; GFX11-GISEL:       ; %bb.0: ; %bb
-; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-GISEL-NEXT:    s_movk_i32 s0, 0x291e
-; GFX11-GISEL-NEXT:    v_mul_f16_e32 v1, 0x291e, v0
-; GFX11-GISEL-NEXT:    v_fmaak_f16 v0, s0, v0, 0x211e
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT:    v_cmp_gt_f16_e32 vcc_lo, 0, v1
-; GFX11-GISEL-NEXT:    v_cmp_gt_f16_e64 s0, 0, v0
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-NEXT:    s_or_b32 s0, vcc_lo, s0
-; GFX11-GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %i = fmul contract half %arg, 0xH291E
   %i1 = fcmp olt half %i, 0xH0000
@@ -277,52 +216,6 @@ define <2 x i32> @test_D139469_v2f16(<2 x half> %arg) {
 ; GFX10-GISEL-NEXT:    s_or_b32 s4, s6, s5
 ; GFX10-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s4
 ; GFX10-GISEL-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX11-SDAG-LABEL: test_D139469_v2f16:
-; GFX11-SDAG:       ; %bb.0: ; %bb
-; GFX11-SDAG-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-SDAG-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-SDAG-NEXT:    s_movk_i32 s0, 0x211e
-; GFX11-SDAG-NEXT:    v_pk_mul_f16 v1, 0x291e, v0 op_sel_hi:[0,1]
-; GFX11-SDAG-NEXT:    v_pk_fma_f16 v0, 0x291e, v0, s0 op_sel_hi:[0,1,0]
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-SDAG-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
-; GFX11-SDAG-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX11-SDAG-NEXT:    v_cmp_gt_f16_e32 vcc_lo, 0, v1
-; GFX11-SDAG-NEXT:    v_cmp_gt_f16_e64 s0, 0, v0
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-SDAG-NEXT:    v_cmp_gt_f16_e64 s1, 0, v2
-; GFX11-SDAG-NEXT:    v_cmp_gt_f16_e64 s2, 0, v3
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT:    s_or_b32 s0, vcc_lo, s0
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11-SDAG-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT:    s_or_b32 s0, s1, s2
-; GFX11-SDAG-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s0
-; GFX11-SDAG-NEXT:    s_setpc_b64 s[30:31]
-;
-; GFX11-GISEL-LABEL: test_D139469_v2f16:
-; GFX11-GISEL:       ; %bb.0: ; %bb
-; GFX11-GISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-GISEL-NEXT:    s_waitcnt_vscnt null, 0x0
-; GFX11-GISEL-NEXT:    s_mov_b32 s0, 0x291e291e
-; GFX11-GISEL-NEXT:    v_pk_mul_f16 v1, v0, 0x291e op_sel_hi:[1,0]
-; GFX11-GISEL-NEXT:    v_pk_fma_f16 v0, v0, s0, 0x211e op_sel_hi:[1,1,0]
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-GISEL-NEXT:    v_lshrrev_b32_e32 v2, 16, v1
-; GFX11-GISEL-NEXT:    v_lshrrev_b32_e32 v3, 16, v0
-; GFX11-GISEL-NEXT:    v_cmp_gt_f16_e32 vcc_lo, 0, v1
-; GFX11-GISEL-NEXT:    v_cmp_gt_f16_e64 s0, 0, v0
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-GISEL-NEXT:    v_cmp_gt_f16_e64 s1, 0, v2
-; GFX11-GISEL-NEXT:    v_cmp_gt_f16_e64 s2, 0, v3
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-NEXT:    s_or_b32 s0, vcc_lo, s0
-; GFX11-GISEL-NEXT:    v_cndmask_b32_e64 v0, 0, 1, s0
-; GFX11-GISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-GISEL-NEXT:    s_or_b32 s0, s1, s2
-; GFX11-GISEL-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s0
-; GFX11-GISEL-NEXT:    s_setpc_b64 s[30:31]
 bb:
   %i = fmul contract <2 x half> %arg, <half 0xH291E, half 0xH291E>
   %i1 = fcmp olt <2 x half> %i, <half 0xH0000, half 0xH0000>

From b6f30623af36b6858669775cbaa8521cccf042ba Mon Sep 17 00:00:00 2001
From: Chuanqi Xu <yedeng.yd@linux.alibaba.com>
Date: Tue, 20 Jun 2023 18:35:58 +0800
Subject: [PATCH 111/130] [Coroutines] Store the index for final suspend point
 in the exception path

Try to address part of
https://github.com/llvm/llvm-project/issues/61900.

It is not completely addressed since the original reproducer is not
fixed due to the final suspend point is optimized out in its special
case. But that is a relatively independent issue.
---
 llvm/lib/Transforms/Coroutines/CoroSplit.cpp  | 32 ++++++++++++-------
 .../Coroutines/coro-split-final-suspend.ll    | 18 +++++++++--
 2 files changed, 37 insertions(+), 13 deletions(-)

diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index e3c4e94cf4d37..ca25fd0d84a4a 100644
--- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -300,6 +300,26 @@ static void markCoroutineAsDone(IRBuilder<> &Builder, const coro::Shape &Shape,
   auto *NullPtr = ConstantPointerNull::get(cast<PointerType>(
       Shape.FrameTy->getTypeAtIndex(coro::Shape::SwitchFieldIndex::Resume)));
   Builder.CreateStore(NullPtr, GepIndex);
+
+  // If the coroutine don't have unwind coro end, we could omit the store to
+  // the final suspend point since we could infer the coroutine is suspended
+  // at the final suspend point by the nullness of ResumeFnAddr.
+  // However, we can't skip it if the coroutine have unwind coro end. Since
+  // the coroutine reaches unwind coro end is considered suspended at the
+  // final suspend point (the ResumeFnAddr is null) but in fact the coroutine
+  // didn't complete yet. We need the IndexVal for the final suspend point
+  // to make the states clear.
+  if (Shape.SwitchLowering.HasUnwindCoroEnd &&
+      Shape.SwitchLowering.HasFinalSuspend) {
+    assert(cast<CoroSuspendInst>(Shape.CoroSuspends.back())->isFinal() &&
+           "The final suspend should only live in the last position of "
+           "CoroSuspends.");
+    ConstantInt *IndexVal = Shape.getIndex(Shape.CoroSuspends.size() - 1);
+    auto *FinalIndex = Builder.CreateStructGEP(
+        Shape.FrameTy, FramePtr, Shape.getSwitchIndexField(), "index.addr");
+
+    Builder.CreateStore(IndexVal, FinalIndex);
+  }
 }
 
 /// Replace an unwind call to llvm.coro.end.
@@ -397,17 +417,7 @@ static void createResumeEntryBlock(Function &F, coro::Shape &Shape) {
       // The coroutine should be marked done if it reaches the final suspend
       // point.
       markCoroutineAsDone(Builder, Shape, FramePtr);
-    }
-
-    // If the coroutine don't have unwind coro end, we could omit the store to
-    // the final suspend point since we could infer the coroutine is suspended
-    // at the final suspend point by the nullness of ResumeFnAddr.
-    // However, we can't skip it if the coroutine have unwind coro end. Since
-    // the coroutine reaches unwind coro end is considered suspended at the
-    // final suspend point (the ResumeFnAddr is null) but in fact the coroutine
-    // didn't complete yet. We need the IndexVal for the final suspend point
-    // to make the states clear.
-    if (!S->isFinal() || Shape.SwitchLowering.HasUnwindCoroEnd) {
+    } else {
       auto *GepIndex = Builder.CreateStructGEP(
           FrameTy, FramePtr, Shape.getSwitchIndexField(), "index.addr");
       Builder.CreateStore(IndexVal, GepIndex);
diff --git a/llvm/test/Transforms/Coroutines/coro-split-final-suspend.ll b/llvm/test/Transforms/Coroutines/coro-split-final-suspend.ll
index 73bd3c547404f..df99e8905b7b9 100644
--- a/llvm/test/Transforms/Coroutines/coro-split-final-suspend.ll
+++ b/llvm/test/Transforms/Coroutines/coro-split-final-suspend.ll
@@ -17,9 +17,10 @@ init_suspend:
   ret ptr %hdl
 
 init_resume:
-  br label %susp
+  invoke void @print(i32 1)
+          to label %final_suspend unwind label %lpad
 
-susp:
+final_suspend:
   %0 = call i8 @llvm.coro.suspend(token none, i1 true)
   switch i8 %0, label %suspend [i8 0, label %resume
                                 i8 1, label %suspend]
@@ -49,6 +50,19 @@ eh.resume:
 
 ; Tests that we need to store the final index if we see unwind coro end.
 ; CHECK: define{{.*}}@unwind_coro_end.resume
+; CHECK: invoke{{.*}}print
+; CHECK-NEXT: to label %[[RESUME:.*]] unwind label %[[LPAD:.*]]
+
+; CHECK: [[RESUME]]:
+; CHECK-NOT: {{.*:}}
+; CHECK: store ptr null, ptr %hdl
+; CHECK-NOT: {{.*:}}
+; CHECK: store i1 true, ptr %index.addr
+
+; CHECK: [[LPAD]]:
+; CHECK-NOT: {{.*:}}
+; CHECK: store ptr null, ptr %hdl
+; CHECK-NOT: {{.*:}}
 ; CHECK: store i1 true, ptr %index.addr
 
 ; Tests the use of final index in the destroy function.

From 36d50348a8b12db01d786c5fa3d0ed7a911e1f68 Mon Sep 17 00:00:00 2001
From: ManuelJBrito <manuel.brito@tecnico.ulisboa.pt>
Date: Sun, 11 Jun 2023 20:39:45 +0100
Subject: [PATCH 112/130] [clang][NFC] Drop alignment in
 builtin-nondeterministic-value test

Drop alignment to allow test to run in different platforms.

Differential Revision: https://reviews.llvm.org/D152547
---
 .../CodeGen/builtins-nondeterministic-value.c | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/clang/test/CodeGen/builtins-nondeterministic-value.c b/clang/test/CodeGen/builtins-nondeterministic-value.c
index aa040edf73d98..cc12f95997f04 100644
--- a/clang/test/CodeGen/builtins-nondeterministic-value.c
+++ b/clang/test/CodeGen/builtins-nondeterministic-value.c
@@ -5,8 +5,8 @@ typedef _Bool bool4 __attribute__((ext_vector_type(4)));
 
 int clang_nondet_i( int x ) {
 // CHECK-LABEL: entry
-// CHECK: [[A:%.*]] = alloca i32, align 4
-// CHECK: store i32 [[X:%.*]], ptr [[A]], align 4
+// CHECK: [[A:%.*]] = alloca i32
+// CHECK: store i32 [[X:%.*]], ptr [[A]]
 // CHECK: [[R:%.*]] = freeze i32 poison
 // CHECK: ret i32 [[R]]
   return __builtin_nondeterministic_value(x);
@@ -14,8 +14,8 @@ int clang_nondet_i( int x ) {
 
 float clang_nondet_f( float x ) {
 // CHECK-LABEL: entry
-// CHECK: [[A:%.*]] = alloca float, align 4
-// CHECK: store float [[X:%.*]], ptr [[A]], align 4
+// CHECK: [[A:%.*]] = alloca float
+// CHECK: store float [[X:%.*]], ptr [[A]]
 // CHECK: [[R:%.*]] = freeze float poison
 // CHECK: ret float [[R]]
   return __builtin_nondeterministic_value(x);
@@ -23,8 +23,8 @@ float clang_nondet_f( float x ) {
 
 double clang_nondet_d( double x ) {
 // CHECK-LABEL: entry
-// CHECK: [[A:%.*]] = alloca double, align 8
-// CHECK: store double [[X:%.*]], ptr [[A]], align 8
+// CHECK: [[A:%.*]] = alloca double
+// CHECK: store double [[X:%.*]], ptr [[A]]
 // CHECK: [[R:%.*]] = freeze double poison
 // CHECK: ret double [[R]]
   return __builtin_nondeterministic_value(x);
@@ -32,9 +32,9 @@ double clang_nondet_d( double x ) {
 
 _Bool clang_nondet_b( _Bool x) {
 // CHECK-LABEL: entry
-// CHECK: [[A:%.*]] = alloca i8, align 1
+// CHECK: [[A:%.*]] = alloca i8
 // CHECK: [[B:%.*]] = zext i1 %x to i8
-// CHECK: store i8 [[B]], ptr [[A]], align 1
+// CHECK: store i8 [[B]], ptr [[A]]
 // CHECK: [[R:%.*]] = freeze i1 poison
 // CHECK: ret i1 [[R]]
   return __builtin_nondeterministic_value(x);
@@ -42,19 +42,19 @@ _Bool clang_nondet_b( _Bool x) {
 
 void clang_nondet_fv( ) {
 // CHECK-LABEL: entry
-// CHECK: [[A:%.*]] = alloca <4 x float>, align
+// CHECK: [[A:%.*]] = alloca <4 x float>
 // CHECK: [[R:%.*]] = freeze <4 x float> poison
-// CHECK: store <4 x float> [[R]], ptr [[A]], align
+// CHECK: store <4 x float> [[R]], ptr [[A]]
 // CHECK: ret void
   float4 x = __builtin_nondeterministic_value(x);
 }
 
 void clang_nondet_bv( ) {
-// CHECK: [[A:%.*]] = alloca i8, align
+// CHECK: [[A:%.*]] = alloca i8
 // CHECK: [[V:%.*]] = freeze <4 x i1> poison
 // CHECK: [[SV:%.*]] = shufflevector <4 x i1> [[V]], <4 x i1> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
 // CHECK: [[BC:%.*]] = bitcast <8 x i1> [[SV]] to i8
-// CHECK: store i8 [[BC]], ptr [[A]], align
+// CHECK: store i8 [[BC]], ptr [[A]]
 // CHECK: ret void
   bool4 x = __builtin_nondeterministic_value(x);
 }

From ef9159c7f6653c1c0f11535623cc6f9a2c9a13f6 Mon Sep 17 00:00:00 2001
From: Harvin Iriawan <harvin.iriawan@arm.com>
Date: Tue, 20 Jun 2023 11:48:18 +0100
Subject: [PATCH 113/130] [AArch64] Add Cortex-A510 specific scheduling

- Update the Cortex-A510 mcpu target to use A510 scheduling info instead of
  A55. Values taken are based on the A510 software optimisation guide
  https://developer.arm.com/documentation/PJDOC-466751330-536816/latest
- Make latency of most integer ops to 1. CPU uarch is able to resolve most
  integer ops in 1 cycle

Differential Revision: https://reviews.llvm.org/D152688
---
 llvm/lib/Target/AArch64/AArch64.td            |     3 +-
 llvm/lib/Target/AArch64/AArch64SchedA510.td   |  1381 +++
 .../AArch64/Cortex/A510-basic-instructions.s  |  3726 ++++++
 .../AArch64/Cortex/A510-neon-instructions.s   |  3213 +++++
 .../AArch64/Cortex/A510-sve-instructions.s    | 10263 ++++++++++++++++
 5 files changed, 18585 insertions(+), 1 deletion(-)
 create mode 100644 llvm/lib/Target/AArch64/AArch64SchedA510.td
 create mode 100644 llvm/test/tools/llvm-mca/AArch64/Cortex/A510-basic-instructions.s
 create mode 100644 llvm/test/tools/llvm-mca/AArch64/Cortex/A510-neon-instructions.s
 create mode 100644 llvm/test/tools/llvm-mca/AArch64/Cortex/A510-sve-instructions.s

diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td
index 75b02acf4ca1c..68760b49fafc5 100644
--- a/llvm/lib/Target/AArch64/AArch64.td
+++ b/llvm/lib/Target/AArch64/AArch64.td
@@ -733,6 +733,7 @@ def SMEUnsupported : AArch64Unsupported {
 
 include "AArch64SchedA53.td"
 include "AArch64SchedA55.td"
+include "AArch64SchedA510.td"
 include "AArch64SchedA57.td"
 include "AArch64SchedCyclone.td"
 include "AArch64SchedFalkor.td"
@@ -1347,7 +1348,7 @@ def : ProcessorModel<"cortex-a53", CortexA53Model, ProcessorFeatures.A53,
                      [TuneA53]>;
 def : ProcessorModel<"cortex-a55", CortexA55Model, ProcessorFeatures.A55,
                      [TuneA55]>;
-def : ProcessorModel<"cortex-a510", CortexA55Model, ProcessorFeatures.A510,
+def : ProcessorModel<"cortex-a510", CortexA510Model, ProcessorFeatures.A510,
                      [TuneA510]>;
 def : ProcessorModel<"cortex-a57", CortexA57Model, ProcessorFeatures.A53,
                      [TuneA57]>;
diff --git a/llvm/lib/Target/AArch64/AArch64SchedA510.td b/llvm/lib/Target/AArch64/AArch64SchedA510.td
new file mode 100644
index 0000000000000..85e73b2fff6bc
--- /dev/null
+++ b/llvm/lib/Target/AArch64/AArch64SchedA510.td
@@ -0,0 +1,1381 @@
+//==- AArch64SchedCortexA510.td - ARM Cortex-A510 Scheduling Definitions -*- tablegen -*-=//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the machine model for the ARM Cortex-A510 processor.
+//
+//===----------------------------------------------------------------------===//
+
+// ===---------------------------------------------------------------------===//
+// The following definitions describe the per-operand machine model.
+// This works with MachineScheduler. See MCSchedModel.h for details.
+
+// Cortex-A510 machine model for scheduling and other instruction cost heuristics.
+def CortexA510Model : SchedMachineModel {
+  let MicroOpBufferSize = 0;  // The Cortex-A510 is an in-order processor
+  let IssueWidth = 3;         // It dual-issues under most circumstances
+  let LoadLatency = 3;        // Cycles for loads to access the cache.
+                              // Most loads have a latency of 2, but some have higher latencies.
+                              // 3 seems to be a good tradeoff
+  let PostRAScheduler = 1;    // Enable PostRA scheduler pass.
+  let CompleteModel = 0;      // Covers instructions applicable to Cortex-A510.
+
+  // FIXME: Remove when all errors have been fixed.
+  let FullInstRWOverlapCheck = 0;
+}
+
+
+//===----------------------------------------------------------------------===//
+// Subtarget-specific SchedWrite types
+
+let SchedModel = CortexA510Model in {
+
+//===----------------------------------------------------------------------===//
+// Define each kind of processor resource and number available.
+
+// Modeling each pipeline as a ProcResource using the BufferSize = 0 since the
+// Cortex-A510 is in-order.
+let BufferSize = 0 in {
+  def CortexA510UnitALU0   : ProcResource<1>;    // Int ALU0
+  def CortexA510UnitALU12  : ProcResource<2>;    // Int ALU1 & ALU2
+  def CortexA510UnitMAC    : ProcResource<1>;    // Int MAC, 64-bi wide
+  def CortexA510UnitDiv    : ProcResource<1>;    // Int Division, not pipelined
+  // There are 2 LS pipes, 1 for Load/Store; 1 for Store only
+  def CortexA510UnitLdSt   : ProcResource<1>;    // Load/Store shared pipe
+  def CortexA510UnitLd1    : ProcResource<1>;    // Load pipe
+  def CortexA510UnitB      : ProcResource<1>;    // Branch
+  def CortexA510UnitPAC    : ProcResource<1>;    // Pointer Authentication (PAC) pipe
+
+  // The FP DIV/SQRT instructions execute totally differently from the FP ALU
+  // instructions, which can mostly be dual-issued; that's why for now we model
+  // them with 2 resources.
+  def CortexA510UnitVALU0  : ProcResource<1>;    // SIMD/FP/SVE ALU0
+  def CortexA510UnitVALU1  : ProcResource<1>;    // SIMD/FP/SVE ALU0
+  def CortexA510UnitVMAC   : ProcResource<2>;    // SIMD/FP/SVE MAC
+  def CortexA510UnitVMC    : ProcResource<1>;    // SIMD/FP/SVE multicycle instrs  (e.g Div, SQRT, cryptography)
+}
+
+def CortexA510UnitLd     : ProcResGroup<[CortexA510UnitLdSt, CortexA510UnitLd1]>;
+def CortexA510UnitVALU   : ProcResGroup<[CortexA510UnitVALU0, CortexA510UnitVALU1]>;
+def CortexA510UnitALU    : ProcResGroup<[CortexA510UnitALU0, CortexA510UnitALU12]>;
+// These latencies are modeled without taking into account forwarding paths
+// (the software optimisation guide lists latencies taking into account
+// typical forwarding paths).
+def : WriteRes<WriteImm, [CortexA510UnitALU]> { let Latency = 1; }    // MOVN, MOVZ
+def : WriteRes<WriteI, [CortexA510UnitALU]> { let Latency = 1; }      // ALU
+def : WriteRes<WriteISReg, [CortexA510UnitALU]> { let Latency = 2; }  // ALU of Shifted-Reg
+def : WriteRes<WriteIEReg, [CortexA510UnitALU]> { let Latency = 2; }  // ALU of Extended-Reg
+def : WriteRes<WriteExtr, [CortexA510UnitALU]> { let Latency = 2; }   // EXTR from a reg pair
+def : WriteRes<WriteIS, [CortexA510UnitALU]> { let Latency = 2; }     // Shift/Scale
+
+// MAC
+def : WriteRes<WriteIM32, [CortexA510UnitMAC]> { let Latency = 3; }   // 32-bit Multiply
+def : WriteRes<WriteIM64, [CortexA510UnitMAC]> { let Latency = 5; let ResourceCycles = [2];}   // 64-bit Multiply
+
+// Div
+def : WriteRes<WriteID32, [CortexA510UnitDiv]> {
+  let Latency = 8; let ResourceCycles = [8];
+}
+def : WriteRes<WriteID64, [CortexA510UnitDiv]> {
+  let Latency = 16; let ResourceCycles = [16];
+}
+
+//===----------------------------------------------------------------------===//
+// Define customized scheduler read/write types specific to the Cortex A510
+
+//===----------------------------------------------------------------------===//
+class CortexA510Write<int n, ProcResourceKind res> : SchedWriteRes<[res]> {
+  let Latency = n;
+}
+
+class CortexA510MCWrite<int n, int m, ProcResourceKind res> : SchedWriteRes<[res]> {
+  let Latency = n;
+  let ResourceCycles = [m];
+  let BeginGroup = 1;
+}
+
+class CortexA510MC_RC0Write<int n, ProcResourceKind res> : SchedWriteRes<[res]> {
+  let Latency = n;
+  let BeginGroup = 1;
+}
+
+//===----------------------------------------------------------------------===//
+// Define generic 2 micro-op types
+def A510Write_10cyc_1VMAC_1VALU : SchedWriteRes<[CortexA510UnitVALU, CortexA510UnitVMAC]> {
+  let Latency     = 10;
+  let NumMicroOps = 2;
+}
+
+def A510Write_15cyc_1VMAC_1VALU : SchedWriteRes<[CortexA510UnitVALU, CortexA510UnitVMAC]> {
+  let Latency     = 15;
+  let NumMicroOps = 2;
+}
+
+class A510Write_PAC_B <int lat> : SchedWriteRes<[CortexA510UnitPAC, CortexA510UnitB]> {
+  let Latency = lat;
+  let NumMicroOps = 2;
+}
+// Load
+def : WriteRes<WriteLD, [CortexA510UnitLd]> { let Latency = 2; }
+def : WriteRes<WriteLDIdx, [CortexA510UnitLd]> { let Latency = 2; }
+def : WriteRes<WriteLDHi, [CortexA510UnitLd]> { let Latency = 2; }
+
+def CortexA510WriteVLD1 : SchedWriteRes<[CortexA510UnitLd]> { let Latency = 3; }
+def CortexA510WriteVLD1SI : SchedWriteRes<[CortexA510UnitLd]> { let Latency = 3; let SingleIssue = 1; }
+def CortexA510WriteVLD2 : SchedWriteRes<[CortexA510UnitLd]> { let Latency = 4;
+                                                  let ResourceCycles = [2]; }
+def CortexA510WriteVLD3 : SchedWriteRes<[CortexA510UnitLd]> { let Latency = 5;
+                                                  let ResourceCycles = [3]; }
+def CortexA510WriteVLD4 : SchedWriteRes<[CortexA510UnitLd]> { let Latency = 6;
+                                                  let ResourceCycles = [4]; }
+def CortexA510WriteVLD6 : SchedWriteRes<[CortexA510UnitLd]> { let Latency = 5;
+                                                  let ResourceCycles = [3]; }
+def CortexA510WriteVLD8 : SchedWriteRes<[CortexA510UnitLd]> { let Latency = 6;
+                                                  let ResourceCycles = [4]; }
+
+def CortexA510WriteLDP1 : SchedWriteRes<[CortexA510UnitLd]> { let Latency = 3; }
+def CortexA510WriteLDP2 : SchedWriteRes<[CortexA510UnitLd]> { let Latency = 3; }
+def CortexA510WriteLDP4 : SchedWriteRes<[CortexA510UnitLd]> { let Latency = 3; }
+
+// Pre/Post Indexing - Performed as part of address generation
+def : WriteRes<WriteAdr, []> { let Latency = 0; }
+
+// Store
+let RetireOOO = 1 in {
+def : WriteRes<WriteST, [CortexA510UnitLdSt]> { let Latency = 1; }
+def : WriteRes<WriteSTP, [CortexA510UnitLdSt]> { let Latency = 1; }
+def : WriteRes<WriteSTIdx, [CortexA510UnitLdSt]> { let Latency = 1; }
+}
+def : WriteRes<WriteSTX, [CortexA510UnitLdSt]> { let Latency = 3; }
+
+// Vector Store - Similar to vector loads, can take 1-3 cycles to issue.
+def : WriteRes<WriteVST, [CortexA510UnitLdSt]> { let Latency = 5;
+                                          let ResourceCycles = [2];}
+def CortexA510WriteVST1 : SchedWriteRes<[CortexA510UnitLdSt]> { let Latency = 4; }
+def CortexA510WriteVST2 : SchedWriteRes<[CortexA510UnitLdSt]> { let Latency = 5;
+                                                  let ResourceCycles = [2]; }
+def CortexA510WriteVST3 : SchedWriteRes<[CortexA510UnitLdSt]> { let Latency = 5;
+                                                  let ResourceCycles = [3]; }
+def CortexA510WriteVST4 : SchedWriteRes<[CortexA510UnitLdSt]> { let Latency = 5;
+                                                  let ResourceCycles = [4]; }
+
+def : WriteRes<WriteAtomic, []> { let Unsupported = 1; }
+
+// Branch
+def : WriteRes<WriteBr, [CortexA510UnitB]>;
+def : WriteRes<WriteBrReg, [CortexA510UnitB]>;
+def : WriteRes<WriteSys, [CortexA510UnitB]>;
+def : WriteRes<WriteBarrier, [CortexA510UnitB]>;
+def : WriteRes<WriteHint, [CortexA510UnitB]>;
+
+// FP ALU
+//   As WriteF result is produced in F5 and it can be mostly forwarded
+//   to consumer at F1, the effectively Latency is set as 4.
+def : WriteRes<WriteF, [CortexA510UnitVALU]> { let Latency = 4; }
+def : WriteRes<WriteFCmp, [CortexA510UnitVALU]> { let Latency = 3; }
+def : WriteRes<WriteFCvt, [CortexA510UnitVALU]> { let Latency = 4; }
+def : WriteRes<WriteFCopy, [CortexA510UnitVALU]> { let Latency = 3; }
+def : WriteRes<WriteFImm, [CortexA510UnitVALU]> { let Latency = 3; }
+
+class CortexA510VSt<int n> : SchedWriteRes<[CortexA510UnitLdSt]> {
+  let RetireOOO = 1;
+  let ResourceCycles = [n];
+}
+
+def CortexA510VSt0      : SchedWriteRes<[CortexA510UnitLdSt]> {
+  let RetireOOO = 1;
+}
+
+def : SchedAlias<WriteVd, CortexA510Write<4, CortexA510UnitVALU>>;
+def : SchedAlias<WriteVq, CortexA510Write<4, CortexA510UnitVALU>>;
+
+// FP ALU specific new schedwrite definitions
+def CortexA510WriteFPALU_F3 : SchedWriteRes<[CortexA510UnitVALU]> { let Latency = 3;}
+def CortexA510WriteFPALU_F4 : SchedWriteRes<[CortexA510UnitVALU]> { let Latency = 4;}
+
+// FP Mul, Div, Sqrt. Div/Sqrt are not pipelined
+def : WriteRes<WriteFMul, [CortexA510UnitVMAC]> { let Latency = 4; }
+
+let RetireOOO = 1 in {
+def : WriteRes<WriteFDiv, [CortexA510UnitVMC]> { let Latency = 22;
+                                            let ResourceCycles = [29]; }
+def CortexA510WriteVMAC : SchedWriteRes<[CortexA510UnitVMAC]> { let Latency = 4; }
+def CortexA510WriteFDivHP : SchedWriteRes<[CortexA510UnitVMC]> { let Latency = 8;
+                                                     let ResourceCycles = [5]; }
+def CortexA510WriteFDivSP : SchedWriteRes<[CortexA510UnitVMC]> { let Latency = 13;
+                                                     let ResourceCycles = [10]; }
+def CortexA510WriteFDivDP : SchedWriteRes<[CortexA510UnitVMC]> { let Latency = 22;
+                                                     let ResourceCycles = [19]; }
+def CortexA510WriteFSqrtHP : SchedWriteRes<[CortexA510UnitVMC]> { let Latency = 8;
+                                                      let ResourceCycles = [5]; }
+def CortexA510WriteFSqrtSP : SchedWriteRes<[CortexA510UnitVMC]> { let Latency = 12;
+                                                      let ResourceCycles = [9]; }
+def CortexA510WriteFSqrtDP : SchedWriteRes<[CortexA510UnitVMC]> { let Latency = 22;
+                                                      let ResourceCycles = [19]; }
+}
+//===----------------------------------------------------------------------===//
+// Subtarget-specific SchedRead types.
+
+def : ReadAdvance<ReadVLD, 0>;
+def : ReadAdvance<ReadExtrHi, 0>;
+def : ReadAdvance<ReadAdrBase, 0>;
+def : ReadAdvance<ReadST, 1>;
+
+def : ReadAdvance<ReadI, 0>;
+def : ReadAdvance<ReadISReg, 0>;
+def : ReadAdvance<ReadIEReg, 0>;
+
+
+// MUL
+def : ReadAdvance<ReadIM, 0>;
+def : ReadAdvance<ReadIMA, 2>;
+
+// Div
+def : ReadAdvance<ReadID, 0>;
+
+//===----------------------------------------------------------------------===//
+// Subtarget-specific InstRWs.
+
+def A510WriteISReg : SchedWriteVariant<[
+       SchedVar<RegShiftedPred, [WriteISReg]>,
+       SchedVar<NoSchedPred, [WriteI]>]>;
+def : InstRW<[A510WriteISReg], (instregex ".*rs$")>;
+def : InstRW<[WriteIS], (instrs RBITWr, RBITXr)>;
+
+// Pointer Authentication Instructions (v8.3 PAC)
+// -----------------------------------------------------------------------------
+
+// Authenticate data address
+// Authenticate instruction address
+// Compute pointer authentication code for data address
+// Compute pointer authentication code, using generic key
+// Compute pointer authentication code for instruction address
+def : InstRW<[CortexA510Write<3, CortexA510UnitPAC>], (instregex "^AUT", "^PAC")>;
+
+// Branch and link, register, with pointer authentication
+// Branch, register, with pointer authentication
+// Branch, return, with pointer authentication
+def : InstRW<[A510Write_PAC_B<1>], (instrs BLRAA, BLRAAZ, BLRAB, BLRABZ, BRAA,
+                                            BRAAZ, BRAB, BRABZ, RETAA, RETAB,
+                                            ERETAA, ERETAB)>;
+
+// Load register, with pointer authentication
+def : InstRW<[CortexA510Write<2, CortexA510UnitPAC>], (instregex "^LDRA[AB](indexed|writeback)")>;
+
+// Strip pointer authentication code
+def : InstRW<[CortexA510Write<5, CortexA510UnitPAC>], (instrs XPACD, XPACI, XPACLRI)>;
+//---
+// Miscellaneous
+//---
+def : InstRW<[CortexA510WriteVLD1SI,CortexA510WriteLDP1], (instregex "LDPS?Wi")>;
+def : InstRW<[CortexA510WriteVLD1,CortexA510WriteLDP1], (instregex "LDPSi")>;
+def : InstRW<[CortexA510WriteVLD1,CortexA510WriteLDP2], (instregex "LDP(X|D)i")>;
+def : InstRW<[CortexA510WriteVLD1,CortexA510WriteLDP4], (instregex "LDPQi")>;
+def : InstRW<[WriteAdr, CortexA510WriteVLD1SI,CortexA510WriteLDP1], (instregex "LDPS?W(pre|post)")>;
+def : InstRW<[WriteAdr, CortexA510WriteVLD1,CortexA510WriteLDP1], (instregex "LDPS(pre|post)")>;
+def : InstRW<[WriteAdr, CortexA510WriteVLD1,CortexA510WriteLDP2], (instregex "LDP(X|D)(pre|post)")>;
+def : InstRW<[WriteAdr, CortexA510WriteVLD1,CortexA510WriteLDP4], (instregex "LDPQ(pre|post)")>;
+def : InstRW<[WriteI], (instrs COPY)>;
+//---
+// Vector Loads - 128-bit per cycle
+//---
+//   1-element structures
+def : InstRW<[CortexA510WriteVLD1], (instregex "LD1i(8|16|32|64)$")>;                // single element
+def : InstRW<[CortexA510WriteVLD1], (instregex "LD1Rv(8b|4h|2s|1d|16b|8h|4s|2d)$")>; // replicate
+def : InstRW<[CortexA510WriteVLD1], (instregex "LD1Onev(8b|4h|2s|1d)$")>;
+def : InstRW<[CortexA510WriteVLD1], (instregex "LD1Onev(16b|8h|4s|2d)$")>;
+def : InstRW<[CortexA510WriteVLD1], (instregex "LD1Twov(8b|4h|2s|1d)$")>; // multiple structures
+def : InstRW<[CortexA510WriteVLD1], (instregex "LD1Twov(16b|8h|4s|2d)$")>;
+def : InstRW<[CortexA510WriteVLD2], (instregex "LD1Threev(8b|4h|2s|1d)$")>;
+def : InstRW<[CortexA510WriteVLD2], (instregex "LD1Threev(16b|8h|4s|2d)$")>;
+def : InstRW<[CortexA510WriteVLD2], (instregex "LD1Fourv(8b|4h|2s|1d)$")>;
+def : InstRW<[CortexA510WriteVLD2], (instregex "LD1Fourv(16b|8h|4s|2d)$")>;
+
+def : InstRW<[CortexA510WriteVLD1, WriteAdr], (instregex "LD1i(8|16|32|64)_POST$")>;
+def : InstRW<[CortexA510WriteVLD1, WriteAdr], (instregex "LD1Rv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+def : InstRW<[CortexA510WriteVLD1, WriteAdr], (instregex "LD1Onev(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[CortexA510WriteVLD2, WriteAdr], (instregex "LD1Onev(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[CortexA510WriteVLD2, WriteAdr], (instregex "LD1Twov(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[CortexA510WriteVLD2, WriteAdr], (instregex "LD1Twov(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[CortexA510WriteVLD2, WriteAdr], (instregex "LD1Threev(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[CortexA510WriteVLD2, WriteAdr], (instregex "LD1Threev(16b|8h|4s|2d)_POST$")>;
+def : InstRW<[CortexA510WriteVLD2, WriteAdr], (instregex "LD1Fourv(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[CortexA510WriteVLD2, WriteAdr], (instregex "LD1Fourv(16b|8h|4s|2d)_POST$")>;
+
+//    2-element structures
+def : InstRW<[CortexA510WriteVLD2], (instregex "LD2i(8|16|32|64)$")>;
+def : InstRW<[CortexA510WriteVLD2], (instregex "LD2Rv(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
+def : InstRW<[CortexA510WriteVLD2], (instregex "LD2Twov(8b|4h|2s)$")>;
+def : InstRW<[CortexA510WriteVLD4], (instregex "LD2Twov(16b|8h|4s|2d)$")>;
+
+def : InstRW<[CortexA510WriteVLD2, WriteAdr], (instregex "LD2i(8|16|32|64)(_POST)?$")>;
+def : InstRW<[CortexA510WriteVLD2, WriteAdr], (instregex "LD2Rv(8b|4h|2s|1d|16b|8h|4s|2d)(_POST)?$")>;
+def : InstRW<[CortexA510WriteVLD2, WriteAdr], (instregex "LD2Twov(8b|4h|2s)(_POST)?$")>;
+def : InstRW<[CortexA510WriteVLD4, WriteAdr], (instregex "LD2Twov(16b|8h|4s|2d)(_POST)?$")>;
+
+//    3-element structures
+def : InstRW<[CortexA510WriteVLD2], (instregex "LD3i(8|16|32|64)$")>;
+def : InstRW<[CortexA510WriteVLD2], (instregex "LD3Rv(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
+def : InstRW<[CortexA510WriteVLD3], (instregex "LD3Threev(8b|4h|2s|1d)$")>;
+def : InstRW<[CortexA510WriteVLD6], (instregex "LD3Threev(16b|8h|4s|2d)$")>;
+
+def : InstRW<[CortexA510WriteVLD2, WriteAdr], (instregex "LD3i(8|16|32|64)_POST$")>;
+def : InstRW<[CortexA510WriteVLD2, WriteAdr], (instregex "LD3Rv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+def : InstRW<[CortexA510WriteVLD3, WriteAdr], (instregex "LD3Threev(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[CortexA510WriteVLD6, WriteAdr], (instregex "LD3Threev(16b|8h|4s|2d)_POST$")>;
+
+//    4-element structures
+def : InstRW<[CortexA510WriteVLD2], (instregex "LD4i(8|16|32|64)$")>;                // load single 4-el structure to one lane of 4 regs.
+def : InstRW<[CortexA510WriteVLD2], (instregex "LD4Rv(8b|4h|2s|1d|16b|8h|4s|2d)$")>; // load single 4-el structure, replicate to all lanes of 4 regs.
+def : InstRW<[CortexA510WriteVLD4], (instregex "LD4Fourv(8b|4h|2s|1d)$")>;           // load multiple 4-el structures to 4 regs.
+def : InstRW<[CortexA510WriteVLD8], (instregex "LD4Fourv(16b|8h|4s|2d)$")>;
+
+def : InstRW<[CortexA510WriteVLD2, WriteAdr], (instregex "LD4i(8|16|32|64)_POST$")>;
+def : InstRW<[CortexA510WriteVLD2, WriteAdr], (instregex "LD4Rv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+def : InstRW<[CortexA510WriteVLD4, WriteAdr], (instregex "LD4Fourv(8b|4h|2s|1d)_POST$")>;
+def : InstRW<[CortexA510WriteVLD8, WriteAdr], (instregex "LD4Fourv(16b|8h|4s|2d)_POST$")>;
+
+//---
+// Vector Stores
+//---
+def : InstRW<[CortexA510WriteVST1], (instregex "ST1i(8|16|32|64)$")>;
+def : InstRW<[CortexA510WriteVST1], (instregex "ST1Onev(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
+def : InstRW<[CortexA510WriteVST1], (instregex "ST1Twov(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
+def : InstRW<[CortexA510WriteVST2], (instregex "ST1Threev(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
+def : InstRW<[CortexA510WriteVST4], (instregex "ST1Fourv(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
+def : InstRW<[CortexA510WriteVST1, WriteAdr], (instregex "ST1i(8|16|32|64)_POST$")>;
+def : InstRW<[CortexA510WriteVST1, WriteAdr], (instregex "ST1Onev(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+def : InstRW<[CortexA510WriteVST1, WriteAdr], (instregex "ST1Twov(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+def : InstRW<[CortexA510WriteVST2, WriteAdr], (instregex "ST1Threev(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+def : InstRW<[CortexA510WriteVST4, WriteAdr], (instregex "ST1Fourv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+
+def : InstRW<[CortexA510WriteVST2], (instregex "ST2i(8|16|32|64)$")>;
+def : InstRW<[CortexA510WriteVST2], (instregex "ST2Twov(8b|4h|2s)$")>;
+def : InstRW<[CortexA510WriteVST4], (instregex "ST2Twov(16b|8h|4s|2d)$")>;
+def : InstRW<[CortexA510WriteVST2, WriteAdr], (instregex "ST2i(8|16|32|64)_POST$")>;
+def : InstRW<[CortexA510WriteVST2, WriteAdr], (instregex "ST2Twov(8b|4h|2s)_POST$")>;
+def : InstRW<[CortexA510WriteVST4, WriteAdr], (instregex "ST2Twov(16b|8h|4s|2d)_POST$")>;
+
+def : InstRW<[CortexA510WriteVST2], (instregex "ST3i(8|16|32|64)$")>;
+def : InstRW<[CortexA510WriteVST4], (instregex "ST3Threev(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
+def : InstRW<[CortexA510WriteVST2, WriteAdr], (instregex "ST3i(8|16|32|64)_POST$")>;
+def : InstRW<[CortexA510WriteVST4, WriteAdr], (instregex "ST3Threev(8b|4h|2s|1d|2d|16b|8h|4s|4d)_POST$")>;
+
+def : InstRW<[CortexA510WriteVST2], (instregex "ST4i(8|16|32|64)$")>;
+def : InstRW<[CortexA510WriteVST4], (instregex "ST4Fourv(8b|4h|2s|1d|16b|8h|4s|2d)$")>;
+def : InstRW<[CortexA510WriteVST2, WriteAdr], (instregex "ST4i(8|16|32|64)_POST$")>;
+def : InstRW<[CortexA510WriteVST4, WriteAdr], (instregex "ST4Fourv(8b|4h|2s|1d|16b|8h|4s|2d)_POST$")>;
+
+//---
+// Floating Point Conversions, MAC, DIV, SQRT
+//---
+def : InstRW<[CortexA510WriteFPALU_F3], (instregex "^DUP(v2i64|v4i32|v8i16|v16i8)")>;
+def : InstRW<[CortexA510WriteFPALU_F4], (instregex "^XTN")>;
+def : InstRW<[CortexA510WriteFPALU_F4], (instregex "^FCVT[ALMNPZ][SU](S|U)?(W|X)")>;
+def : InstRW<[CortexA510WriteFPALU_F4], (instregex "^FCVT(X)?[ALMNPXZ](S|U|N)?v")>;
+
+def : InstRW<[CortexA510WriteFPALU_F4], (instregex "^(S|U)CVTF(S|U)(W|X)(H|S|D)")>;
+def : InstRW<[CortexA510WriteFPALU_F4], (instregex "^(S|U)CVTF(h|s|d)")>;
+def : InstRW<[CortexA510WriteFPALU_F4], (instregex "^(S|U)CVTFv")>;
+
+def : InstRW<[CortexA510WriteVMAC], (instregex "^FN?M(ADD|SUB).*")>;
+def : InstRW<[CortexA510WriteVMAC], (instregex "^FML(A|S)v.*")>;
+def : InstRW<[CortexA510WriteFDivHP], (instrs FDIVHrr)>;
+def : InstRW<[CortexA510WriteFDivSP], (instrs FDIVSrr)>;
+def : InstRW<[CortexA510WriteFDivDP], (instrs FDIVDrr)>;
+def : InstRW<[CortexA510WriteFDivHP], (instregex "^FDIVv.*16$")>;
+def : InstRW<[CortexA510WriteFDivSP], (instregex "^FDIVv.*32$")>;
+def : InstRW<[CortexA510WriteFDivDP], (instregex "^FDIVv.*64$")>;
+def : InstRW<[CortexA510WriteFSqrtHP], (instregex "^.*SQRT.*16$")>;
+def : InstRW<[CortexA510WriteFSqrtSP], (instregex "^.*SQRT.*32$")>;
+def : InstRW<[CortexA510WriteFSqrtDP], (instregex "^.*SQRT.*64$")>;
+
+// 4.15. Advanced SIMD integer instructions
+// ASIMD absolute diff
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "[SU]ABDv(2i32|4i16|8i8)")>;
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "[SU]ABDv(16i8|4i32|8i16)")>;
+// ASIMD absolute diff accum
+def : InstRW<[CortexA510Write<8, CortexA510UnitVALU>], (instregex "[SU]ABAL?v")>;
+// ASIMD absolute diff long
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "[SU]ABDLv")>;
+// ASIMD arith #1
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "(ADD|SUB|NEG)v(1i64|2i32|4i16|8i8)",
+  "[SU]R?HADDv(2i32|4i16|8i8)", "[SU]HSUBv(2i32|4i16|8i8)")>;
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "(ADD|SUB|NEG)v(2i64|4i32|8i16|16i8)",
+  "[SU]R?HADDv(8i16|4i32|16i8)", "[SU]HSUBv(8i16|4i32|16i8)")>;
+// ASIMD arith #2
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "ABSv(1i64|2i32|4i16|8i8)$",
+  "[SU]ADDLPv(2i32_v1i64|4i16_v2i32|8i8_v4i16)$",
+  "([SU]QADD|[SU]QSUB|SQNEG|SUQADD|USQADD)v(1i16|1i32|1i64|1i8|2i32|4i16|8i8)$",
+  "ADDPv(2i32|4i16|8i8)$")>;
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "ABSv(2i64|4i32|8i16|16i8)$",
+  "[SU]ADDLPv(16i8_v8i16|4i32_v2i64|8i16_v4i32)$",
+  "([SU]QADD|[SU]QSUB|SQNEG|SUQADD|USQADD)v(16i8|2i64|4i32|8i16)$",
+  "ADDPv(16i8|2i64|4i32|8i16)$")>;
+// ASIMD arith #3
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex  "SADDLv", "UADDLv", "SADDWv",
+  "UADDWv", "SSUBLv", "USUBLv", "SSUBWv", "USUBWv", "ADDHNv", "SUBHNv")>;
+// ASIMD arith #5
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "RADDHNv", "RSUBHNv")>;
+// ASIMD arith, reduce
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex  "ADDVv", "SADDLVv", "UADDLVv")>;
+// ASIMD compare #1
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "CM(EQ|GE|GT|HI|HS|LE|LT)v(1i64|2i32|4i16|8i8)")>;
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "CM(EQ|GE|GT|HI|HS|LE|LT)v(2i64|4i32|8i16|16i8)")>;
+// ASIMD compare #2
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "CMTSTv(1i64|2i32|4i16|8i8)")>;
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "CMTSTv(2i64|4i32|8i16|16i8)")>;
+// ASIMD logical $1
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "(AND|EOR|NOT|ORN)v8i8",
+  "(ORR|BIC)v(2i32|4i16|8i8)$", "MVNIv(2i|2s|4i16)")>;
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "(AND|EOR|NOT|ORN)v16i8",
+  "(ORR|BIC)v(16i8|4i32|8i16)$", "MVNIv(4i32|4s|8i16)")>;
+// ASIMD max/min, basic
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "[SU](MIN|MAX)P?v(2i32|4i16|8i8)")>;
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "[SU](MIN|MAX)P?v(16i8|4i132|8i16)")>;
+// SIMD max/min, reduce
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "[SU](MAX|MIN)Vv")>;
+// ASIMD multiply, by element
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "MULv(2i32|4i16|4i32|8i16)_indexed$",
+  "SQR?DMULHv(1i16|1i32|2i32|4i16|4i32|8i16)_indexed$")>;
+// ASIMD multiply
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instrs PMULv8i8)>;
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instrs PMULv16i8)>;
+// ASIMD multiply accumulate
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "ML[AS]v(2i32|4i16|8i8)$")>;
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "ML[AS]v(16i8|4i32|8i16)$")>;
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "ML[AS]v(2i32|4i16|4i32|8i16)_indexed$")>;
+// ASIMD multiply accumulate half
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "SQRDML[AS]H[vi]")>;
+// ASIMD multiply accumulate long
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "[SU]ML[AS]Lv")>;
+// ASIMD multiply accumulate long #2
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "SQDML[AS]L[iv]")>;
+// ASIMD dot product
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "[SU]DOTv8i8")>;
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "[SU]DOTv16i8")>;
+// ASIMD dot product, by scalar
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "[SU]DOTlanev")>;
+// ASIMD multiply long
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "[SU]MULLv", "SQDMULL[iv]")>;
+// ASIMD polynomial (8x8) multiply long
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instrs PMULLv8i8, PMULLv16i8)>;
+// ASIMD pairwise add and accumulate
+def : InstRW<[CortexA510MCWrite<8, 2, CortexA510UnitVALU>], (instregex "[SU]ADALPv")>;
+// ASIMD shift accumulate
+def : InstRW<[CortexA510MCWrite<8, 2, CortexA510UnitVALU>], (instregex "[SU]SRA(d|v2i32|v4i16|v8i8)")>;
+def : InstRW<[CortexA510MCWrite<8, 2, CortexA510UnitVALU>], (instregex "[SU]SRAv(16i8|2i64|4i32|8i16)")>;
+// ASIMD shift accumulate #2
+def : InstRW<[CortexA510MCWrite<8, 2, CortexA510UnitVALU>], (instregex "[SU]RSRA[vd]")>;
+// ASIMD shift by immed
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "SHLd$", "SHLv",
+  "SLId$", "SRId$", "[SU]SHR[vd]", "SHRNv")>;
+// ASIMD shift by immed
+// SXTL and UXTL are aliases for SHLL
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "[US]?SHLLv")>;
+// ASIMD shift by immed #2
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "[SU]RSHR(d|v2i32|v4i16|v8i8)",
+  "[SU]RSHRv(16i8|2i64|4i32|8i16)")>;
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "RSHRNv(2i32|4i16|8i8)",
+  "RSHRNv(16i8|4i32|8i16)")>;
+// ASIMD shift by register
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "[SU]SHLv(1i64|2i32|4i16|8i8)")>;
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "[SU]SHLv(2i64|4i32|8i16|16i8)")>;
+// ASIMD shift by register #2
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "[SU]RSHLv(1i64|2i32|4i16|8i8)")>;
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "[SU]RSHLv(2i64|4i32|8i16|16i8)")>;
+
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "[SU]QSHLv(1i64|2i32|4i16|8i8)")>;
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "[SU]QSHLv(2i64|4i32|8i16|16i8)")>;
+
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "[SU]QRSHLv(1i64|2i32|4i16|8i8)")>;
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "[SU]QRSHLv(2i64|4i32|8i16|16i8)")>;
+
+// Cryptography extensions
+// -----------------------------------------------------------------------------
+
+// Crypto AES ops
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^AES[DE]rr$", "^AESI?MCrr")>;
+
+// Crypto polynomial (64x64) multiply long
+def : InstRW<[CortexA510MCWrite<8, 0, CortexA510UnitVMC>], (instrs PMULLv1i64, PMULLv2i64)>;
+
+// Crypto SHA1 hash acceleration op
+// Crypto SHA1 schedule acceleration ops
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^SHA1(H|SU0|SU1)")>;
+
+// Crypto SHA1 hash acceleration ops
+// Crypto SHA256 hash acceleration ops
+def : InstRW<[CortexA510MCWrite<8, 0, CortexA510UnitVMC>], (instregex "^SHA1[CMP]", "^SHA256H2?")>;
+
+// Crypto SHA256 schedule acceleration ops
+def : InstRW<[CortexA510MCWrite<8, 0, CortexA510UnitVMC>], (instregex "^SHA256SU[01]")>;
+
+// Crypto SHA512 hash acceleration ops
+def : InstRW<[CortexA510MCWrite<8, 0, CortexA510UnitVMC>], (instregex "^SHA512(H|H2|SU0|SU1)")>;
+
+// Crypto SHA3 ops
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instrs BCAX, EOR3, XAR)>;
+def : InstRW<[CortexA510MCWrite<8, 0, CortexA510UnitVMC>], (instrs RAX1)>;
+
+
+// Crypto SM3 ops
+def : InstRW<[CortexA510MCWrite<8, 0, CortexA510UnitVMC>], (instregex "^SM3PARTW[12]$", "^SM3SS1$",
+                                                            "^SM3TT[12][AB]$")>;
+
+// Crypto SM4 ops
+def : InstRW<[CortexA510MCWrite<8, 0, CortexA510UnitVMC>], (instrs SM4E, SM4ENCKEY)>;
+
+// CRC
+// -----------------------------------------------------------------------------
+
+def : InstRW<[CortexA510MCWrite<2, 0, CortexA510UnitMAC>], (instregex "^CRC32")>;
+
+// SVE Predicate instructions
+
+// Loop control, based on predicate
+def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instrs BRKA_PPmP, BRKA_PPzP,
+                                                  BRKB_PPmP, BRKB_PPzP)>;
+
+// Loop control, based on predicate and flag setting
+def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instrs BRKAS_PPzP, BRKBS_PPzP)>;
+
+// Loop control, propagating
+def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instrs BRKN_PPzP, BRKPA_PPzPP, BRKPB_PPzPP)>;
+
+// Loop control, propagating and flag setting
+def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instrs BRKNS_PPzP)>;
+def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instrs BRKPAS_PPzPP, BRKPBS_PPzPP)>;
+
+
+// Loop control, based on GPR
+def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>],
+             (instregex "^WHILE(GE|GT|HI|HS|LE|LO|LS|LT)_P(WW|XX)_[BHSD]$")>;
+
+def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instregex "^WHILE(RW|WR)_PXX_[BHSD]$")>;
+
+// Loop terminate
+def : InstRW<[CortexA510Write<1, CortexA510UnitALU>], (instregex "^CTERM(EQ|NE)_(WW|XX)$")>;
+
+// Predicate counting scalar
+def : InstRW<[CortexA510Write<1, CortexA510UnitALU>], (instrs ADDPL_XXI, ADDVL_XXI, RDVLI_XI)>;
+
+def : InstRW<[CortexA510Write<1, CortexA510UnitALU>],
+             (instregex "^CNT[BHWD]_XPiI$")>;
+
+def : InstRW<[CortexA510Write<1, CortexA510UnitALU>],
+             (instregex "^(INC|DEC)[BHWD]_XPiI$")>;
+
+def : InstRW<[CortexA510Write<1, CortexA510UnitALU>],
+             (instregex "^(SQINC|SQDEC|UQINC|UQDEC)[BHWD]_[XW]Pi(Wd)?I$")>;
+
+// Predicate counting scalar, active predicate
+def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>],
+             (instregex "^CNTP_XPP_[BHSD]$")>;
+
+def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>],
+             (instregex "^(DEC|INC)P_XP_[BHSD]$")>;
+
+def : InstRW<[CortexA510Write<8, CortexA510UnitVALU0>],
+             (instregex "^(SQDEC|SQINC|UQDEC|UQINC)P_XP_[BHSD]$",
+                        "^(UQDEC|UQINC)P_WP_[BHSD]$",
+                        "^(SQDEC|SQINC|UQDEC|UQINC)P_XPWd_[BHSD]$")>;
+
+
+// Predicate counting vector, active predicate
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>],
+             (instregex "^(DEC|INC|SQDEC|SQINC|UQDEC|UQINC)P_ZP_[HSD]$")>;
+
+// Predicate logical
+def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>],
+             (instregex "^(AND|BIC|EOR|NAND|NOR|ORN|ORR)_PPzPP$")>;
+
+// Predicate logical, flag setting
+def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>],
+             (instregex "^(ANDS|BICS|EORS|NANDS|NORS|ORNS|ORRS)_PPzPP$")>;
+
+// Predicate reverse
+def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instregex "^REV_PP_[BHSD]$")>;
+
+// Predicate select
+def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instrs SEL_PPPP)>;
+
+// Predicate set
+def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instregex "^PFALSE$", "^PTRUE_[BHSD]$")>;
+
+// Predicate set/initialize, set flags
+def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instregex "^PTRUES_[BHSD]$")>;
+
+// Predicate find first/next
+def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instregex "^PFIRST_B$", "^PNEXT_[BHSD]$")>;
+
+// Predicate test
+def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instrs PTEST_PP)>;
+
+// Predicate transpose
+def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instregex "^TRN[12]_PPP_[BHSDQ]$")>;
+
+// Predicate unpack and widen
+def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instrs PUNPKHI_PP, PUNPKLO_PP)>;
+
+// Predicate zip/unzip
+def : InstRW<[CortexA510Write<6, CortexA510UnitVALU0>], (instregex "^(ZIP|UZP)[12]_PPP_[BHSDQ]$")>;
+
+
+// SVE integer instructions
+// -----------------------------------------------------------------------------
+// Arithmetic, absolute diff
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^[SU]ABD_ZPmZ_[BHSD]$")>;
+
+// Arithmetic, absolute diff accum
+def : InstRW<[CortexA510MCWrite<8, 2, CortexA510UnitVALU>], (instregex "^[SU]ABA_ZZZ_[BHSD]$")>;
+
+// Arithmetic, absolute diff accum long
+def : InstRW<[CortexA510MCWrite<8, 2, CortexA510UnitVALU>], (instregex "^[SU]ABAL[TB]_ZZZ_[HSD]$")>;
+
+// Arithmetic, absolute diff long
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^[SU]ABDL[TB]_ZZZ_[HSD]$")>;
+
+// Arithmetic, basic
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>],
+             (instregex "^(ABS|CNOT|NEG)_ZPmZ_[BHSD]$",
+                        "^(ADD|SUB|SUBR)_ZPmZ_[BHSD]$",
+                        "^(ADD|SUB)_ZZZ_[BHSD]$",
+                        "^(ADD|SUB|SUBR)_ZI_[BHSD]$",
+                        "^ADR_[SU]XTW_ZZZ_D_[0123]$",
+                        "^ADR_LSL_ZZZ_[SD]_[0123]$",
+                        "^[SU](ADD|SUB)[LW][BT]_ZZZ_[HSD]$",
+                        "^SADDLBT_ZZZ_[HSD]$",
+                        "^[SU]H(ADD|SUB|SUBR)_ZPmZ_[BHSD]$",
+                        "^SSUBL(BT|TB)_ZZZ_[HSD]$")>;
+
+// Arithmetic, complex
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>],
+             (instregex "^R?(ADD|SUB)HN[BT]_ZZZ_[BHS]$",
+                        "^SQ(ABS|NEG)_ZPmZ_[BHSD]$",
+                        "^SQ(ADD|SUB|SUBR)_ZPmZ_?[BHSD]$",
+                        "^[SU]Q(ADD|SUB)_ZZZ_[BHSD]$",
+                        "^[SU]Q(ADD|SUB)_ZI_[BHSD]$",
+                        "^(SRH|SUQ|UQ|USQ|URH)ADD_ZPmZ_[BHSD]$",
+                        "^(UQSUB|UQSUBR)_ZPmZ_[BHSD]$")>;
+
+// Arithmetic, large integer
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^(AD|SB)CL[BT]_ZZZ_[SD]$")>;
+
+// Arithmetic, pairwise add
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^ADDP_ZPmZ_[BHSD]$")>;
+
+// Arithmetic, pairwise add and accum long
+def : InstRW<[CortexA510MCWrite<7, 2, CortexA510UnitVALU>], (instregex "^[SU]ADALP_ZPmZ_[HSD]$")>;
+
+// Arithmetic, shift
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>],
+             (instregex "^(ASR|LSL|LSR)_WIDE_ZPmZ_[BHS]$",
+                        "^(ASR|LSL|LSR)_WIDE_ZZZ_[BHS]$",
+                        "^(ASR|LSL|LSR)_ZPmI_[BHSD]$",
+                        "^(ASR|LSL|LSR)_ZPmZ_[BHSD]$",
+                        "^(ASR|LSL|LSR)_ZZI_[BHSD]$",
+                        "^(ASRR|LSLR|LSRR)_ZPmZ_[BHSD]$")>;
+
+// Arithmetic, shift and accumulate
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>],
+             (instregex "^(SSRA|USRA)_ZZI_[BHSD]$")>;
+
+def : InstRW<[CortexA510MCWrite<7, 2, CortexA510UnitVALU>],
+             (instregex "^(SRSRA|URSRA)_ZZI_[BHSD]$")>;
+
+
+// Arithmetic, shift by immediate
+// Arithmetic, shift by immediate and insert
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>],
+             (instregex "^(SHRNB|SHRNT|SSHLLB|SSHLLT|USHLLB|USHLLT|SLI|SRI)_ZZI_[BHSD]$")>;
+
+// Arithmetic, shift complex
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>],
+             (instregex "^(SQ)?RSHRU?N[BT]_ZZI_[BHS]$",
+                        "^(SQRSHL|SQRSHLR|SQSHL|SQSHLR|UQRSHL|UQRSHLR|UQSHL|UQSHLR)_ZPmZ_[BHSD]$",
+                        "^(SQSHL|SQSHLU|UQSHL)_ZPmI_[BHSD]$",
+                        "^SQSHRU?N[BT]_ZZI_[BHS]$",
+                        "^UQR?SHRN[BT]_ZZI_[BHS]$")>;
+
+// Arithmetic, shift right for divide
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^ASRD_ZPmI_[BHSD]$")>;
+
+// Arithmetic, shift rounding
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>],
+             (instregex "^(SRSHL|SRSHLR|URSHL|URSHLR)_ZPmZ_[BHSD]$",
+                        "^[SU]RSHR_ZPmI_[BHSD]$")>;
+
+// Bit manipulation
+def : InstRW<[CortexA510MCWrite<14, 13, CortexA510UnitVMC>],
+             (instregex "^(BDEP|BEXT|BGRP)_ZZZ_B$")>;
+
+def : InstRW<[CortexA510MCWrite<22, 21, CortexA510UnitVMC>],
+             (instregex "^(BDEP|BEXT|BGRP)_ZZZ_H$")>;
+
+def : InstRW<[CortexA510MCWrite<38, 37, CortexA510UnitVMC>],
+             (instregex "^(BDEP|BEXT|BGRP)_ZZZ_S$")>;
+
+def : InstRW<[CortexA510MCWrite<70, 69, CortexA510UnitVMC>],
+             (instregex "^(BDEP|BEXT|BGRP)_ZZZ_D$")>;
+
+
+// Bitwise select
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^(BSL|BSL1N|BSL2N|NBSL)_ZZZZ$")>;
+
+// Count/reverse bits
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^(CLS|CLZ|RBIT)_ZPmZ_[BHSD]$")>;
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^CNT_ZPmZ_[BH]$")>;
+def : InstRW<[CortexA510Write<8, CortexA510UnitVALU>], (instregex "^CNT_ZPmZ_S$")>;
+def : InstRW<[CortexA510Write<12, CortexA510UnitVALU>], (instregex "^CNT_ZPmZ_D$")>;
+// Broadcast logical bitmask immediate to vector
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instrs DUPM_ZI)>;
+
+// Compare and set flags
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>],
+             (instregex "^CMP(EQ|GE|GT|HI|HS|LE|LO|LS|LT|NE)_PPzZ[IZ]_[BHSD]$",
+                        "^CMP(EQ|GE|GT|HI|HS|LE|LO|LS|LT|NE)_WIDE_PPzZZ_[BHS]$")>;
+
+// Complex add
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^CADD_ZZI_[BHSD]$")>;
+
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^SQCADD_ZZI_[BHSD]$")>;
+
+// Complex dot product 8-bit element
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instrs CDOT_ZZZ_S, CDOT_ZZZI_S)>;
+
+// Complex dot product 16-bit element
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instrs CDOT_ZZZ_D, CDOT_ZZZI_D)>;
+
+// Complex multiply-add B, H, S element size
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^CMLA_ZZZ_[BHS]$",
+                                            "^CMLA_ZZZI_[HS]$")>;
+
+// Complex multiply-add D element size
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instrs CMLA_ZZZ_D)>;
+
+// Conditional extract operations, scalar form
+def : InstRW<[CortexA510MCWrite<8, 2, CortexA510UnitVALU>], (instregex "^CLAST[AB]_RPZ_[BHSD]$")>;
+
+// Conditional extract operations, SIMD&FP scalar and vector forms
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^CLAST[AB]_[VZ]PZ_[BHSD]$",
+                                            "^COMPACT_ZPZ_[SD]$",
+                                            "^SPLICE_ZPZZ?_[BHSD]$")>;
+
+// Convert to floating point, 64b to float or convert to double
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^[SU]CVTF_ZPmZ_Dto[SD]")>;
+
+// Convert to floating point, 64b to half
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^[SU]CVTF_ZPmZ_DtoH")>;
+
+// Convert to floating point, 32b to single or half
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^[SU]CVTF_ZPmZ_Sto[HS]")>;
+
+// Convert to floating point, 32b to double
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^[SU]CVTF_ZPmZ_StoD")>;
+
+// Convert to floating point, 16b to half
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^[SU]CVTF_ZPmZ_HtoH")>;
+
+// Copy, scalar
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU0>],(instregex "^CPY_ZPmR_[BHSD]$")>;
+
+// Copy, scalar SIMD&FP or imm
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^CPY_ZPm[IV]_[BHSD]$",
+                                           "^CPY_ZPzI_[BHSD]$")>;
+
+// Divides, 32 bit
+def : InstRW<[CortexA510MCWrite<15, 12, CortexA510UnitVMC>], (instregex "^[SU]DIVR?_ZPmZ_S$")>;
+
+// Divides, 64 bit
+def : InstRW<[CortexA510MCWrite<26, 23, CortexA510UnitVMC>], (instregex "^[SU]DIVR?_ZPmZ_D$")>;
+
+// Dot product, 8 bit
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^[SU]DOT_ZZZI?_S$")>;
+
+// Dot product, 8 bit, using signed and unsigned integers
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instrs SUDOT_ZZZI, USDOT_ZZZI, USDOT_ZZZ)>;
+
+// Dot product, 16 bit
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^[SU]DOT_ZZZI?_D$")>;
+
+// Duplicate, immediate and indexed form
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^DUP_ZI_[BHSD]$",
+                                           "^DUP_ZZI_[BHSDQ]$")>;
+
+// Duplicate, scalar form
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^DUP_ZR_[BHSD]$")>;
+
+// Extend, sign or zero
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^[SU]XTB_ZPmZ_[HSD]$",
+                                            "^[SU]XTH_ZPmZ_[SD]$",
+                                            "^[SU]XTW_ZPmZ_[D]$")>;
+
+// Extract
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instrs EXT_ZZI, EXT_ZZI_B)>;
+
+// Extract narrow saturating
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^[SU]QXTN[BT]_ZZ_[BHS]$",
+                                            "^SQXTUN[BT]_ZZ_[BHS]$")>;
+
+// Extract/insert operation, SIMD and FP scalar form
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^LAST[AB]_VPZ_[BHSD]$",
+                                            "^INSR_ZV_[BHSD]$")>;
+
+// Extract/insert operation, scalar
+def : InstRW<[CortexA510MCWrite<8, 2, CortexA510UnitVALU0>], (instregex "^LAST[AB]_RPZ_[BHSD]$",
+                                                "^INSR_ZR_[BHSD]$")>;
+
+// Histogram operations
+def : InstRW<[CortexA510MCWrite<8, 2, CortexA510UnitVALU0>], (instregex "^HISTCNT_ZPzZZ_[SD]$",
+                                                  "^HISTSEG_ZZZ$")>;
+
+// Horizontal operations, B, H, S form, immediate operands only
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^INDEX_II_[BHS]$")>;
+
+// Horizontal operations, B, H, S form, scalar, immediate operands/ scalar
+// operands only / immediate, scalar operands
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^INDEX_(IR|RI|RR)_[BHS]$")>;
+
+// Horizontal operations, D form, immediate operands only
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instrs INDEX_II_D)>;
+
+// Horizontal operations, D form, scalar, immediate operands)/ scalar operands
+// only / immediate, scalar operands
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^INDEX_(IR|RI|RR)_D$")>;
+
+// Logical
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>],
+             (instregex "^(AND|EOR|ORR)_ZI$",
+                        "^(AND|BIC|EOR|EOR(BT|TB)?|ORR)_ZZZ$",
+                        "^(AND|BIC|EOR|NOT|ORR)_ZPmZ_[BHSD]$")>;
+
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>],
+             (instregex "^EOR(BT|TB)_ZZZ_[BHSD]$")>;
+
+// Max/min, basic and pairwise
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^[SU](MAX|MIN)_ZI_[BHSD]$",
+                                           "^[SU](MAX|MIN)P?_ZPmZ_[BHSD]$")>;
+
+// Matching operations
+def : InstRW<[CortexA510MCWrite<7, 2, CortexA510UnitVALU>], (instregex "^N?MATCH_PPzZZ_[BH]$")>;
+
+// Matrix multiply-accumulate
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instrs SMMLA_ZZZ, UMMLA_ZZZ, USMMLA_ZZZ)>;
+
+// Move prefix
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^MOVPRFX_ZP[mz]Z_[BHSD]$",
+                                           "^MOVPRFX_ZZ$")>;
+
+// Multiply, B, H, S element size
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^MUL_(ZI|ZPmZ|ZZZI|ZZZ)_[BHS]$",
+                                            "^[SU]MULH_(ZPmZ|ZZZ)_[BHS]$")>;
+
+// Multiply, D element size
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^MUL_(ZI|ZPmZ|ZZZI|ZZZ)_D$",
+                                            "^[SU]MULH_(ZPmZ|ZZZ)_D$")>;
+
+// Multiply long
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^[SU]MULL[BT]_ZZZI_[SD]$",
+                                            "^[SU]MULL[BT]_ZZZ_[HSD]$")>;
+
+// Multiply accumulate, B, H, S element size
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^ML[AS]_ZZZI_[BHS]$",
+                                            "^(ML[AS]|MAD|MSB)_ZPmZZ_[BHS]$")>;
+
+// Multiply accumulate, D element size
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^ML[AS]_ZZZI_D$",
+                                            "^(ML[AS]|MAD|MSB)_ZPmZZ_D$")>;
+
+// Multiply accumulate long
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^[SU]ML[AS]L[BT]_ZZZ_[HSD]$",
+                                            "^[SU]ML[AS]L[BT]_ZZZI_[SD]$")>;
+
+// Multiply accumulate saturating doubling long regular
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^SQDML[AS](LB|LT|LBT)_ZZZ_[HSD]$",
+                                            "^SQDML[AS](LB|LT)_ZZZI_[SD]$")>;
+
+// Multiply saturating doubling high, B, H, S element size
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^SQDMULH_ZZZ_[BHS]$",
+                                            "^SQDMULH_ZZZI_[HS]$")>;
+
+// Multiply saturating doubling high, D element size
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instrs SQDMULH_ZZZ_D, SQDMULH_ZZZI_D)>;
+
+// Multiply saturating doubling long
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^SQDMULL[BT]_ZZZ_[HSD]$",
+                                            "^SQDMULL[BT]_ZZZI_[SD]$")>;
+
+// Multiply saturating rounding doubling regular/complex accumulate, B, H, S
+// element size
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^SQRDML[AS]H_ZZZ_[BHS]$",
+                                            "^SQRDCMLAH_ZZZ_[BHS]$",
+                                            "^SQRDML[AS]H_ZZZI_[HS]$",
+                                            "^SQRDCMLAH_ZZZI_[HS]$")>;
+
+// Multiply saturating rounding doubling regular/complex accumulate, D element
+// size
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^SQRDML[AS]H_ZZZI?_D$",
+                                            "^SQRDCMLAH_ZZZ_D$")>;
+
+// Multiply saturating rounding doubling regular/complex, B, H, S element size
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^SQRDMULH_ZZZ_[BHS]$",
+                                            "^SQRDMULH_ZZZI_[HS]$")>;
+
+// Multiply saturating rounding doubling regular/complex, D element size
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^SQRDMULH_ZZZI?_D$")>;
+
+// Multiply/multiply long, (8x8) polynomial
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^PMUL_ZZZ_B$")>;
+
+def : InstRW<[CortexA510Write<6, CortexA510UnitVMC>], (instregex "^PMULL[BT]_ZZZ_[HDQ]$")>;
+
+
+// Predicate counting vector
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>],
+             (instregex "^(DEC|INC|SQDEC|SQINC|UQDEC|UQINC)[HWD]_ZPiI$")>;
+
+// Reciprocal estimate
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instrs URECPE_ZPmZ_S, URSQRTE_ZPmZ_S)>;
+
+// Reduction, arithmetic, B form
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU0>], (instregex "^[SU](ADD|MAX|MIN)V_VPZ_B")>;
+
+// Reduction, arithmetic, H form
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU0>], (instregex "^[SU](ADD|MAX|MIN)V_VPZ_H")>;
+
+// Reduction, arithmetic, S form
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU0>], (instregex "^[SU](ADD|MAX|MIN)V_VPZ_S")>;
+
+// Reduction, arithmetic, D form
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU0>], (instregex "^[SU](ADD|MAX|MIN)V_VPZ_D")>;
+
+// Reduction, logical
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU0>], (instregex "^(ANDV|EORV|ORV)_VPZ_[BHSD]$")>;
+
+// Reverse, vector
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^REV_ZZ_[BHSD]$",
+                                           "^REVB_ZPmZ_[HSD]$",
+                                           "^REVH_ZPmZ_[SD]$",
+                                           "^REVW_ZPmZ_D$")>;
+
+// Select, vector form
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^SEL_ZPZZ_[BHSD]$")>;
+
+// Table lookup
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^TBL_ZZZZ?_[BHSD]$")>;
+
+// Table lookup extension
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^TBX_ZZZ_[BHSD]$")>;
+
+// Transpose, vector form
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^TRN[12]_ZZZ_[BHSDQ]$")>;
+
+// Unpack and extend
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^[SU]UNPK(HI|LO)_ZZ_[HSD]$")>;
+
+// Zip/unzip
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^(UZP|ZIP)[12]_ZZZ_[BHSDQ]$")>;
+
+// SVE floating-point instructions
+// -----------------------------------------------------------------------------
+
+// Floating point absolute value/difference
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^FAB[SD]_ZPmZ_[HSD]$")>;
+
+// Floating point arithmetic
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^F(ADD|SUB)_(ZPm[IZ]|ZZZ)_[HSD]$",
+                                           "^FADDP_ZPmZZ_[HSD]$",
+                                           "^FNEG_ZPmZ_[HSD]$",
+                                           "^FSUBR_ZPm[IZ]_[HSD]$")>;
+
+// Floating point associative add, F16
+def : InstRW<[CortexA510MCWrite<32, 29, CortexA510UnitVALU>], (instrs FADDA_VPZ_H)>;
+
+// Floating point associative add, F32
+def : InstRW<[CortexA510MCWrite<16, 13, CortexA510UnitVALU>], (instrs FADDA_VPZ_S)>;
+
+// Floating point associative add, F64
+def : InstRW<[CortexA510MCWrite<8, 5, CortexA510UnitVALU>], (instrs FADDA_VPZ_D)>;
+
+// Floating point compare
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^FACG[ET]_PPzZZ_[HSD]$",
+                                            "^FCM(EQ|GE|GT|NE)_PPzZ[0Z]_[HSD]$",
+                                            "^FCM(LE|LT)_PPzZ0_[HSD]$",
+                                            "^FCMUO_PPzZZ_[HSD]$")>;
+
+// Floating point complex add
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^FCADD_ZPmZ_[HSD]$")>;
+
+// Floating point complex multiply add
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^FCMLA_ZPmZZ_[HSD]$",
+                                           "^FCMLA_ZZZI_[HS]$")>;
+
+// Floating point convert, long or narrow (F16 to F32 or F32 to F16)
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^FCVT_ZPmZ_(HtoS|StoH)",
+                                            "^FCVTLT_ZPmZ_HtoS",
+                                            "^FCVTNT_ZPmZ_StoH")>;
+
+// Floating point convert, long or narrow (F16 to F64, F32 to F64, F64 to F32
+// or F64 to F16)
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^FCVT_ZPmZ_(HtoD|StoD|DtoS|DtoH)",
+                                            "^FCVTLT_ZPmZ_StoD",
+                                            "^FCVTNT_ZPmZ_DtoS")>;
+
+// Floating point convert, round to odd
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^FCVTX_ZPmZ_DtoS", "FCVTXNT_ZPmZ_DtoS")>;
+
+// Floating point base2 log, F16
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instrs FLOGB_ZPmZ_H)>;
+
+// Floating point base2 log, F32
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instrs FLOGB_ZPmZ_S)>;
+
+// Floating point base2 log, F64
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instrs FLOGB_ZPmZ_D)>;
+
+// Floating point convert to integer, F16
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^FCVTZ[SU]_ZPmZ_HtoH")>;
+
+// Floating point convert to integer, F32
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^FCVTZ[SU]_ZPmZ_(HtoS|StoS)")>;
+
+// Floating point convert to integer, F64
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>],
+             (instregex "^FCVTZ[SU]_ZPmZ_(HtoD|StoD|DtoS|DtoD)")>;
+
+// Floating point copy
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU0>], (instregex "^FCPY_ZPmI_[HSD]$",
+                                           "^FDUP_ZI_[HSD]$")>;
+
+// Floating point divide, F16
+def : InstRW<[CortexA510MCWrite<8, 5, CortexA510UnitVMC>], (instregex "^FDIVR?_ZPmZ_H$")>;
+
+// Floating point divide, F32
+def : InstRW<[CortexA510MCWrite<13, 10, CortexA510UnitVMC>], (instregex "^FDIVR?_ZPmZ_S$")>;
+
+// Floating point divide, F64
+def : InstRW<[CortexA510MCWrite<22, 19, CortexA510UnitVMC>], (instregex "^FDIVR?_ZPmZ_D$")>;
+
+// Floating point min/max pairwise
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^F(MAX|MIN)(NM)?P_ZPmZZ_[HSD]$")>;
+
+// Floating point min/max
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^F(MAX|MIN)(NM)?_ZPm[IZ]_[HSD]$")>;
+
+// Floating point multiply
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^(FSCALE|FMULX)_ZPmZ_[HSD]$",
+                                           "^FMUL_(ZPm[IZ]|ZZZI?)_[HSD]$")>;
+
+// Floating point multiply accumulate
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>],
+             (instregex "^FML[AS]_(ZPmZZ|ZZZI)_[HSD]$",
+                        "^(FMAD|FNMAD|FNML[AS]|FN?MSB)_ZPmZZ_[HSD]$")>;
+
+// Floating point multiply add/sub accumulate long
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^FML[AS]L[BT]_ZZZI?_SHH$")>;
+
+// Floating point reciprocal estimate, F16
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instrs FRECPE_ZZ_H, FRECPX_ZPmZ_H,
+                                         FRSQRTE_ZZ_H)>;
+
+// Floating point reciprocal estimate, F32
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instrs FRECPE_ZZ_S, FRECPX_ZPmZ_S,
+                                         FRSQRTE_ZZ_S)>;
+
+// Floating point reciprocal estimate, F64
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instrs FRECPE_ZZ_D, FRECPX_ZPmZ_D,
+                                         FRSQRTE_ZZ_D)>;
+
+// Floating point reciprocal step
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^F(RECPS|RSQRTS)_ZZZ_[HSD]$")>;
+
+// Floating point reduction, F16
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU0>],
+             (instregex "^(FMAXNMV|FMAXV|FMINNMV|FMINV)_VPZ_[HSD]$")>;
+
+// Floating point reduction, F32
+def : InstRW<[CortexA510MCWrite<12, 11, CortexA510UnitVALU0>],
+             (instregex "^FADDV_VPZ_H$")>;
+
+def : InstRW<[CortexA510MCWrite<8, 5, CortexA510UnitVALU0>],
+             (instregex "^FADDV_VPZ_S$")>;
+
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU0>],
+             (instregex "^FADDV_VPZ_D$")>;
+
+
+// Floating point round to integral, F16
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^FRINT[AIMNPXZ]_ZPmZ_H$")>;
+
+// Floating point round to integral, F32
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^FRINT[AIMNPXZ]_ZPmZ_S$")>;
+
+// Floating point round to integral, F64
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^FRINT[AIMNPXZ]_ZPmZ_D$")>;
+
+// Floating point square root, F16
+def : InstRW<[CortexA510MCWrite<8, 5, CortexA510UnitVMC>], (instrs FSQRT_ZPmZ_H)>;
+
+// Floating point square root, F32
+def : InstRW<[CortexA510MCWrite<12, 9, CortexA510UnitVMC>], (instrs FSQRT_ZPmZ_S)>;
+
+// Floating point square root, F64
+def : InstRW<[CortexA510MCWrite<22, 19, CortexA510UnitVMC>], (instrs FSQRT_ZPmZ_D)>;
+
+// Floating point trigonometric exponentiation
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^FEXPA_ZZ_[HSD]$")>;
+
+// Floating point trigonometric multiply add
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^FTMAD_ZZI_[HSD]$")>;
+
+// Floating point trigonometric, miscellaneous
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^FTSMUL_ZZZ_[HSD]$")>;
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instregex "^FTSSEL_ZZZ_[HSD]$")>;
+
+
+// SVE BFloat16 (BF16) instructions
+// -----------------------------------------------------------------------------
+
+// Convert, F32 to BF16
+def : InstRW<[CortexA510Write<4, CortexA510UnitVALU>], (instrs BFCVT_ZPmZ, BFCVTNT_ZPmZ)>;
+
+// Dot product
+def : InstRW<[A510Write_10cyc_1VMAC_1VALU], (instrs BFDOT_ZZI, BFDOT_ZZZ)>;
+
+// Matrix multiply accumulate
+def : InstRW<[A510Write_15cyc_1VMAC_1VALU], (instrs BFMMLA_ZZZ)>;
+
+// Multiply accumulate long
+def : InstRW<[CortexA510Write<4, CortexA510UnitVMAC>], (instregex "^BFMLAL[BT]_ZZZ(I)?$")>;
+
+// SVE Load instructions
+// -----------------------------------------------------------------------------
+
+// Load vector
+def : InstRW<[CortexA510Write<3, CortexA510UnitLd>], (instrs LDR_ZXI)>;
+
+// Load predicate
+def : InstRW<[CortexA510Write<3, CortexA510UnitLdSt>], (instrs LDR_PXI)>;
+
+// Contiguous load, scalar + imm
+def : InstRW<[CortexA510Write<3, CortexA510UnitLd>], (instregex "^LD1[BHWD]_IMM_REAL$",
+                                           "^LD1S?B_[HSD]_IMM_REAL$",
+                                           "^LD1S?H_[SD]_IMM_REAL$",
+                                           "^LD1S?W_D_IMM_REAL$" )>;
+// Contiguous load, scalar + scalar
+def : InstRW<[CortexA510Write<3, CortexA510UnitLd>], (instregex "^LD1[BHWD]$",
+                                             "^LD1S?B_[HSD]$",
+                                             "^LD1S?H_[SD]$",
+                                             "^LD1S?W_D$" )>;
+
+// Contiguous load broadcast, scalar + imm
+def : InstRW<[CortexA510Write<3, CortexA510UnitLd>], (instregex "^LD1R[BHWD]_IMM$",
+                                           "^LD1RSW_IMM$",
+                                           "^LD1RS?B_[HSD]_IMM$",
+                                           "^LD1RS?H_[SD]_IMM$",
+                                           "^LD1RS?W_D_IMM$",
+                                           "^LD1RQ_[BHWD]_IMM$")>;
+
+// Contiguous load broadcast, scalar + scalar
+def : InstRW<[CortexA510Write<3, CortexA510UnitLdSt>], (instregex "^LD1RQ_[BHWD]$")>;
+
+// Non temporal load, scalar + imm
+def : InstRW<[CortexA510Write<3, CortexA510UnitLdSt>], (instregex "^LDNT1[BHWD]_ZRI$")>;
+
+// Non temporal load, scalar + scalar
+def : InstRW<[CortexA510Write<3, CortexA510UnitLdSt>], (instregex "^LDNT1[BHWD]_ZRR$")>;
+
+// Non temporal gather load, vector + scalar 32-bit element size
+def : InstRW<[CortexA510MCWrite<9, 9, CortexA510UnitLdSt>], (instregex "^LDNT1[BHW]_ZZR_S_REAL$",
+                                              "^LDNT1S[BH]_ZZR_S_REAL$")>;
+
+// Non temporal gather load, vector + scalar 64-bit element size
+def : InstRW<[CortexA510MCWrite<7, 7, CortexA510UnitLdSt>], (instregex "^LDNT1S?[BHW]_ZZR_D_REAL$")>;
+def : InstRW<[CortexA510MCWrite<7, 7, CortexA510UnitLdSt>], (instrs LDNT1D_ZZR_D_REAL)>;
+
+// Contiguous first faulting load, scalar + scalar
+def : InstRW<[CortexA510Write<3, CortexA510UnitLd>], (instregex "^LDFF1[BHWD]_REAL$",
+                                              "^LDFF1S?B_[HSD]_REAL$",
+                                              "^LDFF1S?H_[SD]_REAL$",
+                                              "^LDFF1S?W_D_REAL$")>;
+
+// Contiguous non faulting load, scalar + imm
+def : InstRW<[CortexA510Write<3, CortexA510UnitLd>], (instregex "^LDNF1[BHWD]_IMM_REAL$",
+                                           "^LDNF1S?B_[HSD]_IMM_REAL$",
+                                           "^LDNF1S?H_[SD]_IMM_REAL$",
+                                           "^LDNF1S?W_D_IMM_REAL$")>;
+
+// Contiguous Load two structures to two vectors, scalar + imm
+def : InstRW<[CortexA510MCWrite<3, 1, CortexA510UnitLdSt>], (instregex "^LD2[BHWD]_IMM$")>;
+
+// Contiguous Load two structures to two vectors, scalar + scalar
+def : InstRW<[CortexA510MCWrite<3, 2, CortexA510UnitLdSt>], (instregex "^LD2[BHWD]$")>;
+
+// Contiguous Load three structures to three vectors, scalar + imm
+def : InstRW<[CortexA510MCWrite<5, 3, CortexA510UnitLdSt>], (instregex "^LD3[BHWD]_IMM$")>;
+
+// Contiguous Load three structures to three vectors, scalar + scalar
+def : InstRW<[CortexA510MCWrite<5, 3, CortexA510UnitLdSt>], (instregex "^LD3[BHWD]$")>;
+
+// Contiguous Load four structures to four vectors, scalar + imm
+def : InstRW<[CortexA510MCWrite<5, 3, CortexA510UnitLdSt>], (instregex "^LD4[BHWD]_IMM$")>;
+
+// Contiguous Load four structures to four vectors, scalar + scalar
+def : InstRW<[CortexA510MCWrite<5, 3, CortexA510UnitLdSt>], (instregex "^LD4[BHWD]$")>;
+
+// Gather load, vector + imm, 32-bit element size
+def : InstRW<[CortexA510MCWrite<9, 9, CortexA510UnitLdSt>], (instregex "^GLD(FF)?1S?[BH]_S_IMM_REAL$",
+                                              "^GLD(FF)?1W_IMM_REAL$")>;
+
+// Gather load, vector + imm, 64-bit element size
+def : InstRW<[CortexA510MCWrite<7, 7, CortexA510UnitLdSt>], (instregex "^GLD(FF)?1S?[BHW]_D_IMM_REAL$",
+                                              "^GLD(FF)?1D_IMM_REAL$")>;
+
+// Gather load, 64-bit element size
+def : InstRW<[CortexA510MCWrite<7, 7, CortexA510UnitLdSt>],
+             (instregex "^GLD(FF)?1S?[BHW]_D_[SU]XTW_(SCALED_)?REAL$",
+                        "^GLD(FF)?1S?[BHW]_D_(SCALED_)?REAL$",
+                        "^GLD(FF)?1D_[SU]XTW_(SCALED_)?REAL$",
+                        "^GLD(FF)?1D_(SCALED_)?REAL$")>;
+
+// Gather load, 32-bit scaled offset
+def : InstRW<[CortexA510MCWrite<9, 9, CortexA510UnitLd>],
+             (instregex "^GLD(FF)?1S?[HW]_S_[SU]XTW_SCALED_REAL$",
+                        "^GLD(FF)?1W_[SU]XTW_SCALED_REAL")>;
+
+// Gather load, 32-bit unpacked unscaled offset
+def : InstRW<[CortexA510MCWrite<9, 9, CortexA510UnitLd>], (instregex "^GLD(FF)?1S?[BH]_S_[SU]XTW_REAL$",
+                                              "^GLD(FF)?1W_[SU]XTW_REAL$")>;
+
+def : InstRW<[CortexA510Write<0, CortexA510UnitVALU>], (instregex "^PRF(B|H|W|D).*")>;
+// SVE Store instructions
+// -----------------------------------------------------------------------------
+
+// Store from predicate reg
+def : InstRW<[CortexA510VSt0], (instrs STR_PXI)>;
+
+// Store from vector reg
+def : InstRW<[CortexA510VSt0], (instrs STR_ZXI)>;
+
+// Contiguous store, scalar + imm
+def : InstRW<[CortexA510VSt0], (instregex "^ST1[BHWD]_IMM$",
+                                                "^ST1B_[HSD]_IMM$",
+                                                "^ST1H_[SD]_IMM$",
+                                                "^ST1W_D_IMM$")>;
+
+// Contiguous store, scalar + scalar
+def : InstRW<[CortexA510VSt0], (instregex "^ST1H(_[SD])?$")>;
+def : InstRW<[CortexA510VSt0], (instregex "^ST1[BWD]$",
+                                                "^ST1B_[HSD]$",
+                                                "^ST1W_D$")>;
+
+// Contiguous store two structures from two vectors, scalar + imm
+def : InstRW<[CortexA510VSt<11>], (instregex "^ST2[BHWD]_IMM$")>;
+
+// Contiguous store two structures from two vectors, scalar + scalar
+def : InstRW<[CortexA510VSt<11>], (instrs ST2H)>;
+
+// Contiguous store two structures from two vectors, scalar + scalar
+def : InstRW<[CortexA510VSt<11>], (instregex "^ST2[BWD]$")>;
+
+// Contiguous store three structures from three vectors, scalar + imm
+def : InstRW<[CortexA510VSt<25>], (instregex "^ST3[BHW]_IMM$")>;
+def : InstRW<[CortexA510VSt<14>], (instregex "^ST3D_IMM$")>;
+
+// Contiguous store three structures from three vectors, scalar + scalar
+def : InstRW<[CortexA510VSt<25>], (instregex "^ST3[BHW]$")>;
+def : InstRW<[CortexA510VSt<14>], (instregex "^ST3D$")>;
+
+// Contiguous store four structures from four vectors, scalar + imm
+def : InstRW<[CortexA510VSt<50>], (instregex "^ST4[BHW]_IMM$")>;
+def : InstRW<[CortexA510VSt<25>], (instregex "^ST4D_IMM$")>;
+
+// Contiguous store four structures from four vectors, scalar + scalar
+def : InstRW<[CortexA510VSt<50>], (instregex "^ST4[BHW]$")>;
+
+// Contiguous store four structures from four vectors, scalar + scalar
+def : InstRW<[CortexA510VSt<25>], (instregex "^ST4D$")>;
+
+// Non temporal store, scalar + imm
+def : InstRW<[CortexA510VSt0], (instregex "^STNT1[BHWD]_ZRI$")>;
+
+// Non temporal store, scalar + scalar
+def : InstRW<[CortexA510VSt0], (instrs STNT1H_ZRR)>;
+def : InstRW<[CortexA510VSt0], (instregex "^STNT1[BWD]_ZRR$")>;
+
+// Scatter non temporal store, vector + scalar 32-bit element size
+def : InstRW<[CortexA510VSt<9>], (instregex "^STNT1[BHW]_ZZR_S")>;
+
+// Scatter non temporal store, vector + scalar 64-bit element size
+def : InstRW<[CortexA510VSt<7>], (instregex "^STNT1[BHWD]_ZZR_D")>;
+
+// Scatter store vector + imm 32-bit element size
+def : InstRW<[CortexA510VSt<9>], (instregex "^SST1[BH]_S_IMM$",
+                                                "^SST1W_IMM$")>;
+
+// Scatter store vector + imm 64-bit element size
+def : InstRW<[CortexA510VSt<7>], (instregex "^SST1[BHW]_D_IMM$",
+                                                "^SST1D_IMM$")>;
+
+// Scatter store, 32-bit scaled offset
+def : InstRW<[CortexA510VSt<8>],
+             (instregex "^SST1(H_S|W)_[SU]XTW_SCALED$")>;
+
+// Scatter store, 32-bit unpacked unscaled offset
+def : InstRW<[CortexA510VSt<8>], (instregex "^SST1[BHW]_D_[SU]XTW$",
+                                                "^SST1D_[SU]XTW$")>;
+
+// Scatter store, 32-bit unpacked scaled offset
+def : InstRW<[CortexA510VSt<8>], (instregex "^SST1[HW]_D_[SU]XTW_SCALED$",
+                                                "^SST1D_[SU]XTW_SCALED$")>;
+
+// Scatter store, 32-bit unscaled offset
+def : InstRW<[CortexA510VSt<8>], (instregex "^SST1[BH]_S_[SU]XTW$",
+                                                "^SST1W_[SU]XTW$")>;
+
+// Scatter store, 64-bit scaled offset
+def : InstRW<[CortexA510VSt<8>], (instregex "^SST1[HW]_D_SCALED$",
+                                                "^SST1D_SCALED$")>;
+
+// Scatter store, 64-bit unscaled offset
+def : InstRW<[CortexA510VSt<8>], (instregex "^SST1[BHW]_D$",
+                                                "^SST1D$")>;
+
+// SVE Miscellaneous instructions
+// -----------------------------------------------------------------------------
+
+// Read first fault register, unpredicated
+def : InstRW<[CortexA510Write<1, CortexA510UnitALU>], (instrs RDFFR_P_REAL)>;
+
+// Read first fault register, predicated
+def : InstRW<[CortexA510Write<3, CortexA510UnitALU0>], (instrs RDFFR_PPz_REAL)>;
+
+// Read first fault register and set flags
+def : InstRW<[CortexA510Write<3, CortexA510UnitALU0>], (instrs RDFFRS_PPz)>;
+
+// Set first fault register
+// Write to first fault register
+def : InstRW<[CortexA510Write<1, CortexA510UnitALU>], (instrs SETFFR, WRFFR)>;
+
+// SVE Cryptographic instructions
+// -----------------------------------------------------------------------------
+
+// Crypto AES ops
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^AES[DE]_ZZZ_B$",
+                                           "^AESI?MC_ZZ_B$")>;
+
+// Crypto SHA3 ops
+def : InstRW<[CortexA510Write<3, CortexA510UnitVALU>], (instregex "^(BCAX|EOR3)_ZZZZ$",
+                                            "^XAR_ZZZI_[BHSD]$")>;
+
+def : InstRW<[CortexA510MC_RC0Write<8, CortexA510UnitVMC>], (instregex "^RAX1_ZZZ_D$")>;
+
+// Crypto SM4 ops
+def : InstRW<[CortexA510MC_RC0Write<8, CortexA510UnitVMC>], (instregex "^SM4E(KEY)?_ZZZ_S$")>;
+
+}
diff --git a/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-basic-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-basic-instructions.s
new file mode 100644
index 0000000000000..1edb0b8ad21e0
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-basic-instructions.s
@@ -0,0 +1,3726 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=aarch64 -mcpu=cortex-a510 -instruction-tables < %s | FileCheck %s
+
+#------------------------------------------------------------------------------
+# Add/sub (immediate)
+#------------------------------------------------------------------------------
+
+add      w2, w3, #4095
+add      w30, w29, #1, lsl #12
+add      w13, w5, #4095, lsl #12
+add      x5, x7, #1638
+add      w20, wsp, #801
+add      wsp, wsp, #1104
+add      wsp, w30, #4084
+add      x0, x24, #291
+add      x3, x24, #4095, lsl #12
+add      x8, sp, #1074
+add      sp, x29, #3816
+sub      w0, wsp, #4077
+sub      w4, w20, #546, lsl #12
+sub      sp, sp, #288
+sub      wsp, w19, #16
+adds     w13, w23, #291, lsl #12
+cmn      w2, #4095
+adds     w20, wsp, #0
+cmn      x3, #1, lsl #12
+cmp      sp, #20, lsl #12
+cmp      x30, #4095
+subs     x4, sp, #3822
+cmn      w3, #291, lsl #12
+cmn      wsp, #1365
+cmn      sp, #1092, lsl #12
+mov      sp, x30
+mov      wsp, w20
+mov      x11, sp
+mov      w24, wsp
+
+#------------------------------------------------------------------------------
+# Add-subtract (shifted register)
+#------------------------------------------------------------------------------
+
+add      w3, w5, w7
+add      wzr, w3, w5
+add      w20, wzr, w4
+add      w4, w6, wzr
+add      w11, w13, w15
+add      w9, w3, wzr, lsl #10
+add      w17, w29, w20, lsl #31
+add      w21, w22, w23, lsr #0
+add      w24, w25, w26, lsr #18
+add      w27, w28, w29, lsr #31
+add      w2, w3, w4, asr #0
+add      w5, w6, w7, asr #21
+add      w8, w9, w10, asr #31
+add      x3, x5, x7
+add      xzr, x3, x5
+add      x20, xzr, x4
+add      x4, x6, xzr
+add      x11, x13, x15
+add      x9, x3, xzr, lsl #10
+add      x17, x29, x20, lsl #63
+add      x21, x22, x23, lsr #0
+add      x24, x25, x26, lsr #18
+add      x27, x28, x29, lsr #63
+add      x2, x3, x4, asr #0
+add      x5, x6, x7, asr #21
+add      x8, x9, x10, asr #63
+adds     w3, w5, w7
+cmn      w3, w5
+adds     w20, wzr, w4
+adds     w4, w6, wzr
+adds     w11, w13, w15
+adds     w9, w3, wzr, lsl #10
+adds     w17, w29, w20, lsl #31
+adds     w21, w22, w23, lsr #0
+adds     w24, w25, w26, lsr #18
+adds     w27, w28, w29, lsr #31
+adds     w2, w3, w4, asr #0
+adds     w5, w6, w7, asr #21
+adds     w8, w9, w10, asr #31
+adds     x3, x5, x7
+cmn      x3, x5
+adds     x20, xzr, x4
+adds     x4, x6, xzr
+adds     x11, x13, x15
+adds     x9, x3, xzr, lsl #10
+adds     x17, x29, x20, lsl #63
+adds     x21, x22, x23, lsr #0
+adds     x24, x25, x26, lsr #18
+adds     x27, x28, x29, lsr #63
+adds     x2, x3, x4, asr #0
+adds     x5, x6, x7, asr #21
+adds     x8, x9, x10, asr #63
+sub      w3, w5, w7
+sub      wzr, w3, w5
+sub      w4, w6, wzr
+sub      w11, w13, w15
+sub      w9, w3, wzr, lsl #10
+sub      w17, w29, w20, lsl #31
+sub      w21, w22, w23, lsr #0
+sub      w24, w25, w26, lsr #18
+sub      w27, w28, w29, lsr #31
+sub      w2, w3, w4, asr #0
+sub      w5, w6, w7, asr #21
+sub      w8, w9, w10, asr #31
+sub      x3, x5, x7
+sub      xzr, x3, x5
+sub      x4, x6, xzr
+sub      x11, x13, x15
+sub      x9, x3, xzr, lsl #10
+sub      x17, x29, x20, lsl #63
+sub      x21, x22, x23, lsr #0
+sub      x24, x25, x26, lsr #18
+sub      x27, x28, x29, lsr #63
+sub      x2, x3, x4, asr #0
+sub      x5, x6, x7, asr #21
+sub      x8, x9, x10, asr #63
+subs     w3, w5, w7
+cmp      w3, w5
+subs     w4, w6, wzr
+subs     w11, w13, w15
+subs     w9, w3, wzr, lsl #10
+subs     w17, w29, w20, lsl #31
+subs     w21, w22, w23, lsr #0
+subs     w24, w25, w26, lsr #18
+subs     w27, w28, w29, lsr #31
+subs     w2, w3, w4, asr #0
+subs     w5, w6, w7, asr #21
+subs     w8, w9, w10, asr #31
+subs     x3, x5, x7
+cmp      x3, x5
+subs     x4, x6, xzr
+subs     x11, x13, x15
+subs     x9, x3, xzr, lsl #10
+subs     x17, x29, x20, lsl #63
+subs     x21, x22, x23, lsr #0
+subs     x24, x25, x26, lsr #18
+subs     x27, x28, x29, lsr #63
+subs     x2, x3, x4, asr #0
+subs     x5, x6, x7, asr #21
+subs     x8, x9, x10, asr #63
+cmn      wzr, w4
+cmn      w5, wzr
+cmn      w6, w7
+cmn      w8, w9, lsl #15
+cmn      w10, w11, lsl #31
+cmn      w12, w13, lsr #0
+cmn      w14, w15, lsr #21
+cmn      w16, w17, lsr #31
+cmn      w18, w19, asr #0
+cmn      w20, w21, asr #22
+cmn      w22, w23, asr #31
+cmn      x0, x3
+cmn      xzr, x4
+cmn      x5, xzr
+cmn      x6, x7
+cmn      x8, x9, lsl #15
+cmn      x10, x11, lsl #63
+cmn      x12, x13, lsr #0
+cmn      x14, x15, lsr #41
+cmn      x16, x17, lsr #63
+cmn      x18, x19, asr #0
+cmn      x20, x21, asr #55
+cmn      x22, x23, asr #63
+cmp      w0, w3
+cmp      wzr, w4
+cmp      w5, wzr
+cmp      w6, w7
+cmp      w8, w9, lsl #15
+cmp      w10, w11, lsl #31
+cmp      w12, w13, lsr #0
+cmp      w14, w15, lsr #21
+cmp      w18, w19, asr #0
+cmp      w20, w21, asr #22
+cmp      w22, w23, asr #31
+cmp      x0, x3
+cmp      xzr, x4
+cmp      x5, xzr
+cmp      x6, x7
+cmp      x8, x9, lsl #15
+cmp      x10, x11, lsl #63
+cmp      x12, x13, lsr #0
+cmp      x14, x15, lsr #41
+cmp      x16, x17, lsr #63
+cmp      x18, x19, asr #0
+cmp      x20, x21, asr #55
+cmp      x22, x23, asr #63
+cmp      wzr, w0
+cmp      xzr, x0
+
+#------------------------------------------------------------------------------
+# Add-subtract (shifted register)
+#------------------------------------------------------------------------------
+
+adc      w29, w27, w25
+adc      wzr, w3, w4
+adc      w9, wzr, w10
+adc      w20, w0, wzr
+adc      x29, x27, x25
+adc      xzr, x3, x4
+adc      x9, xzr, x10
+adc      x20, x0, xzr
+adcs     w29, w27, w25
+adcs     wzr, w3, w4
+adcs     w9, wzr, w10
+adcs     w20, w0, wzr
+adcs     x29, x27, x25
+adcs     xzr, x3, x4
+adcs     x9, xzr, x10
+adcs     x20, x0, xzr
+sbc      w29, w27, w25
+sbc      wzr, w3, w4
+ngc      w9, w10
+sbc      w20, w0, wzr
+sbc      x29, x27, x25
+sbc      xzr, x3, x4
+ngc      x9, x10
+sbc      x20, x0, xzr
+sbcs     w29, w27, w25
+sbcs     wzr, w3, w4
+ngcs     w9, w10
+sbcs     w20, w0, wzr
+sbcs     x29, x27, x25
+sbcs     xzr, x3, x4
+ngcs     x9, x10
+sbcs     x20, x0, xzr
+ngc      w3, w12
+ngc      wzr, w9
+ngc      w23, wzr
+ngc      x29, x30
+ngc      xzr, x0
+ngc      x0, xzr
+ngcs     w3, w12
+ngcs     wzr, w9
+ngcs     w23, wzr
+ngcs     x29, x30
+ngcs     xzr, x0
+ngcs     x0, xzr
+
+#------------------------------------------------------------------------------
+# Compare and branch (immediate)
+#------------------------------------------------------------------------------
+
+sbfx     x1, x2, #3, #2
+asr      x3, x4, #63
+asr      wzr, wzr, #31
+sbfx     w12, w9, #0, #1
+ubfiz    x4, x5, #52, #11
+ubfx     xzr, x4, #0, #1
+ubfiz    x4, xzr, #1, #6
+lsr      x5, x6, #12
+bfi      x4, x5, #52, #11
+bfxil    xzr, x4, #0, #1
+bfi      x4, xzr, #1, #6
+bfxil    x5, x6, #12, #52
+sxtb     w1, w2
+sxtb     xzr, w3
+sxth     w9, w10
+sxth     x0, w1
+sxtw     x3, w30
+uxtb     w1, w2
+uxth     w9, w10
+ubfx     x3, x30, #0, #32
+asr      w3, w2, #0
+asr      w9, w10, #31
+asr      x20, x21, #63
+asr      w1, wzr, #3
+lsr      w3, w2, #0
+lsr      w9, w10, #31
+lsr      x20, x21, #63
+lsr      wzr, wzr, #3
+lsr      w3, w2, #0
+lsl      w9, w10, #31
+lsl      x20, x21, #63
+lsl      w1, wzr, #3
+sbfx     w9, w10, #0, #1
+sbfiz    x2, x3, #63, #1
+asr      x19, x20, #0
+sbfiz    x9, x10, #5, #59
+asr      w9, w10, #0
+sbfiz    w11, w12, #31, #1
+sbfiz    w13, w14, #29, #3
+sbfiz    xzr, xzr, #10, #11
+sbfx     w9, w10, #0, #1
+asr      x2, x3, #63
+asr      x19, x20, #0
+asr      x9, x10, #5
+asr      w9, w10, #0
+asr      w11, w12, #31
+asr      w13, w14, #29
+sbfx     xzr, xzr, #10, #11
+bfxil    w9, w10, #0, #1
+bfi      x2, x3, #63, #1
+bfxil    x19, x20, #0, #64
+bfi      x9, x10, #5, #59
+bfxil    w9, w10, #0, #32
+bfi      w11, w12, #31, #1
+bfi      w13, w14, #29, #3
+bfi      xzr, xzr, #10, #11
+bfxil    w9, w10, #0, #1
+bfxil    x2, x3, #63, #1
+bfxil    x19, x20, #0, #64
+bfxil    x9, x10, #5, #59
+bfxil    w9, w10, #0, #32
+bfxil    w11, w12, #31, #1
+bfxil    w13, w14, #29, #3
+bfxil    xzr, xzr, #10, #11
+ubfx     w9, w10, #0, #1
+lsl      x2, x3, #63
+lsr      x19, x20, #0
+lsl      x9, x10, #5
+lsr      w9, w10, #0
+lsl      w11, w12, #31
+lsl      w13, w14, #29
+ubfiz    xzr, xzr, #10, #11
+ubfx     w9, w10, #0, #1
+lsr      x2, x3, #63
+lsr      x19, x20, #0
+lsr      x9, x10, #5
+lsr      w9, w10, #0
+lsr      w11, w12, #31
+lsr      w13, w14, #29
+ubfx     xzr, xzr, #10, #11
+
+#------------------------------------------------------------------------------
+# Compare and branch (immediate)
+#------------------------------------------------------------------------------
+
+cbz      w5, #4
+cbz      x5, #0
+cbnz     x2, #-4
+cbnz     x26, #1048572
+cbz      wzr, #0
+cbnz     xzr, #0
+
+#------------------------------------------------------------------------------
+# Conditional branch (immediate)
+#------------------------------------------------------------------------------
+
+b.ne #4
+b.ge #1048572
+b.ge #-4
+
+#------------------------------------------------------------------------------
+# Conditional compare (immediate)
+#------------------------------------------------------------------------------
+
+ccmp w1, #31, #0, eq
+ccmp w3, #0, #15, hs
+ccmp wzr, #15, #13, hs
+ccmp x9, #31, #0, le
+ccmp x3, #0, #15, gt
+ccmp xzr, #5, #7, ne
+ccmn w1, #31, #0, eq
+ccmn w3, #0, #15, hs
+ccmn wzr, #15, #13, hs
+ccmn x9, #31, #0, le
+ccmn x3, #0, #15, gt
+ccmn xzr, #5, #7, ne
+
+#------------------------------------------------------------------------------
+# Conditional compare (register)
+#------------------------------------------------------------------------------
+
+ccmp w1, wzr, #0, eq
+ccmp w3, w0, #15, hs
+ccmp wzr, w15, #13, hs
+ccmp x9, xzr, #0, le
+ccmp x3, x0, #15, gt
+ccmp xzr, x5, #7, ne
+ccmn w1, wzr, #0, eq
+ccmn w3, w0, #15, hs
+ccmn wzr, w15, #13, hs
+ccmn x9, xzr, #0, le
+ccmn x3, x0, #15, gt
+ccmn xzr, x5, #7, ne
+
+#------------------------------------------------------------------------------
+# Conditional branch (immediate)
+#------------------------------------------------------------------------------
+
+csel     w1, w0, w19, ne
+csel     wzr, w5, w9, eq
+csel     w9, wzr, w30, gt
+csel     w1, w28, wzr, mi
+csel     x19, x23, x29, lt
+csel     xzr, x3, x4, ge
+csel     x5, xzr, x6, hs
+csel     x7, x8, xzr, lo
+csinc    w1, w0, w19, ne
+csinc    wzr, w5, w9, eq
+csinc    w9, wzr, w30, gt
+csinc    w1, w28, wzr, mi
+csinc    x19, x23, x29, lt
+csinc    xzr, x3, x4, ge
+csinc    x5, xzr, x6, hs
+csinc    x7, x8, xzr, lo
+csinv    w1, w0, w19, ne
+csinv    wzr, w5, w9, eq
+csinv    w9, wzr, w30, gt
+csinv    w1, w28, wzr, mi
+csinv    x19, x23, x29, lt
+csinv    xzr, x3, x4, ge
+csinv    x5, xzr, x6, hs
+csinv    x7, x8, xzr, lo
+csneg    w1, w0, w19, ne
+csneg    wzr, w5, w9, eq
+csneg    w9, wzr, w30, gt
+csneg    w1, w28, wzr, mi
+csneg    x19, x23, x29, lt
+csneg    xzr, x3, x4, ge
+csneg    x5, xzr, x6, hs
+csneg    x7, x8, xzr, lo
+cset    w3, eq
+cset    x9, pl
+csetm    w20, ne
+csetm    x30, ge
+csinc    w2, wzr, wzr, al
+csinv    x3, xzr, xzr, nv
+cinc    w3, w5, gt
+cinc    wzr, w4, le
+cset    w9, lt
+cinc    x3, x5, gt
+cinc    xzr, x4, le
+cset    x9, lt
+csinc   w5, w6, w6, nv
+csinc   x1, x2, x2, al
+cinv    w3, w5, gt
+cinv    wzr, w4, le
+csetm   w9, lt
+cinv    x3, x5, gt
+cinv    xzr, x4, le
+csetm   x9, lt
+csinv   x1, x0, x0, al
+csinv   w9, w8, w8, nv
+cneg     w3, w5, gt
+cneg     wzr, w4, le
+cneg     w9, wzr, lt
+cneg     x3, x5, gt
+cneg     xzr, x4, le
+cneg     x9, xzr, lt
+csneg    x4, x8, x8, al
+csinv    w9, w8, w8, nv
+
+#------------------------------------------------------------------------------
+# Data-processing (1 source)
+#------------------------------------------------------------------------------
+
+rbit	w0, w7
+rbit   x18, x3
+rev16	w17, w1
+rev16	x5, x2
+rev	w18, w0
+rev32	x20, x1
+rev	x22, x2
+clz	w24, w3
+clz	x26, x4
+cls	w3, w5
+cls	x20, x5
+
+#------------------------------------------------------------------------------
+# Data-processing (2 source)
+#------------------------------------------------------------------------------
+
+udiv	w0, w7, w10
+udiv	x9, x22, x4
+sdiv	w12, w21, w0
+sdiv	x13, x2, x1
+lsl	w11, w12, w13
+lsl	x14, x15, x16
+lsr	w17, w18, w19
+lsr	x20, x21, x22
+asr	w23, w24, w25
+asr	x26, x27, x28
+ror	w0, w1, w2
+ror    x3, x4, x5
+lsl	w6, w7, w8
+lsl	x9, x10, x11
+lsr	w12, w13, w14
+lsr	x15, x16, x17
+asr	w18, w19, w20
+asr	x21, x22, x23
+ror	w24, w25, w26
+ror	x27, x28, x29
+
+#------------------------------------------------------------------------------
+# Data-processing (3 sources)
+#------------------------------------------------------------------------------
+
+smulh    x30, x29, x28
+smulh    xzr, x27, x26
+umulh    x30, x29, x28
+umulh    x23, x30, xzr
+madd     w1, w3, w7, w4
+madd     wzr, w0, w9, w11
+madd     w13, wzr, w4, w4
+madd     w19, w30, wzr, w29
+mul      w4, w5, w6
+madd     x1, x3, x7, x4
+madd     xzr, x0, x9, x11
+madd     x13, xzr, x4, x4
+madd     x19, x30, xzr, x29
+mul      x4, x5, x6
+msub     w1, w3, w7, w4
+msub     wzr, w0, w9, w11
+msub     w13, wzr, w4, w4
+msub     w19, w30, wzr, w29
+mneg     w4, w5, w6
+msub     x1, x3, x7, x4
+msub     xzr, x0, x9, x11
+msub     x13, xzr, x4, x4
+msub     x19, x30, xzr, x29
+mneg     x4, x5, x6
+smaddl   x3, w5, w2, x9
+smaddl   xzr, w10, w11, x12
+smaddl   x13, wzr, w14, x15
+smaddl   x16, w17, wzr, x18
+smull    x19, w20, w21
+smsubl   x3, w5, w2, x9
+smsubl   xzr, w10, w11, x12
+smsubl   x13, wzr, w14, x15
+smsubl   x16, w17, wzr, x18
+smnegl   x19, w20, w21
+umaddl   x3, w5, w2, x9
+umaddl   xzr, w10, w11, x12
+umaddl   x13, wzr, w14, x15
+umaddl   x16, w17, wzr, x18
+umull    x19, w20, w21
+umsubl   x3, w5, w2, x9
+umsubl   x16, w17, wzr, x18
+umnegl   x19, w20, w21
+smulh    x30, x29, x28
+smulh    x23, x22, xzr
+umulh    x23, x22, xzr
+mul      x19, x20, xzr
+mneg     w21, w22, w23
+smull    x11, w13, w17
+umull    x11, w13, w17
+smnegl   x11, w13, w17
+umnegl   x11, w13, w17
+
+#------------------------------------------------------------------------------
+# Extract (immediate)
+#------------------------------------------------------------------------------
+
+extr     w3, w5, w7, #0
+extr     w11, w13, w17, #31
+extr     x3, x5, x7, #15
+extr     x11, x13, x17, #63
+ror     x19, x23, #24
+ror     x29, xzr, #63
+ror     w9, w13, #31
+
+#------------------------------------------------------------------------------
+# Floating-point compare
+#------------------------------------------------------------------------------
+
+fcmp    s3, s5
+fcmp    s31, #0.0
+fcmp    s31, #0.0
+fcmpe   s29, s30
+fcmpe   s15, #0.0
+fcmpe   s15, #0.0
+fcmp    d4, d12
+fcmp    d23, #0.0
+fcmp    d23, #0.0
+fcmpe   d26, d22
+fcmpe   d29, #0.0
+fcmpe   d29, #0.0
+
+#------------------------------------------------------------------------------
+# Floating-point conditional compare
+#------------------------------------------------------------------------------
+
+fccmp s1, s31, #0, eq
+fccmp s3, s0, #15, hs
+fccmp s31, s15, #13, hs
+fccmp d9, d31, #0, le
+fccmp d3, d0, #15, gt
+fccmp d31, d5, #7, ne
+fccmpe s1, s31, #0, eq
+fccmpe s3, s0, #15, hs
+fccmpe s31, s15, #13, hs
+fccmpe d9, d31, #0, le
+fccmpe d3, d0, #15, gt
+fccmpe d31, d5, #7, ne
+
+#-------------------------------------------------------------------------------
+# Floating-point conditional compare
+#-------------------------------------------------------------------------------
+
+fcsel s3, s20, s9, pl
+fcsel d9, d10, d11, mi
+
+#------------------------------------------------------------------------------
+# Floating-point data-processing (1 source)
+#------------------------------------------------------------------------------
+
+fmov     s0, s1
+fabs     s2, s3
+fneg     s4, s5
+fsqrt    s6, s7
+fcvt     d8, s9
+fcvt     h10, s11
+frintn   s12, s13
+frintp   s14, s15
+frintm   s16, s17
+frintz   s18, s19
+frinta   s20, s21
+frintx   s22, s23
+frinti   s24, s25
+fmov     d0, d1
+fabs     d2, d3
+fneg     d4, d5
+fsqrt    d6, d7
+fcvt     s8, d9
+fcvt     h10, d11
+frintn   d12, d13
+frintp   d14, d15
+frintm   d16, d17
+frintz   d18, d19
+frinta   d20, d21
+frintx   d22, d23
+frinti   d24, d25
+fcvt     s26, h27
+fcvt     d28, h29
+
+#------------------------------------------------------------------------------
+# Floating-point data-processing (2 sources)
+#------------------------------------------------------------------------------
+
+fmul     s20, s19, s17
+fdiv     s1, s2, s3
+fadd     s4, s5, s6
+fsub     s7, s8, s9
+fmax     s10, s11, s12
+fmin     s13, s14, s15
+fmaxnm   s16, s17, s18
+fminnm   s19, s20, s21
+fnmul    s22, s23, s2
+fmul     d20, d19, d17
+fdiv     d1, d2, d3
+fadd     d4, d5, d6
+fsub     d7, d8, d9
+fmax     d10, d11, d12
+fmin     d13, d14, d15
+fmaxnm   d16, d17, d18
+fminnm   d19, d20, d21
+fnmul    d22, d23, d24
+
+#------------------------------------------------------------------------------
+# Floating-point data-processing (1 source)
+#------------------------------------------------------------------------------
+
+fmadd s3, s5, s6, s31
+fmadd d3, d13, d0, d23
+fmsub s3, s5, s6, s31
+fmsub d3, d13, d0, d23
+fnmadd s3, s5, s6, s31
+fnmadd d3, d13, d0, d23
+fnmsub s3, s5, s6, s31
+fnmsub d3, d13, d0, d23
+
+#------------------------------------------------------------------------------
+# Floating-point <-> fixed-point conversion
+#------------------------------------------------------------------------------
+
+fcvtzs  w3, h5, #1
+fcvtzs  wzr, h20, #13
+fcvtzs  w19, h0, #32
+fcvtzs  x3, h5, #1
+fcvtzs  x12, h30, #45
+fcvtzs  x19, h0, #64
+fcvtzs  w3, s5, #1
+fcvtzs  wzr, s20, #13
+fcvtzs  w19, s0, #32
+fcvtzs  x3, s5, #1
+fcvtzs  x12, s30, #45
+fcvtzs  x19, s0, #64
+fcvtzs  w3, d5, #1
+fcvtzs  wzr, d20, #13
+fcvtzs  w19, d0, #32
+fcvtzs  x3, d5, #1
+fcvtzs  x12, d30, #45
+fcvtzs  x19, d0, #64
+fcvtzu  w3, h5, #1
+fcvtzu  wzr, h20, #13
+fcvtzu  w19, h0, #32
+fcvtzu  x3, h5, #1
+fcvtzu  x12, h30, #45
+fcvtzu  x19, h0, #64
+fcvtzu  w3, s5, #1
+fcvtzu  wzr, s20, #13
+fcvtzu  w19, s0, #32
+fcvtzu  x3, s5, #1
+fcvtzu  x12, s30, #45
+fcvtzu  x19, s0, #64
+fcvtzu  w3, d5, #1
+fcvtzu  wzr, d20, #13
+fcvtzu  w19, d0, #32
+fcvtzu  x3, d5, #1
+fcvtzu  x12, d30, #45
+fcvtzu  x19, d0, #64
+scvtf   h23, w19, #1
+scvtf   h31, wzr, #20
+scvtf   h14, w0, #32
+scvtf   h23, x19, #1
+scvtf   h31, xzr, #20
+scvtf   h14, x0, #64
+scvtf   s23, w19, #1
+scvtf   s31, wzr, #20
+scvtf   s14, w0, #32
+scvtf   s23, x19, #1
+scvtf   s31, xzr, #20
+scvtf   s14, x0, #64
+scvtf   d23, w19, #1
+scvtf   d31, wzr, #20
+scvtf   d14, w0, #32
+scvtf   d23, x19, #1
+scvtf   d31, xzr, #20
+scvtf   d14, x0, #64
+ucvtf   h23, w19, #1
+ucvtf   h31, wzr, #20
+ucvtf   h14, w0, #32
+ucvtf   h23, x19, #1
+ucvtf   h31, xzr, #20
+ucvtf   h14, x0, #64
+ucvtf   s23, w19, #1
+ucvtf   s31, wzr, #20
+ucvtf   s14, w0, #32
+ucvtf   s23, x19, #1
+ucvtf   s31, xzr, #20
+ucvtf   s14, x0, #64
+ucvtf   d23, w19, #1
+ucvtf   d31, wzr, #20
+ucvtf   d14, w0, #32
+ucvtf   d23, x19, #1
+ucvtf   d31, xzr, #20
+ucvtf   d14, x0, #64
+
+#------------------------------------------------------------------------------
+# Floating-point <-> integer conversion
+#------------------------------------------------------------------------------
+
+fcvtns   w3, h31
+fcvtns   xzr, h12
+fcvtnu   wzr, h12
+fcvtnu   x0, h0
+fcvtps   wzr, h9
+fcvtps   x12, h20
+fcvtpu   w30, h23
+fcvtpu   x29, h3
+fcvtms   w2, h3
+fcvtms   x4, h5
+fcvtmu   w6, h7
+fcvtmu   x8, h9
+fcvtzs   w10, h11
+fcvtzs   x12, h13
+fcvtzu   w14, h15
+fcvtzu   x15, h16
+scvtf    h17, w18
+scvtf    h19, x20
+ucvtf    h21, w22
+scvtf    h23, x24
+fcvtas   w25, h26
+fcvtas   x27, h28
+fcvtau   w29, h30
+fcvtau   xzr, h0
+fcvtns   w3, s31
+fcvtns   xzr, s12
+fcvtnu   wzr, s12
+fcvtnu   x0, s0
+fcvtps   wzr, s9
+fcvtps   x12, s20
+fcvtpu   w30, s23
+fcvtpu   x29, s3
+fcvtms   w2, s3
+fcvtms   x4, s5
+fcvtmu   w6, s7
+fcvtmu   x8, s9
+fcvtzs   w10, s11
+fcvtzs   x12, s13
+fcvtzu   w14, s15
+fcvtzu   x15, s16
+scvtf    s17, w18
+scvtf    s19, x20
+ucvtf    s21, w22
+scvtf    s23, x24
+fcvtas   w25, s26
+fcvtas   x27, s28
+fcvtau   w29, s30
+fcvtau   xzr, s0
+fcvtns   w3, d31
+fcvtns   xzr, d12
+fcvtnu   wzr, d12
+fcvtnu   x0, d0
+fcvtps   wzr, d9
+fcvtps   x12, d20
+fcvtpu   w30, d23
+fcvtpu   x29, d3
+fcvtms   w2, d3
+fcvtms   x4, d5
+fcvtmu   w6, d7
+fcvtmu   x8, d9
+fcvtzs   w10, d11
+fcvtzs   x12, d13
+fcvtzu   w14, d15
+fcvtzu   x15, d16
+scvtf    d17, w18
+scvtf    d19, x20
+ucvtf    d21, w22
+ucvtf    d23, x24
+fcvtas   w25, d26
+fcvtas   x27, d28
+fcvtau   w29, d30
+fcvtau   xzr, d0
+fmov     w3, s9
+fmov     s9, w3
+fmov     x20, d31
+fmov     d1, x15
+fmov     x3, v12.d[1]
+fmov     v1.d[1], x19
+
+#------------------------------------------------------------------------------
+# Floating-point immediate
+#------------------------------------------------------------------------------
+
+fmov     s2, #0.12500000
+fmov     s3, #1.00000000
+fmov     d30, #16.00000000
+fmov     s4, #1.06250000
+fmov     d10, #1.93750000
+fmov     s12, #-1.00000000
+fmov     d16, #8.50000000
+
+#------------------------------------------------------------------------------
+# Load-register (literal)
+#------------------------------------------------------------------------------
+
+ldr       w3, #0
+ldr       x29, #4
+ldrsw     xzr, #-4
+ldr       s0, #8
+ldr       d0, #1048572
+ldr       q0, #-1048576
+prfm      pldl1strm, #0
+prfm      #22, #0
+
+#------------------------------------------------------------------------------
+# Load/store exclusive
+#------------------------------------------------------------------------------
+
+stxrb      w18, w8, [sp]
+stxrh      w24, w15, [x16]
+stxr       w5, w6, [x17]
+stxr       w1, x10, [x21]
+ldxrb      w30, [x0]
+ldxrh      w17, [x4]
+ldxr       w22, [sp]
+ldxr       x11, [x29]
+ldxr       x11, [x29]
+ldxr       x11, [x29]
+stxp       w12, w11, w10, [sp]
+stxp       wzr, x27, x9, [x12]
+ldxp       w0, wzr, [sp]
+ldxp       x17, x0, [x18]
+ldxp       x17, x0, [x18]
+stlxrb     w12, w22, [x0]
+stlxrh     w10, w1, [x1]
+stlxr      w9, w2, [x2]
+stlxr      w9, x3, [sp]
+ldaxrb     w8, [x4]
+ldaxrh     w7, [x5]
+ldaxr      w6, [sp]
+ldaxr      x5, [x6]
+ldaxr      x5, [x6]
+ldaxr      x5, [x6]
+stlxp      w4, w5, w6, [sp]
+stlxp      wzr, x6, x7, [x1]
+ldaxp      w5, w18, [sp]
+ldaxp      x6, x19, [x22]
+ldaxp      x6, x19, [x22]
+stlrb      w24, [sp]
+stlrh      w25, [x30]
+stlr       w26, [x29]
+stlr       x27, [x28]
+stlr       x27, [x28]
+stlr       x27, [x28]
+ldarb      w23, [sp]
+ldarh      w22, [x30]
+ldar       wzr, [x29]
+ldar       x21, [x28]
+ldar       x21, [x28]
+ldar       x21, [x28]
+
+#------------------------------------------------------------------------------
+# Load/store (unscaled  immediate)
+#------------------------------------------------------------------------------
+
+sturb    w9, [sp]
+sturh    wzr, [x12, #255]
+stur     w16, [x0, #-256]
+stur     x28, [x14, #1]
+ldurb    w1, [x20, #255]
+ldurh    w20, [x1, #255]
+ldur     w12, [sp, #255]
+ldur     xzr, [x12, #255]
+ldursb   x9, [x7, #-256]
+ldursh   x17, [x19, #-256]
+ldursw   x20, [x15, #-256]
+prfum    pldl2keep, [sp, #-256]
+ldursb   w19, [x1, #-256]
+ldursh   w15, [x21, #-256]
+stur     b0, [sp, #1]
+stur     h12, [x12, #-1]
+stur     s15, [x0, #255]
+stur     d31, [x5, #25]
+stur     q9, [x5]
+ldur     b3, [sp]
+ldur     h5, [x4, #-256]
+ldur     s7, [x12, #-1]
+ldur     d11, [x19, #4]
+ldur     q13, [x1, #2]
+
+#------------------------------------------------------------------------------
+# Load/store (immediate post-indexed)
+#------------------------------------------------------------------------------
+
+strb     w9, [x2], #255
+strb     w10, [x3], #1
+strb     w10, [x3], #-256
+strh     w9, [x2], #255
+strh     w9, [x2], #1
+strh     w10, [x3], #-256
+str      w19, [sp], #255
+str      w20, [x30], #1
+str      w21, [x12], #-256
+str      xzr, [x9], #255
+str      x2, [x3], #1
+str      x19, [x12], #-256
+ldrb     w9, [x2], #255
+ldrb     w10, [x3], #1
+ldrb     w10, [x3], #-256
+ldrh     w9, [x2], #255
+ldrh     w9, [x2], #1
+ldrh     w10, [x3], #-256
+ldr      w19, [sp], #255
+ldr      w20, [x30], #1
+ldr      w21, [x12], #-256
+ldr      xzr, [x9], #255
+ldr      x2, [x3], #1
+ldr      x19, [x12], #-256
+ldrsb    xzr, [x9], #255
+ldrsb    x2, [x3], #1
+ldrsb    x19, [x12], #-256
+ldrsh    xzr, [x9], #255
+ldrsh    x2, [x3], #1
+ldrsh    x19, [x12], #-256
+ldrsw    xzr, [x9], #255
+ldrsw    x2, [x3], #1
+ldrsw    x19, [x12], #-256
+ldrsb    wzr, [x9], #255
+ldrsb    w2, [x3], #1
+ldrsb    w19, [x12], #-256
+ldrsh    wzr, [x9], #255
+ldrsh    w2, [x3], #1
+ldrsh    w19, [x12], #-256
+str      b0, [x0], #255
+str      b3, [x3], #1
+str      b5, [sp], #-256
+str      h10, [x10], #255
+str      h13, [x23], #1
+str      h15, [sp], #-256
+str      s20, [x20], #255
+str      s23, [x23], #1
+str      s25, [x0], #-256
+str      d20, [x20], #255
+str      d23, [x23], #1
+str      d25, [x0], #-256
+ldr      b0, [x0], #255
+ldr      b3, [x3], #1
+ldr      b5, [sp], #-256
+ldr      h10, [x10], #255
+ldr      h13, [x23], #1
+ldr      h15, [sp], #-256
+ldr      s20, [x20], #255
+ldr      s23, [x23], #1
+ldr      s25, [x0], #-256
+ldr      d20, [x20], #255
+ldr      d23, [x23], #1
+ldr      d25, [x0], #-256
+ldr      q20, [x1], #255
+ldr      q23, [x9], #1
+ldr      q25, [x20], #-256
+str      q10, [x1], #255
+str      q22, [sp], #1
+str      q21, [x20], #-256
+
+#-------------------------------------------------------------------------------
+# Load-store register (immediate pre-indexed)
+#-------------------------------------------------------------------------------
+
+ldr      x3, [x4, #0]!
+strb     w9, [x2, #255]!
+strb     w10, [x3, #1]!
+strb     w10, [x3, #-256]!
+strh     w9, [x2, #255]!
+strh     w9, [x2, #1]!
+strh     w10, [x3, #-256]!
+str      w19, [sp, #255]!
+str      w20, [x30, #1]!
+str      w21, [x12, #-256]!
+str      xzr, [x9, #255]!
+str      x2, [x3, #1]!
+str      x19, [x12, #-256]!
+ldrb     w9, [x2, #255]!
+ldrb     w10, [x3, #1]!
+ldrb     w10, [x3, #-256]!
+ldrh     w9, [x2, #255]!
+ldrh     w9, [x2, #1]!
+ldrh     w10, [x3, #-256]!
+ldr      w19, [sp, #255]!
+ldr      w20, [x30, #1]!
+ldr      w21, [x12, #-256]!
+ldr      xzr, [x9, #255]!
+ldr      x2, [x3, #1]!
+ldr      x19, [x12, #-256]!
+ldrsb    xzr, [x9, #255]!
+ldrsb    x2, [x3, #1]!
+ldrsb    x19, [x12, #-256]!
+ldrsh    xzr, [x9, #255]!
+ldrsh    x2, [x3, #1]!
+ldrsh    x19, [x12, #-256]!
+ldrsw    xzr, [x9, #255]!
+ldrsw    x2, [x3, #1]!
+ldrsw    x19, [x12, #-256]!
+ldrsb    wzr, [x9, #255]!
+ldrsb    w2, [x3, #1]!
+ldrsb    w19, [x12, #-256]!
+ldrsh    wzr, [x9, #255]!
+ldrsh    w2, [x3, #1]!
+ldrsh    w19, [x12, #-256]!
+str      b0, [x0, #255]!
+str      b3, [x3, #1]!
+str      b5, [sp, #-256]!
+str      h10, [x10, #255]!
+str      h13, [x23, #1]!
+str      h15, [sp, #-256]!
+str      s20, [x20, #255]!
+str      s23, [x23, #1]!
+str      s25, [x0, #-256]!
+str      d20, [x20, #255]!
+str      d23, [x23, #1]!
+str      d25, [x0, #-256]!
+ldr      b0, [x0, #255]!
+ldr      b3, [x3, #1]!
+ldr      b5, [sp, #-256]!
+ldr      h10, [x10, #255]!
+ldr      h13, [x23, #1]!
+ldr      h15, [sp, #-256]!
+ldr      s20, [x20, #255]!
+ldr      s23, [x23, #1]!
+ldr      s25, [x0, #-256]!
+ldr      d20, [x20, #255]!
+ldr      d23, [x23, #1]!
+ldr      d25, [x0, #-256]!
+ldr      q20, [x1, #255]!
+ldr      q23, [x9, #1]!
+ldr      q25, [x20, #-256]!
+str      q10, [x1, #255]!
+str      q22, [sp, #1]!
+str      q21, [x20, #-256]!
+
+#------------------------------------------------------------------------------
+# Load/store (unprivileged)
+#------------------------------------------------------------------------------
+
+sttrb    w9, [sp]
+sttrh    wzr, [x12, #255]
+sttr     w16, [x0, #-256]
+sttr     x28, [x14, #1]
+ldtrb    w1, [x20, #255]
+ldtrh    w20, [x1, #255]
+ldtr     w12, [sp, #255]
+ldtr     xzr, [x12, #255]
+ldtrsb   x9, [x7, #-256]
+ldtrsh   x17, [x19, #-256]
+ldtrsw   x20, [x15, #-256]
+ldtrsb   w19, [x1, #-256]
+ldtrsh   w15, [x21, #-256]
+
+#------------------------------------------------------------------------------
+# Load/store (unsigned  immediate)
+#------------------------------------------------------------------------------
+
+ldr      x4, [x29]
+ldr      x30, [x12, #32760]
+ldr      x20, [sp, #8]
+ldr      xzr, [sp]
+ldr      w2, [sp]
+ldr      w17, [sp, #16380]
+ldr      w13, [x2, #4]
+ldrsw    x2, [x5, #4]
+ldrsw    x23, [sp, #16380]
+ldrh     w2, [x4]
+ldrsh    w23, [x6, #8190]
+ldrsh    wzr, [sp, #2]
+ldrsh    x29, [x2, #2]
+ldrb     w26, [x3, #121]
+ldrb     w12, [x2]
+ldrsb    w27, [sp, #4095]
+ldrsb    xzr, [x15]
+str      x30, [sp]
+str      w20, [x4, #16380]
+strh     w17, [sp, #8190]
+strb     w23, [x3, #4095]
+strb     wzr, [x2]
+ldr      b31, [sp, #4095]
+ldr      h20, [x2, #8190]
+ldr      s10, [x19, #16380]
+ldr      d3, [x10, #32760]
+str      q12, [sp, #65520]
+
+#------------------------------------------------------------------------------
+# Load/store (register offset)
+#------------------------------------------------------------------------------
+
+ldrb     w3, [sp, x5]
+ldrb     w9, [x27, x6]
+ldrsb    w10, [x30, x7]
+ldrb     w11, [x29, x3, sxtx]
+strb     w12, [x28, xzr, sxtx]
+ldrb     w14, [x26, w6, uxtw]
+ldrsb    w15, [x25, w7, uxtw]
+ldrb     w17, [x23, w9, sxtw]
+ldrsb    x18, [x22, w10, sxtw]
+ldrsh    w3, [sp, x5]
+ldrsh    w9, [x27, x6]
+ldrh     w10, [x30, x7, lsl #1]
+strh     w11, [x29, x3, sxtx]
+ldrh     w12, [x28, xzr, sxtx]
+ldrsh    x13, [x27, x5, sxtx #1]
+ldrh     w14, [x26, w6, uxtw]
+ldrh     w15, [x25, w7, uxtw]
+ldrsh    w16, [x24, w8, uxtw #1]
+ldrh     w17, [x23, w9, sxtw]
+ldrh     w18, [x22, w10, sxtw]
+strh     w19, [x21, wzr, sxtw #1]
+ldr      w3, [sp, x5]
+ldr      s9, [x27, x6]
+ldr      w10, [x30, x7, lsl #2]
+ldr      w11, [x29, x3, sxtx]
+str      s12, [x28, xzr, sxtx]
+str      w13, [x27, x5, sxtx #2]
+str      w14, [x26, w6, uxtw]
+ldr      w15, [x25, w7, uxtw]
+ldr      w16, [x24, w8, uxtw #2]
+ldrsw    x17, [x23, w9, sxtw]
+ldr      w18, [x22, w10, sxtw]
+ldrsw    x19, [x21, wzr, sxtw #2]
+ldr      x3, [sp, x5]
+str      x9, [x27, x6]
+ldr      d10, [x30, x7, lsl #3]
+str      x11, [x29, x3, sxtx]
+ldr      x12, [x28, xzr, sxtx]
+ldr      x13, [x27, x5, sxtx #3]
+prfm     pldl1keep, [x26, w6, uxtw]
+ldr      x15, [x25, w7, uxtw]
+ldr      x16, [x24, w8, uxtw #3]
+ldr      x17, [x23, w9, sxtw]
+ldr      x18, [x22, w10, sxtw]
+str      d19, [x21, wzr, sxtw #3]
+ldr      q3, [sp, x5]
+ldr      q9, [x27, x6]
+ldr      q10, [x30, x7, lsl #4]
+str      q11, [x29, x3, sxtx]
+str      q12, [x28, xzr, sxtx]
+str      q13, [x27, x5, sxtx #4]
+ldr      q14, [x26, w6, uxtw]
+ldr      q15, [x25, w7, uxtw]
+ldr      q16, [x24, w8, uxtw #4]
+ldr      q17, [x23, w9, sxtw]
+str      q18, [x22, w10, sxtw]
+ldr      q19, [x21, wzr, sxtw #4]
+
+#------------------------------------------------------------------------------
+# Load/store register pair (offset)
+#------------------------------------------------------------------------------
+
+ldp      w3, w5, [sp]
+stp      wzr, w9, [sp, #252]
+ldp      w2, wzr, [sp, #-256]
+ldp      w9, w10, [sp, #4]
+ldpsw    x9, x10, [sp, #4]
+ldpsw    x9, x10, [x2, #-256]
+ldpsw    x20, x30, [sp, #252]
+ldp      x21, x29, [x2, #504]
+ldp      x22, x23, [x3, #-512]
+ldp      x24, x25, [x4, #8]
+ldp      s29, s28, [sp, #252]
+stp      s27, s26, [sp, #-256]
+ldp      s1, s2, [x3, #44]
+stp      d3, d5, [x9, #504]
+stp      d7, d11, [x10, #-512]
+ldp      d2, d3, [x30, #-8]
+stp      q3, q5, [sp]
+stp      q17, q19, [sp, #1008]
+ldp      q23, q29, [x1, #-1024]
+
+#------------------------------------------------------------------------------
+# Load/store register pair (post-indexed)
+#------------------------------------------------------------------------------
+
+ldp      w3, w5, [sp], #0
+stp      wzr, w9, [sp], #252
+ldp      w2, wzr, [sp], #-256
+ldp      w9, w10, [sp], #4
+ldpsw    x9, x10, [sp], #4
+ldpsw    x9, x10, [x2], #-256
+ldpsw    x20, x30, [sp], #252
+ldp      x21, x29, [x2], #504
+ldp      x22, x23, [x3], #-512
+ldp      x24, x25, [x4], #8
+ldp      s29, s28, [sp], #252
+stp      s27, s26, [sp], #-256
+ldp      s1, s2, [x3], #44
+stp      d3, d5, [x9], #504
+stp      d7, d11, [x10], #-512
+ldp      d2, d3, [x30], #-8
+stp      q3, q5, [sp], #0
+stp      q17, q19, [sp], #1008
+ldp      q23, q29, [x1], #-1024
+
+#------------------------------------------------------------------------------
+# Load/store register pair (pre-indexed)
+#------------------------------------------------------------------------------
+
+ldp      w3, w5, [sp, #0]!
+stp      wzr, w9, [sp, #252]!
+ldp      w2, wzr, [sp, #-256]!
+ldp      w9, w10, [sp, #4]!
+ldpsw    x9, x10, [sp, #4]!
+ldpsw    x9, x10, [x2, #-256]!
+ldpsw    x20, x30, [sp, #252]!
+ldp      x21, x29, [x2, #504]!
+ldp      x22, x23, [x3, #-512]!
+ldp      x24, x25, [x4, #8]!
+ldp      s29, s28, [sp, #252]!
+stp      s27, s26, [sp, #-256]!
+ldp      s1, s2, [x3, #44]!
+stp      d3, d5, [x9, #504]!
+stp      d7, d11, [x10, #-512]!
+ldp      d2, d3, [x30, #-8]!
+stp      q3, q5, [sp, #0]!
+stp      q17, q19, [sp, #1008]!
+ldp      q23, q29, [x1, #-1024]!
+
+#------------------------------------------------------------------------------
+# Load/store register pair (offset)
+#------------------------------------------------------------------------------
+
+ldnp      w3, w5, [sp]
+stnp      wzr, w9, [sp, #252]
+ldnp      w2, wzr, [sp, #-256]
+ldnp      w9, w10, [sp, #4]
+ldnp      x21, x29, [x2, #504]
+ldnp      x22, x23, [x3, #-512]
+ldnp      x24, x25, [x4, #8]
+ldnp      s29, s28, [sp, #252]
+stnp      s27, s26, [sp, #-256]
+ldnp      s1, s2, [x3, #44]
+stnp      d3, d5, [x9, #504]
+stnp      d7, d11, [x10, #-512]
+ldnp      d2, d3, [x30, #-8]
+stnp      q3, q5, [sp]
+stnp      q17, q19, [sp, #1008]
+ldnp      q23, q29, [x1, #-1024]
+
+#------------------------------------------------------------------------------
+# Logical (immediate)
+#------------------------------------------------------------------------------
+
+mov      w3, #983055
+mov      x10, #-6148914691236517206
+
+#------------------------------------------------------------------------------
+# Logical (shifted register)
+#------------------------------------------------------------------------------
+
+and      w12, w23, w21
+and      w16, w15, w1, lsl #1
+and      w9, w4, w10, lsl #31
+and      w3, w30, w11
+and      x3, x5, x7, lsl #63
+and      x5, x14, x19, asr #4
+and      w3, w17, w19, ror #31
+and      w0, w2, wzr, lsr #17
+and      w3, w30, w11, asr #2
+and      xzr, x4, x26
+and      w3, wzr, w20, ror #2
+and      x7, x20, xzr, asr #63
+bic      x13, x20, x14, lsl #47
+bic      w2, w7, w9
+orr      w2, w7, w0, asr #31
+orr      x8, x9, x10, lsl #12
+orn      x3, x5, x7, asr #2
+orn      w2, w5, w29
+ands     w7, wzr, w9, lsl #1
+ands     x3, x5, x20, ror #63
+bics     w3, w5, w7
+bics     x3, xzr, x3, lsl #1
+tst      w3, w7, lsl #31
+tst      x2, x20, asr #2
+mov      x3, x6
+mov      x3, xzr
+mov      wzr, w2
+mov      w3, w5
+
+#------------------------------------------------------------------------------
+# Move wide (immediate)
+#------------------------------------------------------------------------------
+
+movz     w2, #0, lsl #16
+mov     w2, #-1235
+mov     x2, #5299989643264
+mov      x2, #0
+movk     w3, #0
+movz     x4, #0, lsl #16
+movk     w5, #0, lsl #16
+movz     x6, #0, lsl #32
+movk     x7, #0, lsl #32
+movz     x8, #0, lsl #48
+movk     x9, #0, lsl #48
+
+#------------------------------------------------------------------------------
+# PC-relative addressing
+#------------------------------------------------------------------------------
+
+adr      x2, #1600
+adrp     x21, #6553600
+adr      x0, #262144
+
+#------------------------------------------------------------------------------
+# Test and branch (immediate)
+#------------------------------------------------------------------------------
+
+tbz     x12, #62, #0
+tbz     x12, #62, #4
+tbz     x12, #62, #-32768
+tbnz    x12, #60, #32764
+
+#------------------------------------------------------------------------------
+# Unconditional branch (immediate)
+#------------------------------------------------------------------------------
+
+b        #4
+b        #-4
+b        #134217724
+
+#------------------------------------------------------------------------------
+# Unconditional branch (register)
+#------------------------------------------------------------------------------
+
+br       x20
+blr      xzr
+ret      x10
+ret
+eret
+drps
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      1     0.33                        add	w2, w3, #4095
+# CHECK-NEXT:  1      1     0.33                        add	w30, w29, #1, lsl #12
+# CHECK-NEXT:  1      1     0.33                        add	w13, w5, #4095, lsl #12
+# CHECK-NEXT:  1      1     0.33                        add	x5, x7, #1638
+# CHECK-NEXT:  1      1     0.33                        add	w20, wsp, #801
+# CHECK-NEXT:  1      1     0.33                        add	wsp, wsp, #1104
+# CHECK-NEXT:  1      1     0.33                        add	wsp, w30, #4084
+# CHECK-NEXT:  1      1     0.33                        add	x0, x24, #291
+# CHECK-NEXT:  1      1     0.33                        add	x3, x24, #4095, lsl #12
+# CHECK-NEXT:  1      1     0.33                        add	x8, sp, #1074
+# CHECK-NEXT:  1      1     0.33                        add	sp, x29, #3816
+# CHECK-NEXT:  1      1     0.33                        sub	w0, wsp, #4077
+# CHECK-NEXT:  1      1     0.33                        sub	w4, w20, #546, lsl #12
+# CHECK-NEXT:  1      1     0.33                        sub	sp, sp, #288
+# CHECK-NEXT:  1      1     0.33                        sub	wsp, w19, #16
+# CHECK-NEXT:  1      1     0.33                        adds	w13, w23, #291, lsl #12
+# CHECK-NEXT:  1      1     0.33                        cmn	w2, #4095
+# CHECK-NEXT:  1      1     0.33                        adds	w20, wsp, #0
+# CHECK-NEXT:  1      1     0.33                        cmn	x3, #1, lsl #12
+# CHECK-NEXT:  1      1     0.33                        cmp	sp, #20, lsl #12
+# CHECK-NEXT:  1      1     0.33                        cmp	x30, #4095
+# CHECK-NEXT:  1      1     0.33                        subs	x4, sp, #3822
+# CHECK-NEXT:  1      1     0.33                        cmn	w3, #291, lsl #12
+# CHECK-NEXT:  1      1     0.33                        cmn	wsp, #1365
+# CHECK-NEXT:  1      1     0.33                        cmn	sp, #1092, lsl #12
+# CHECK-NEXT:  1      1     0.33                        mov	sp, x30
+# CHECK-NEXT:  1      1     0.33                        mov	wsp, w20
+# CHECK-NEXT:  1      1     0.33                        mov	x11, sp
+# CHECK-NEXT:  1      1     0.33                        mov	w24, wsp
+# CHECK-NEXT:  1      1     0.33                        add	w3, w5, w7
+# CHECK-NEXT:  1      1     0.33                        add	wzr, w3, w5
+# CHECK-NEXT:  1      1     0.33                        add	w20, wzr, w4
+# CHECK-NEXT:  1      1     0.33                        add	w4, w6, wzr
+# CHECK-NEXT:  1      1     0.33                        add	w11, w13, w15
+# CHECK-NEXT:  1      2     0.33                        add	w9, w3, wzr, lsl #10
+# CHECK-NEXT:  1      2     0.33                        add	w17, w29, w20, lsl #31
+# CHECK-NEXT:  1      2     0.33                        add	w21, w22, w23, lsr #0
+# CHECK-NEXT:  1      2     0.33                        add	w24, w25, w26, lsr #18
+# CHECK-NEXT:  1      2     0.33                        add	w27, w28, w29, lsr #31
+# CHECK-NEXT:  1      2     0.33                        add	w2, w3, w4, asr #0
+# CHECK-NEXT:  1      2     0.33                        add	w5, w6, w7, asr #21
+# CHECK-NEXT:  1      2     0.33                        add	w8, w9, w10, asr #31
+# CHECK-NEXT:  1      1     0.33                        add	x3, x5, x7
+# CHECK-NEXT:  1      1     0.33                        add	xzr, x3, x5
+# CHECK-NEXT:  1      1     0.33                        add	x20, xzr, x4
+# CHECK-NEXT:  1      1     0.33                        add	x4, x6, xzr
+# CHECK-NEXT:  1      1     0.33                        add	x11, x13, x15
+# CHECK-NEXT:  1      2     0.33                        add	x9, x3, xzr, lsl #10
+# CHECK-NEXT:  1      2     0.33                        add	x17, x29, x20, lsl #63
+# CHECK-NEXT:  1      2     0.33                        add	x21, x22, x23, lsr #0
+# CHECK-NEXT:  1      2     0.33                        add	x24, x25, x26, lsr #18
+# CHECK-NEXT:  1      2     0.33                        add	x27, x28, x29, lsr #63
+# CHECK-NEXT:  1      2     0.33                        add	x2, x3, x4, asr #0
+# CHECK-NEXT:  1      2     0.33                        add	x5, x6, x7, asr #21
+# CHECK-NEXT:  1      2     0.33                        add	x8, x9, x10, asr #63
+# CHECK-NEXT:  1      1     0.33                        adds	w3, w5, w7
+# CHECK-NEXT:  1      1     0.33                        cmn	w3, w5
+# CHECK-NEXT:  1      1     0.33                        adds	w20, wzr, w4
+# CHECK-NEXT:  1      1     0.33                        adds	w4, w6, wzr
+# CHECK-NEXT:  1      1     0.33                        adds	w11, w13, w15
+# CHECK-NEXT:  1      2     0.33                        adds	w9, w3, wzr, lsl #10
+# CHECK-NEXT:  1      2     0.33                        adds	w17, w29, w20, lsl #31
+# CHECK-NEXT:  1      2     0.33                        adds	w21, w22, w23, lsr #0
+# CHECK-NEXT:  1      2     0.33                        adds	w24, w25, w26, lsr #18
+# CHECK-NEXT:  1      2     0.33                        adds	w27, w28, w29, lsr #31
+# CHECK-NEXT:  1      2     0.33                        adds	w2, w3, w4, asr #0
+# CHECK-NEXT:  1      2     0.33                        adds	w5, w6, w7, asr #21
+# CHECK-NEXT:  1      2     0.33                        adds	w8, w9, w10, asr #31
+# CHECK-NEXT:  1      1     0.33                        adds	x3, x5, x7
+# CHECK-NEXT:  1      1     0.33                        cmn	x3, x5
+# CHECK-NEXT:  1      1     0.33                        adds	x20, xzr, x4
+# CHECK-NEXT:  1      1     0.33                        adds	x4, x6, xzr
+# CHECK-NEXT:  1      1     0.33                        adds	x11, x13, x15
+# CHECK-NEXT:  1      2     0.33                        adds	x9, x3, xzr, lsl #10
+# CHECK-NEXT:  1      2     0.33                        adds	x17, x29, x20, lsl #63
+# CHECK-NEXT:  1      2     0.33                        adds	x21, x22, x23, lsr #0
+# CHECK-NEXT:  1      2     0.33                        adds	x24, x25, x26, lsr #18
+# CHECK-NEXT:  1      2     0.33                        adds	x27, x28, x29, lsr #63
+# CHECK-NEXT:  1      2     0.33                        adds	x2, x3, x4, asr #0
+# CHECK-NEXT:  1      2     0.33                        adds	x5, x6, x7, asr #21
+# CHECK-NEXT:  1      2     0.33                        adds	x8, x9, x10, asr #63
+# CHECK-NEXT:  1      1     0.33                        sub	w3, w5, w7
+# CHECK-NEXT:  1      1     0.33                        sub	wzr, w3, w5
+# CHECK-NEXT:  1      1     0.33                        sub	w4, w6, wzr
+# CHECK-NEXT:  1      1     0.33                        sub	w11, w13, w15
+# CHECK-NEXT:  1      2     0.33                        sub	w9, w3, wzr, lsl #10
+# CHECK-NEXT:  1      2     0.33                        sub	w17, w29, w20, lsl #31
+# CHECK-NEXT:  1      2     0.33                        sub	w21, w22, w23, lsr #0
+# CHECK-NEXT:  1      2     0.33                        sub	w24, w25, w26, lsr #18
+# CHECK-NEXT:  1      2     0.33                        sub	w27, w28, w29, lsr #31
+# CHECK-NEXT:  1      2     0.33                        sub	w2, w3, w4, asr #0
+# CHECK-NEXT:  1      2     0.33                        sub	w5, w6, w7, asr #21
+# CHECK-NEXT:  1      2     0.33                        sub	w8, w9, w10, asr #31
+# CHECK-NEXT:  1      1     0.33                        sub	x3, x5, x7
+# CHECK-NEXT:  1      1     0.33                        sub	xzr, x3, x5
+# CHECK-NEXT:  1      1     0.33                        sub	x4, x6, xzr
+# CHECK-NEXT:  1      1     0.33                        sub	x11, x13, x15
+# CHECK-NEXT:  1      2     0.33                        sub	x9, x3, xzr, lsl #10
+# CHECK-NEXT:  1      2     0.33                        sub	x17, x29, x20, lsl #63
+# CHECK-NEXT:  1      2     0.33                        sub	x21, x22, x23, lsr #0
+# CHECK-NEXT:  1      2     0.33                        sub	x24, x25, x26, lsr #18
+# CHECK-NEXT:  1      2     0.33                        sub	x27, x28, x29, lsr #63
+# CHECK-NEXT:  1      2     0.33                        sub	x2, x3, x4, asr #0
+# CHECK-NEXT:  1      2     0.33                        sub	x5, x6, x7, asr #21
+# CHECK-NEXT:  1      2     0.33                        sub	x8, x9, x10, asr #63
+# CHECK-NEXT:  1      1     0.33                        subs	w3, w5, w7
+# CHECK-NEXT:  1      1     0.33                        cmp	w3, w5
+# CHECK-NEXT:  1      1     0.33                        subs	w4, w6, wzr
+# CHECK-NEXT:  1      1     0.33                        subs	w11, w13, w15
+# CHECK-NEXT:  1      2     0.33                        subs	w9, w3, wzr, lsl #10
+# CHECK-NEXT:  1      2     0.33                        subs	w17, w29, w20, lsl #31
+# CHECK-NEXT:  1      2     0.33                        subs	w21, w22, w23, lsr #0
+# CHECK-NEXT:  1      2     0.33                        subs	w24, w25, w26, lsr #18
+# CHECK-NEXT:  1      2     0.33                        subs	w27, w28, w29, lsr #31
+# CHECK-NEXT:  1      2     0.33                        subs	w2, w3, w4, asr #0
+# CHECK-NEXT:  1      2     0.33                        subs	w5, w6, w7, asr #21
+# CHECK-NEXT:  1      2     0.33                        subs	w8, w9, w10, asr #31
+# CHECK-NEXT:  1      1     0.33                        subs	x3, x5, x7
+# CHECK-NEXT:  1      1     0.33                        cmp	x3, x5
+# CHECK-NEXT:  1      1     0.33                        subs	x4, x6, xzr
+# CHECK-NEXT:  1      1     0.33                        subs	x11, x13, x15
+# CHECK-NEXT:  1      2     0.33                        subs	x9, x3, xzr, lsl #10
+# CHECK-NEXT:  1      2     0.33                        subs	x17, x29, x20, lsl #63
+# CHECK-NEXT:  1      2     0.33                        subs	x21, x22, x23, lsr #0
+# CHECK-NEXT:  1      2     0.33                        subs	x24, x25, x26, lsr #18
+# CHECK-NEXT:  1      2     0.33                        subs	x27, x28, x29, lsr #63
+# CHECK-NEXT:  1      2     0.33                        subs	x2, x3, x4, asr #0
+# CHECK-NEXT:  1      2     0.33                        subs	x5, x6, x7, asr #21
+# CHECK-NEXT:  1      2     0.33                        subs	x8, x9, x10, asr #63
+# CHECK-NEXT:  1      1     0.33                        cmn	wzr, w4
+# CHECK-NEXT:  1      1     0.33                        cmn	w5, wzr
+# CHECK-NEXT:  1      1     0.33                        cmn	w6, w7
+# CHECK-NEXT:  1      2     0.33                        cmn	w8, w9, lsl #15
+# CHECK-NEXT:  1      2     0.33                        cmn	w10, w11, lsl #31
+# CHECK-NEXT:  1      2     0.33                        cmn	w12, w13, lsr #0
+# CHECK-NEXT:  1      2     0.33                        cmn	w14, w15, lsr #21
+# CHECK-NEXT:  1      2     0.33                        cmn	w16, w17, lsr #31
+# CHECK-NEXT:  1      2     0.33                        cmn	w18, w19, asr #0
+# CHECK-NEXT:  1      2     0.33                        cmn	w20, w21, asr #22
+# CHECK-NEXT:  1      2     0.33                        cmn	w22, w23, asr #31
+# CHECK-NEXT:  1      1     0.33                        cmn	x0, x3
+# CHECK-NEXT:  1      1     0.33                        cmn	xzr, x4
+# CHECK-NEXT:  1      1     0.33                        cmn	x5, xzr
+# CHECK-NEXT:  1      1     0.33                        cmn	x6, x7
+# CHECK-NEXT:  1      2     0.33                        cmn	x8, x9, lsl #15
+# CHECK-NEXT:  1      2     0.33                        cmn	x10, x11, lsl #63
+# CHECK-NEXT:  1      2     0.33                        cmn	x12, x13, lsr #0
+# CHECK-NEXT:  1      2     0.33                        cmn	x14, x15, lsr #41
+# CHECK-NEXT:  1      2     0.33                        cmn	x16, x17, lsr #63
+# CHECK-NEXT:  1      2     0.33                        cmn	x18, x19, asr #0
+# CHECK-NEXT:  1      2     0.33                        cmn	x20, x21, asr #55
+# CHECK-NEXT:  1      2     0.33                        cmn	x22, x23, asr #63
+# CHECK-NEXT:  1      1     0.33                        cmp	w0, w3
+# CHECK-NEXT:  1      1     0.33                        cmp	wzr, w4
+# CHECK-NEXT:  1      1     0.33                        cmp	w5, wzr
+# CHECK-NEXT:  1      1     0.33                        cmp	w6, w7
+# CHECK-NEXT:  1      2     0.33                        cmp	w8, w9, lsl #15
+# CHECK-NEXT:  1      2     0.33                        cmp	w10, w11, lsl #31
+# CHECK-NEXT:  1      2     0.33                        cmp	w12, w13, lsr #0
+# CHECK-NEXT:  1      2     0.33                        cmp	w14, w15, lsr #21
+# CHECK-NEXT:  1      2     0.33                        cmp	w18, w19, asr #0
+# CHECK-NEXT:  1      2     0.33                        cmp	w20, w21, asr #22
+# CHECK-NEXT:  1      2     0.33                        cmp	w22, w23, asr #31
+# CHECK-NEXT:  1      1     0.33                        cmp	x0, x3
+# CHECK-NEXT:  1      1     0.33                        cmp	xzr, x4
+# CHECK-NEXT:  1      1     0.33                        cmp	x5, xzr
+# CHECK-NEXT:  1      1     0.33                        cmp	x6, x7
+# CHECK-NEXT:  1      2     0.33                        cmp	x8, x9, lsl #15
+# CHECK-NEXT:  1      2     0.33                        cmp	x10, x11, lsl #63
+# CHECK-NEXT:  1      2     0.33                        cmp	x12, x13, lsr #0
+# CHECK-NEXT:  1      2     0.33                        cmp	x14, x15, lsr #41
+# CHECK-NEXT:  1      2     0.33                        cmp	x16, x17, lsr #63
+# CHECK-NEXT:  1      2     0.33                        cmp	x18, x19, asr #0
+# CHECK-NEXT:  1      2     0.33                        cmp	x20, x21, asr #55
+# CHECK-NEXT:  1      2     0.33                        cmp	x22, x23, asr #63
+# CHECK-NEXT:  1      1     0.33                        cmp	wzr, w0
+# CHECK-NEXT:  1      1     0.33                        cmp	xzr, x0
+# CHECK-NEXT:  1      1     0.33                        adc	w29, w27, w25
+# CHECK-NEXT:  1      1     0.33                        adc	wzr, w3, w4
+# CHECK-NEXT:  1      1     0.33                        adc	w9, wzr, w10
+# CHECK-NEXT:  1      1     0.33                        adc	w20, w0, wzr
+# CHECK-NEXT:  1      1     0.33                        adc	x29, x27, x25
+# CHECK-NEXT:  1      1     0.33                        adc	xzr, x3, x4
+# CHECK-NEXT:  1      1     0.33                        adc	x9, xzr, x10
+# CHECK-NEXT:  1      1     0.33                        adc	x20, x0, xzr
+# CHECK-NEXT:  1      1     0.33                        adcs	w29, w27, w25
+# CHECK-NEXT:  1      1     0.33                        adcs	wzr, w3, w4
+# CHECK-NEXT:  1      1     0.33                        adcs	w9, wzr, w10
+# CHECK-NEXT:  1      1     0.33                        adcs	w20, w0, wzr
+# CHECK-NEXT:  1      1     0.33                        adcs	x29, x27, x25
+# CHECK-NEXT:  1      1     0.33                        adcs	xzr, x3, x4
+# CHECK-NEXT:  1      1     0.33                        adcs	x9, xzr, x10
+# CHECK-NEXT:  1      1     0.33                        adcs	x20, x0, xzr
+# CHECK-NEXT:  1      1     0.33                        sbc	w29, w27, w25
+# CHECK-NEXT:  1      1     0.33                        sbc	wzr, w3, w4
+# CHECK-NEXT:  1      1     0.33                        ngc	w9, w10
+# CHECK-NEXT:  1      1     0.33                        sbc	w20, w0, wzr
+# CHECK-NEXT:  1      1     0.33                        sbc	x29, x27, x25
+# CHECK-NEXT:  1      1     0.33                        sbc	xzr, x3, x4
+# CHECK-NEXT:  1      1     0.33                        ngc	x9, x10
+# CHECK-NEXT:  1      1     0.33                        sbc	x20, x0, xzr
+# CHECK-NEXT:  1      1     0.33                        sbcs	w29, w27, w25
+# CHECK-NEXT:  1      1     0.33                        sbcs	wzr, w3, w4
+# CHECK-NEXT:  1      1     0.33                        ngcs	w9, w10
+# CHECK-NEXT:  1      1     0.33                        sbcs	w20, w0, wzr
+# CHECK-NEXT:  1      1     0.33                        sbcs	x29, x27, x25
+# CHECK-NEXT:  1      1     0.33                        sbcs	xzr, x3, x4
+# CHECK-NEXT:  1      1     0.33                        ngcs	x9, x10
+# CHECK-NEXT:  1      1     0.33                        sbcs	x20, x0, xzr
+# CHECK-NEXT:  1      1     0.33                        ngc	w3, w12
+# CHECK-NEXT:  1      1     0.33                        ngc	wzr, w9
+# CHECK-NEXT:  1      1     0.33                        ngc	w23, wzr
+# CHECK-NEXT:  1      1     0.33                        ngc	x29, x30
+# CHECK-NEXT:  1      1     0.33                        ngc	xzr, x0
+# CHECK-NEXT:  1      1     0.33                        ngc	x0, xzr
+# CHECK-NEXT:  1      1     0.33                        ngcs	w3, w12
+# CHECK-NEXT:  1      1     0.33                        ngcs	wzr, w9
+# CHECK-NEXT:  1      1     0.33                        ngcs	w23, wzr
+# CHECK-NEXT:  1      1     0.33                        ngcs	x29, x30
+# CHECK-NEXT:  1      1     0.33                        ngcs	xzr, x0
+# CHECK-NEXT:  1      1     0.33                        ngcs	x0, xzr
+# CHECK-NEXT:  1      2     0.33                        sbfx	x1, x2, #3, #2
+# CHECK-NEXT:  1      2     0.33                        asr	x3, x4, #63
+# CHECK-NEXT:  1      2     0.33                        asr	wzr, wzr, #31
+# CHECK-NEXT:  1      2     0.33                        sbfx	w12, w9, #0, #1
+# CHECK-NEXT:  1      2     0.33                        ubfiz	x4, x5, #52, #11
+# CHECK-NEXT:  1      2     0.33                        ubfx	xzr, x4, #0, #1
+# CHECK-NEXT:  1      2     0.33                        ubfiz	x4, xzr, #1, #6
+# CHECK-NEXT:  1      2     0.33                        lsr	x5, x6, #12
+# CHECK-NEXT:  1      2     0.33                        bfi	x4, x5, #52, #11
+# CHECK-NEXT:  1      2     0.33                        bfxil	xzr, x4, #0, #1
+# CHECK-NEXT:  1      2     0.33                        bfc	x4, #1, #6
+# CHECK-NEXT:  1      2     0.33                        bfxil	x5, x6, #12, #52
+# CHECK-NEXT:  1      2     0.33                        sxtb	w1, w2
+# CHECK-NEXT:  1      2     0.33                        sxtb	xzr, w3
+# CHECK-NEXT:  1      2     0.33                        sxth	w9, w10
+# CHECK-NEXT:  1      2     0.33                        sxth	x0, w1
+# CHECK-NEXT:  1      2     0.33                        sxtw	x3, w30
+# CHECK-NEXT:  1      2     0.33                        uxtb	w1, w2
+# CHECK-NEXT:  1      2     0.33                        uxth	w9, w10
+# CHECK-NEXT:  1      2     0.33                        ubfx	x3, x30, #0, #32
+# CHECK-NEXT:  1      2     0.33                        asr	w3, w2, #0
+# CHECK-NEXT:  1      2     0.33                        asr	w9, w10, #31
+# CHECK-NEXT:  1      2     0.33                        asr	x20, x21, #63
+# CHECK-NEXT:  1      2     0.33                        asr	w1, wzr, #3
+# CHECK-NEXT:  1      2     0.33                        lsr	w3, w2, #0
+# CHECK-NEXT:  1      2     0.33                        lsr	w9, w10, #31
+# CHECK-NEXT:  1      2     0.33                        lsr	x20, x21, #63
+# CHECK-NEXT:  1      2     0.33                        lsr	wzr, wzr, #3
+# CHECK-NEXT:  1      2     0.33                        lsr	w3, w2, #0
+# CHECK-NEXT:  1      2     0.33                        lsl	w9, w10, #31
+# CHECK-NEXT:  1      2     0.33                        lsl	x20, x21, #63
+# CHECK-NEXT:  1      2     0.33                        lsl	w1, wzr, #3
+# CHECK-NEXT:  1      2     0.33                        sbfx	w9, w10, #0, #1
+# CHECK-NEXT:  1      2     0.33                        sbfiz	x2, x3, #63, #1
+# CHECK-NEXT:  1      2     0.33                        asr	x19, x20, #0
+# CHECK-NEXT:  1      2     0.33                        sbfiz	x9, x10, #5, #59
+# CHECK-NEXT:  1      2     0.33                        asr	w9, w10, #0
+# CHECK-NEXT:  1      2     0.33                        sbfiz	w11, w12, #31, #1
+# CHECK-NEXT:  1      2     0.33                        sbfiz	w13, w14, #29, #3
+# CHECK-NEXT:  1      2     0.33                        sbfiz	xzr, xzr, #10, #11
+# CHECK-NEXT:  1      2     0.33                        sbfx	w9, w10, #0, #1
+# CHECK-NEXT:  1      2     0.33                        asr	x2, x3, #63
+# CHECK-NEXT:  1      2     0.33                        asr	x19, x20, #0
+# CHECK-NEXT:  1      2     0.33                        asr	x9, x10, #5
+# CHECK-NEXT:  1      2     0.33                        asr	w9, w10, #0
+# CHECK-NEXT:  1      2     0.33                        asr	w11, w12, #31
+# CHECK-NEXT:  1      2     0.33                        asr	w13, w14, #29
+# CHECK-NEXT:  1      2     0.33                        sbfx	xzr, xzr, #10, #11
+# CHECK-NEXT:  1      2     0.33                        bfxil	w9, w10, #0, #1
+# CHECK-NEXT:  1      2     0.33                        bfi	x2, x3, #63, #1
+# CHECK-NEXT:  1      2     0.33                        bfxil	x19, x20, #0, #64
+# CHECK-NEXT:  1      2     0.33                        bfi	x9, x10, #5, #59
+# CHECK-NEXT:  1      2     0.33                        bfxil	w9, w10, #0, #32
+# CHECK-NEXT:  1      2     0.33                        bfi	w11, w12, #31, #1
+# CHECK-NEXT:  1      2     0.33                        bfi	w13, w14, #29, #3
+# CHECK-NEXT:  1      2     0.33                        bfc	xzr, #10, #11
+# CHECK-NEXT:  1      2     0.33                        bfxil	w9, w10, #0, #1
+# CHECK-NEXT:  1      2     0.33                        bfxil	x2, x3, #63, #1
+# CHECK-NEXT:  1      2     0.33                        bfxil	x19, x20, #0, #64
+# CHECK-NEXT:  1      2     0.33                        bfxil	x9, x10, #5, #59
+# CHECK-NEXT:  1      2     0.33                        bfxil	w9, w10, #0, #32
+# CHECK-NEXT:  1      2     0.33                        bfxil	w11, w12, #31, #1
+# CHECK-NEXT:  1      2     0.33                        bfxil	w13, w14, #29, #3
+# CHECK-NEXT:  1      2     0.33                        bfxil	xzr, xzr, #10, #11
+# CHECK-NEXT:  1      2     0.33                        ubfx	w9, w10, #0, #1
+# CHECK-NEXT:  1      2     0.33                        lsl	x2, x3, #63
+# CHECK-NEXT:  1      2     0.33                        lsr	x19, x20, #0
+# CHECK-NEXT:  1      2     0.33                        lsl	x9, x10, #5
+# CHECK-NEXT:  1      2     0.33                        lsr	w9, w10, #0
+# CHECK-NEXT:  1      2     0.33                        lsl	w11, w12, #31
+# CHECK-NEXT:  1      2     0.33                        lsl	w13, w14, #29
+# CHECK-NEXT:  1      2     0.33                        ubfiz	xzr, xzr, #10, #11
+# CHECK-NEXT:  1      2     0.33                        ubfx	w9, w10, #0, #1
+# CHECK-NEXT:  1      2     0.33                        lsr	x2, x3, #63
+# CHECK-NEXT:  1      2     0.33                        lsr	x19, x20, #0
+# CHECK-NEXT:  1      2     0.33                        lsr	x9, x10, #5
+# CHECK-NEXT:  1      2     0.33                        lsr	w9, w10, #0
+# CHECK-NEXT:  1      2     0.33                        lsr	w11, w12, #31
+# CHECK-NEXT:  1      2     0.33                        lsr	w13, w14, #29
+# CHECK-NEXT:  1      2     0.33                        ubfx	xzr, xzr, #10, #11
+# CHECK-NEXT:  1      1     1.00                        cbz	w5, #4
+# CHECK-NEXT:  1      1     1.00                        cbz	x5, #0
+# CHECK-NEXT:  1      1     1.00                        cbnz	x2, #-4
+# CHECK-NEXT:  1      1     1.00                        cbnz	x26, #1048572
+# CHECK-NEXT:  1      1     1.00                        cbz	wzr, #0
+# CHECK-NEXT:  1      1     1.00                        cbnz	xzr, #0
+# CHECK-NEXT:  1      1     1.00                        b.ne	#4
+# CHECK-NEXT:  1      1     1.00                        b.ge	#1048572
+# CHECK-NEXT:  1      1     1.00                        b.ge	#-4
+# CHECK-NEXT:  1      1     0.33                        ccmp	w1, #31, #0, eq
+# CHECK-NEXT:  1      1     0.33                        ccmp	w3, #0, #15, hs
+# CHECK-NEXT:  1      1     0.33                        ccmp	wzr, #15, #13, hs
+# CHECK-NEXT:  1      1     0.33                        ccmp	x9, #31, #0, le
+# CHECK-NEXT:  1      1     0.33                        ccmp	x3, #0, #15, gt
+# CHECK-NEXT:  1      1     0.33                        ccmp	xzr, #5, #7, ne
+# CHECK-NEXT:  1      1     0.33                        ccmn	w1, #31, #0, eq
+# CHECK-NEXT:  1      1     0.33                        ccmn	w3, #0, #15, hs
+# CHECK-NEXT:  1      1     0.33                        ccmn	wzr, #15, #13, hs
+# CHECK-NEXT:  1      1     0.33                        ccmn	x9, #31, #0, le
+# CHECK-NEXT:  1      1     0.33                        ccmn	x3, #0, #15, gt
+# CHECK-NEXT:  1      1     0.33                        ccmn	xzr, #5, #7, ne
+# CHECK-NEXT:  1      1     0.33                        ccmp	w1, wzr, #0, eq
+# CHECK-NEXT:  1      1     0.33                        ccmp	w3, w0, #15, hs
+# CHECK-NEXT:  1      1     0.33                        ccmp	wzr, w15, #13, hs
+# CHECK-NEXT:  1      1     0.33                        ccmp	x9, xzr, #0, le
+# CHECK-NEXT:  1      1     0.33                        ccmp	x3, x0, #15, gt
+# CHECK-NEXT:  1      1     0.33                        ccmp	xzr, x5, #7, ne
+# CHECK-NEXT:  1      1     0.33                        ccmn	w1, wzr, #0, eq
+# CHECK-NEXT:  1      1     0.33                        ccmn	w3, w0, #15, hs
+# CHECK-NEXT:  1      1     0.33                        ccmn	wzr, w15, #13, hs
+# CHECK-NEXT:  1      1     0.33                        ccmn	x9, xzr, #0, le
+# CHECK-NEXT:  1      1     0.33                        ccmn	x3, x0, #15, gt
+# CHECK-NEXT:  1      1     0.33                        ccmn	xzr, x5, #7, ne
+# CHECK-NEXT:  1      1     0.33                        csel	w1, w0, w19, ne
+# CHECK-NEXT:  1      1     0.33                        csel	wzr, w5, w9, eq
+# CHECK-NEXT:  1      1     0.33                        csel	w9, wzr, w30, gt
+# CHECK-NEXT:  1      1     0.33                        csel	w1, w28, wzr, mi
+# CHECK-NEXT:  1      1     0.33                        csel	x19, x23, x29, lt
+# CHECK-NEXT:  1      1     0.33                        csel	xzr, x3, x4, ge
+# CHECK-NEXT:  1      1     0.33                        csel	x5, xzr, x6, hs
+# CHECK-NEXT:  1      1     0.33                        csel	x7, x8, xzr, lo
+# CHECK-NEXT:  1      1     0.33                        csinc	w1, w0, w19, ne
+# CHECK-NEXT:  1      1     0.33                        csinc	wzr, w5, w9, eq
+# CHECK-NEXT:  1      1     0.33                        csinc	w9, wzr, w30, gt
+# CHECK-NEXT:  1      1     0.33                        csinc	w1, w28, wzr, mi
+# CHECK-NEXT:  1      1     0.33                        csinc	x19, x23, x29, lt
+# CHECK-NEXT:  1      1     0.33                        csinc	xzr, x3, x4, ge
+# CHECK-NEXT:  1      1     0.33                        csinc	x5, xzr, x6, hs
+# CHECK-NEXT:  1      1     0.33                        csinc	x7, x8, xzr, lo
+# CHECK-NEXT:  1      1     0.33                        csinv	w1, w0, w19, ne
+# CHECK-NEXT:  1      1     0.33                        csinv	wzr, w5, w9, eq
+# CHECK-NEXT:  1      1     0.33                        csinv	w9, wzr, w30, gt
+# CHECK-NEXT:  1      1     0.33                        csinv	w1, w28, wzr, mi
+# CHECK-NEXT:  1      1     0.33                        csinv	x19, x23, x29, lt
+# CHECK-NEXT:  1      1     0.33                        csinv	xzr, x3, x4, ge
+# CHECK-NEXT:  1      1     0.33                        csinv	x5, xzr, x6, hs
+# CHECK-NEXT:  1      1     0.33                        csinv	x7, x8, xzr, lo
+# CHECK-NEXT:  1      1     0.33                        csneg	w1, w0, w19, ne
+# CHECK-NEXT:  1      1     0.33                        csneg	wzr, w5, w9, eq
+# CHECK-NEXT:  1      1     0.33                        csneg	w9, wzr, w30, gt
+# CHECK-NEXT:  1      1     0.33                        csneg	w1, w28, wzr, mi
+# CHECK-NEXT:  1      1     0.33                        csneg	x19, x23, x29, lt
+# CHECK-NEXT:  1      1     0.33                        csneg	xzr, x3, x4, ge
+# CHECK-NEXT:  1      1     0.33                        csneg	x5, xzr, x6, hs
+# CHECK-NEXT:  1      1     0.33                        csneg	x7, x8, xzr, lo
+# CHECK-NEXT:  1      1     0.33                        cset	w3, eq
+# CHECK-NEXT:  1      1     0.33                        cset	x9, pl
+# CHECK-NEXT:  1      1     0.33                        csetm	w20, ne
+# CHECK-NEXT:  1      1     0.33                        csetm	x30, ge
+# CHECK-NEXT:  1      1     0.33                        csinc	w2, wzr, wzr, al
+# CHECK-NEXT:  1      1     0.33                        csinv	x3, xzr, xzr, nv
+# CHECK-NEXT:  1      1     0.33                        cinc	w3, w5, gt
+# CHECK-NEXT:  1      1     0.33                        cinc	wzr, w4, le
+# CHECK-NEXT:  1      1     0.33                        cset	w9, lt
+# CHECK-NEXT:  1      1     0.33                        cinc	x3, x5, gt
+# CHECK-NEXT:  1      1     0.33                        cinc	xzr, x4, le
+# CHECK-NEXT:  1      1     0.33                        cset	x9, lt
+# CHECK-NEXT:  1      1     0.33                        csinc	w5, w6, w6, nv
+# CHECK-NEXT:  1      1     0.33                        csinc	x1, x2, x2, al
+# CHECK-NEXT:  1      1     0.33                        cinv	w3, w5, gt
+# CHECK-NEXT:  1      1     0.33                        cinv	wzr, w4, le
+# CHECK-NEXT:  1      1     0.33                        csetm	w9, lt
+# CHECK-NEXT:  1      1     0.33                        cinv	x3, x5, gt
+# CHECK-NEXT:  1      1     0.33                        cinv	xzr, x4, le
+# CHECK-NEXT:  1      1     0.33                        csetm	x9, lt
+# CHECK-NEXT:  1      1     0.33                        csinv	x1, x0, x0, al
+# CHECK-NEXT:  1      1     0.33                        csinv	w9, w8, w8, nv
+# CHECK-NEXT:  1      1     0.33                        cneg	w3, w5, gt
+# CHECK-NEXT:  1      1     0.33                        cneg	wzr, w4, le
+# CHECK-NEXT:  1      1     0.33                        cneg	w9, wzr, lt
+# CHECK-NEXT:  1      1     0.33                        cneg	x3, x5, gt
+# CHECK-NEXT:  1      1     0.33                        cneg	xzr, x4, le
+# CHECK-NEXT:  1      1     0.33                        cneg	x9, xzr, lt
+# CHECK-NEXT:  1      1     0.33                        csneg	x4, x8, x8, al
+# CHECK-NEXT:  1      1     0.33                        csinv	w9, w8, w8, nv
+# CHECK-NEXT:  1      2     0.33                        rbit	w0, w7
+# CHECK-NEXT:  1      2     0.33                        rbit	x18, x3
+# CHECK-NEXT:  1      1     0.33                        rev16	w17, w1
+# CHECK-NEXT:  1      1     0.33                        rev16	x5, x2
+# CHECK-NEXT:  1      1     0.33                        rev	w18, w0
+# CHECK-NEXT:  1      1     0.33                        rev32	x20, x1
+# CHECK-NEXT:  1      1     0.33                        rev	x22, x2
+# CHECK-NEXT:  1      1     0.33                        clz	w24, w3
+# CHECK-NEXT:  1      1     0.33                        clz	x26, x4
+# CHECK-NEXT:  1      1     0.33                        cls	w3, w5
+# CHECK-NEXT:  1      1     0.33                        cls	x20, x5
+# CHECK-NEXT:  1      8     8.00                        udiv	w0, w7, w10
+# CHECK-NEXT:  1      16    16.00                       udiv	x9, x22, x4
+# CHECK-NEXT:  1      8     8.00                        sdiv	w12, w21, w0
+# CHECK-NEXT:  1      16    16.00                       sdiv	x13, x2, x1
+# CHECK-NEXT:  1      2     0.33                        lsl	w11, w12, w13
+# CHECK-NEXT:  1      2     0.33                        lsl	x14, x15, x16
+# CHECK-NEXT:  1      2     0.33                        lsr	w17, w18, w19
+# CHECK-NEXT:  1      2     0.33                        lsr	x20, x21, x22
+# CHECK-NEXT:  1      2     0.33                        asr	w23, w24, w25
+# CHECK-NEXT:  1      2     0.33                        asr	x26, x27, x28
+# CHECK-NEXT:  1      2     0.33                        ror	w0, w1, w2
+# CHECK-NEXT:  1      2     0.33                        ror	x3, x4, x5
+# CHECK-NEXT:  1      2     0.33                        lsl	w6, w7, w8
+# CHECK-NEXT:  1      2     0.33                        lsl	x9, x10, x11
+# CHECK-NEXT:  1      2     0.33                        lsr	w12, w13, w14
+# CHECK-NEXT:  1      2     0.33                        lsr	x15, x16, x17
+# CHECK-NEXT:  1      2     0.33                        asr	w18, w19, w20
+# CHECK-NEXT:  1      2     0.33                        asr	x21, x22, x23
+# CHECK-NEXT:  1      2     0.33                        ror	w24, w25, w26
+# CHECK-NEXT:  1      2     0.33                        ror	x27, x28, x29
+# CHECK-NEXT:  1      5     2.00                        smulh	x30, x29, x28
+# CHECK-NEXT:  1      5     2.00                        smulh	xzr, x27, x26
+# CHECK-NEXT:  1      5     2.00                        umulh	x30, x29, x28
+# CHECK-NEXT:  1      5     2.00                        umulh	x23, x30, xzr
+# CHECK-NEXT:  1      3     1.00                        madd	w1, w3, w7, w4
+# CHECK-NEXT:  1      3     1.00                        madd	wzr, w0, w9, w11
+# CHECK-NEXT:  1      3     1.00                        madd	w13, wzr, w4, w4
+# CHECK-NEXT:  1      3     1.00                        madd	w19, w30, wzr, w29
+# CHECK-NEXT:  1      3     1.00                        mul	w4, w5, w6
+# CHECK-NEXT:  1      5     2.00                        madd	x1, x3, x7, x4
+# CHECK-NEXT:  1      5     2.00                        madd	xzr, x0, x9, x11
+# CHECK-NEXT:  1      5     2.00                        madd	x13, xzr, x4, x4
+# CHECK-NEXT:  1      5     2.00                        madd	x19, x30, xzr, x29
+# CHECK-NEXT:  1      5     2.00                        mul	x4, x5, x6
+# CHECK-NEXT:  1      3     1.00                        msub	w1, w3, w7, w4
+# CHECK-NEXT:  1      3     1.00                        msub	wzr, w0, w9, w11
+# CHECK-NEXT:  1      3     1.00                        msub	w13, wzr, w4, w4
+# CHECK-NEXT:  1      3     1.00                        msub	w19, w30, wzr, w29
+# CHECK-NEXT:  1      3     1.00                        mneg	w4, w5, w6
+# CHECK-NEXT:  1      5     2.00                        msub	x1, x3, x7, x4
+# CHECK-NEXT:  1      5     2.00                        msub	xzr, x0, x9, x11
+# CHECK-NEXT:  1      5     2.00                        msub	x13, xzr, x4, x4
+# CHECK-NEXT:  1      5     2.00                        msub	x19, x30, xzr, x29
+# CHECK-NEXT:  1      5     2.00                        mneg	x4, x5, x6
+# CHECK-NEXT:  1      3     1.00                        smaddl	x3, w5, w2, x9
+# CHECK-NEXT:  1      3     1.00                        smaddl	xzr, w10, w11, x12
+# CHECK-NEXT:  1      3     1.00                        smaddl	x13, wzr, w14, x15
+# CHECK-NEXT:  1      3     1.00                        smaddl	x16, w17, wzr, x18
+# CHECK-NEXT:  1      3     1.00                        smull	x19, w20, w21
+# CHECK-NEXT:  1      3     1.00                        smsubl	x3, w5, w2, x9
+# CHECK-NEXT:  1      3     1.00                        smsubl	xzr, w10, w11, x12
+# CHECK-NEXT:  1      3     1.00                        smsubl	x13, wzr, w14, x15
+# CHECK-NEXT:  1      3     1.00                        smsubl	x16, w17, wzr, x18
+# CHECK-NEXT:  1      3     1.00                        smnegl	x19, w20, w21
+# CHECK-NEXT:  1      3     1.00                        umaddl	x3, w5, w2, x9
+# CHECK-NEXT:  1      3     1.00                        umaddl	xzr, w10, w11, x12
+# CHECK-NEXT:  1      3     1.00                        umaddl	x13, wzr, w14, x15
+# CHECK-NEXT:  1      3     1.00                        umaddl	x16, w17, wzr, x18
+# CHECK-NEXT:  1      3     1.00                        umull	x19, w20, w21
+# CHECK-NEXT:  1      3     1.00                        umsubl	x3, w5, w2, x9
+# CHECK-NEXT:  1      3     1.00                        umsubl	x16, w17, wzr, x18
+# CHECK-NEXT:  1      3     1.00                        umnegl	x19, w20, w21
+# CHECK-NEXT:  1      5     2.00                        smulh	x30, x29, x28
+# CHECK-NEXT:  1      5     2.00                        smulh	x23, x22, xzr
+# CHECK-NEXT:  1      5     2.00                        umulh	x23, x22, xzr
+# CHECK-NEXT:  1      5     2.00                        mul	x19, x20, xzr
+# CHECK-NEXT:  1      3     1.00                        mneg	w21, w22, w23
+# CHECK-NEXT:  1      3     1.00                        smull	x11, w13, w17
+# CHECK-NEXT:  1      3     1.00                        umull	x11, w13, w17
+# CHECK-NEXT:  1      3     1.00                        smnegl	x11, w13, w17
+# CHECK-NEXT:  1      3     1.00                        umnegl	x11, w13, w17
+# CHECK-NEXT:  1      2     0.33                        extr	w3, w5, w7, #0
+# CHECK-NEXT:  1      2     0.33                        extr	w11, w13, w17, #31
+# CHECK-NEXT:  1      2     0.33                        extr	x3, x5, x7, #15
+# CHECK-NEXT:  1      2     0.33                        extr	x11, x13, x17, #63
+# CHECK-NEXT:  1      2     0.33                        ror	x19, x23, #24
+# CHECK-NEXT:  1      2     0.33                        ror	x29, xzr, #63
+# CHECK-NEXT:  1      2     0.33                        ror	w9, w13, #31
+# CHECK-NEXT:  1      3     0.50                        fcmp	s3, s5
+# CHECK-NEXT:  1      3     0.50                        fcmp	s31, #0.0
+# CHECK-NEXT:  1      3     0.50                        fcmp	s31, #0.0
+# CHECK-NEXT:  1      3     0.50                        fcmpe	s29, s30
+# CHECK-NEXT:  1      3     0.50                        fcmpe	s15, #0.0
+# CHECK-NEXT:  1      3     0.50                        fcmpe	s15, #0.0
+# CHECK-NEXT:  1      3     0.50                        fcmp	d4, d12
+# CHECK-NEXT:  1      3     0.50                        fcmp	d23, #0.0
+# CHECK-NEXT:  1      3     0.50                        fcmp	d23, #0.0
+# CHECK-NEXT:  1      3     0.50                        fcmpe	d26, d22
+# CHECK-NEXT:  1      3     0.50                        fcmpe	d29, #0.0
+# CHECK-NEXT:  1      3     0.50                        fcmpe	d29, #0.0
+# CHECK-NEXT:  1      3     0.50                        fccmp	s1, s31, #0, eq
+# CHECK-NEXT:  1      3     0.50                        fccmp	s3, s0, #15, hs
+# CHECK-NEXT:  1      3     0.50                        fccmp	s31, s15, #13, hs
+# CHECK-NEXT:  1      3     0.50                        fccmp	d9, d31, #0, le
+# CHECK-NEXT:  1      3     0.50                        fccmp	d3, d0, #15, gt
+# CHECK-NEXT:  1      3     0.50                        fccmp	d31, d5, #7, ne
+# CHECK-NEXT:  1      3     0.50                        fccmpe	s1, s31, #0, eq
+# CHECK-NEXT:  1      3     0.50                        fccmpe	s3, s0, #15, hs
+# CHECK-NEXT:  1      3     0.50                        fccmpe	s31, s15, #13, hs
+# CHECK-NEXT:  1      3     0.50                        fccmpe	d9, d31, #0, le
+# CHECK-NEXT:  1      3     0.50                        fccmpe	d3, d0, #15, gt
+# CHECK-NEXT:  1      3     0.50                        fccmpe	d31, d5, #7, ne
+# CHECK-NEXT:  1      4     0.50                        fcsel	s3, s20, s9, pl
+# CHECK-NEXT:  1      4     0.50                        fcsel	d9, d10, d11, mi
+# CHECK-NEXT:  1      4     0.50                        fmov	s0, s1
+# CHECK-NEXT:  1      4     0.50                        fabs	s2, s3
+# CHECK-NEXT:  1      4     0.50                        fneg	s4, s5
+# CHECK-NEXT:  1      22    29.00                       fsqrt	s6, s7
+# CHECK-NEXT:  1      4     0.50                        fcvt	d8, s9
+# CHECK-NEXT:  1      4     0.50                        fcvt	h10, s11
+# CHECK-NEXT:  1      4     0.50                        frintn	s12, s13
+# CHECK-NEXT:  1      4     0.50                        frintp	s14, s15
+# CHECK-NEXT:  1      4     0.50                        frintm	s16, s17
+# CHECK-NEXT:  1      4     0.50                        frintz	s18, s19
+# CHECK-NEXT:  1      4     0.50                        frinta	s20, s21
+# CHECK-NEXT:  1      4     0.50                        frintx	s22, s23
+# CHECK-NEXT:  1      4     0.50                        frinti	s24, s25
+# CHECK-NEXT:  1      4     0.50                        fmov	d0, d1
+# CHECK-NEXT:  1      4     0.50                        fabs	d2, d3
+# CHECK-NEXT:  1      4     0.50                        fneg	d4, d5
+# CHECK-NEXT:  1      22    29.00                       fsqrt	d6, d7
+# CHECK-NEXT:  1      4     0.50                        fcvt	s8, d9
+# CHECK-NEXT:  1      4     0.50                        fcvt	h10, d11
+# CHECK-NEXT:  1      4     0.50                        frintn	d12, d13
+# CHECK-NEXT:  1      4     0.50                        frintp	d14, d15
+# CHECK-NEXT:  1      4     0.50                        frintm	d16, d17
+# CHECK-NEXT:  1      4     0.50                        frintz	d18, d19
+# CHECK-NEXT:  1      4     0.50                        frinta	d20, d21
+# CHECK-NEXT:  1      4     0.50                        frintx	d22, d23
+# CHECK-NEXT:  1      4     0.50                        frinti	d24, d25
+# CHECK-NEXT:  1      4     0.50                        fcvt	s26, h27
+# CHECK-NEXT:  1      4     0.50                        fcvt	d28, h29
+# CHECK-NEXT:  1      4     0.50                        fmul	s20, s19, s17
+# CHECK-NEXT:  1      13    10.00                       fdiv	s1, s2, s3
+# CHECK-NEXT:  1      4     0.50                        fadd	s4, s5, s6
+# CHECK-NEXT:  1      4     0.50                        fsub	s7, s8, s9
+# CHECK-NEXT:  1      4     0.50                        fmax	s10, s11, s12
+# CHECK-NEXT:  1      4     0.50                        fmin	s13, s14, s15
+# CHECK-NEXT:  1      4     0.50                        fmaxnm	s16, s17, s18
+# CHECK-NEXT:  1      4     0.50                        fminnm	s19, s20, s21
+# CHECK-NEXT:  1      4     0.50                        fnmul	s22, s23, s2
+# CHECK-NEXT:  1      4     0.50                        fmul	d20, d19, d17
+# CHECK-NEXT:  1      22    19.00                       fdiv	d1, d2, d3
+# CHECK-NEXT:  1      4     0.50                        fadd	d4, d5, d6
+# CHECK-NEXT:  1      4     0.50                        fsub	d7, d8, d9
+# CHECK-NEXT:  1      4     0.50                        fmax	d10, d11, d12
+# CHECK-NEXT:  1      4     0.50                        fmin	d13, d14, d15
+# CHECK-NEXT:  1      4     0.50                        fmaxnm	d16, d17, d18
+# CHECK-NEXT:  1      4     0.50                        fminnm	d19, d20, d21
+# CHECK-NEXT:  1      4     0.50                        fnmul	d22, d23, d24
+# CHECK-NEXT:  1      4     0.50                        fmadd	s3, s5, s6, s31
+# CHECK-NEXT:  1      4     0.50                        fmadd	d3, d13, d0, d23
+# CHECK-NEXT:  1      4     0.50                        fmsub	s3, s5, s6, s31
+# CHECK-NEXT:  1      4     0.50                        fmsub	d3, d13, d0, d23
+# CHECK-NEXT:  1      4     0.50                        fnmadd	s3, s5, s6, s31
+# CHECK-NEXT:  1      4     0.50                        fnmadd	d3, d13, d0, d23
+# CHECK-NEXT:  1      4     0.50                        fnmsub	s3, s5, s6, s31
+# CHECK-NEXT:  1      4     0.50                        fnmsub	d3, d13, d0, d23
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	w3, h5, #1
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	wzr, h20, #13
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	w19, h0, #32
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	x3, h5, #1
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	x12, h30, #45
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	x19, h0, #64
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	w3, s5, #1
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	wzr, s20, #13
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	w19, s0, #32
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	x3, s5, #1
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	x12, s30, #45
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	x19, s0, #64
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	w3, d5, #1
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	wzr, d20, #13
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	w19, d0, #32
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	x3, d5, #1
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	x12, d30, #45
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	x19, d0, #64
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	w3, h5, #1
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	wzr, h20, #13
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	w19, h0, #32
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	x3, h5, #1
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	x12, h30, #45
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	x19, h0, #64
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	w3, s5, #1
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	wzr, s20, #13
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	w19, s0, #32
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	x3, s5, #1
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	x12, s30, #45
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	x19, s0, #64
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	w3, d5, #1
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	wzr, d20, #13
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	w19, d0, #32
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	x3, d5, #1
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	x12, d30, #45
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	x19, d0, #64
+# CHECK-NEXT:  1      4     0.50                        scvtf	h23, w19, #1
+# CHECK-NEXT:  1      4     0.50                        scvtf	h31, wzr, #20
+# CHECK-NEXT:  1      4     0.50                        scvtf	h14, w0, #32
+# CHECK-NEXT:  1      4     0.50                        scvtf	h23, x19, #1
+# CHECK-NEXT:  1      4     0.50                        scvtf	h31, xzr, #20
+# CHECK-NEXT:  1      4     0.50                        scvtf	h14, x0, #64
+# CHECK-NEXT:  1      4     0.50                        scvtf	s23, w19, #1
+# CHECK-NEXT:  1      4     0.50                        scvtf	s31, wzr, #20
+# CHECK-NEXT:  1      4     0.50                        scvtf	s14, w0, #32
+# CHECK-NEXT:  1      4     0.50                        scvtf	s23, x19, #1
+# CHECK-NEXT:  1      4     0.50                        scvtf	s31, xzr, #20
+# CHECK-NEXT:  1      4     0.50                        scvtf	s14, x0, #64
+# CHECK-NEXT:  1      4     0.50                        scvtf	d23, w19, #1
+# CHECK-NEXT:  1      4     0.50                        scvtf	d31, wzr, #20
+# CHECK-NEXT:  1      4     0.50                        scvtf	d14, w0, #32
+# CHECK-NEXT:  1      4     0.50                        scvtf	d23, x19, #1
+# CHECK-NEXT:  1      4     0.50                        scvtf	d31, xzr, #20
+# CHECK-NEXT:  1      4     0.50                        scvtf	d14, x0, #64
+# CHECK-NEXT:  1      4     0.50                        ucvtf	h23, w19, #1
+# CHECK-NEXT:  1      4     0.50                        ucvtf	h31, wzr, #20
+# CHECK-NEXT:  1      4     0.50                        ucvtf	h14, w0, #32
+# CHECK-NEXT:  1      4     0.50                        ucvtf	h23, x19, #1
+# CHECK-NEXT:  1      4     0.50                        ucvtf	h31, xzr, #20
+# CHECK-NEXT:  1      4     0.50                        ucvtf	h14, x0, #64
+# CHECK-NEXT:  1      4     0.50                        ucvtf	s23, w19, #1
+# CHECK-NEXT:  1      4     0.50                        ucvtf	s31, wzr, #20
+# CHECK-NEXT:  1      4     0.50                        ucvtf	s14, w0, #32
+# CHECK-NEXT:  1      4     0.50                        ucvtf	s23, x19, #1
+# CHECK-NEXT:  1      4     0.50                        ucvtf	s31, xzr, #20
+# CHECK-NEXT:  1      4     0.50                        ucvtf	s14, x0, #64
+# CHECK-NEXT:  1      4     0.50                        ucvtf	d23, w19, #1
+# CHECK-NEXT:  1      4     0.50                        ucvtf	d31, wzr, #20
+# CHECK-NEXT:  1      4     0.50                        ucvtf	d14, w0, #32
+# CHECK-NEXT:  1      4     0.50                        ucvtf	d23, x19, #1
+# CHECK-NEXT:  1      4     0.50                        ucvtf	d31, xzr, #20
+# CHECK-NEXT:  1      4     0.50                        ucvtf	d14, x0, #64
+# CHECK-NEXT:  1      4     0.50                        fcvtns	w3, h31
+# CHECK-NEXT:  1      4     0.50                        fcvtns	xzr, h12
+# CHECK-NEXT:  1      4     0.50                        fcvtnu	wzr, h12
+# CHECK-NEXT:  1      4     0.50                        fcvtnu	x0, h0
+# CHECK-NEXT:  1      4     0.50                        fcvtps	wzr, h9
+# CHECK-NEXT:  1      4     0.50                        fcvtps	x12, h20
+# CHECK-NEXT:  1      4     0.50                        fcvtpu	w30, h23
+# CHECK-NEXT:  1      4     0.50                        fcvtpu	x29, h3
+# CHECK-NEXT:  1      4     0.50                        fcvtms	w2, h3
+# CHECK-NEXT:  1      4     0.50                        fcvtms	x4, h5
+# CHECK-NEXT:  1      4     0.50                        fcvtmu	w6, h7
+# CHECK-NEXT:  1      4     0.50                        fcvtmu	x8, h9
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	w10, h11
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	x12, h13
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	w14, h15
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	x15, h16
+# CHECK-NEXT:  1      4     0.50                        scvtf	h17, w18
+# CHECK-NEXT:  1      4     0.50                        scvtf	h19, x20
+# CHECK-NEXT:  1      4     0.50                        ucvtf	h21, w22
+# CHECK-NEXT:  1      4     0.50                        scvtf	h23, x24
+# CHECK-NEXT:  1      4     0.50                        fcvtas	w25, h26
+# CHECK-NEXT:  1      4     0.50                        fcvtas	x27, h28
+# CHECK-NEXT:  1      4     0.50                        fcvtau	w29, h30
+# CHECK-NEXT:  1      4     0.50                        fcvtau	xzr, h0
+# CHECK-NEXT:  1      4     0.50                        fcvtns	w3, s31
+# CHECK-NEXT:  1      4     0.50                        fcvtns	xzr, s12
+# CHECK-NEXT:  1      4     0.50                        fcvtnu	wzr, s12
+# CHECK-NEXT:  1      4     0.50                        fcvtnu	x0, s0
+# CHECK-NEXT:  1      4     0.50                        fcvtps	wzr, s9
+# CHECK-NEXT:  1      4     0.50                        fcvtps	x12, s20
+# CHECK-NEXT:  1      4     0.50                        fcvtpu	w30, s23
+# CHECK-NEXT:  1      4     0.50                        fcvtpu	x29, s3
+# CHECK-NEXT:  1      4     0.50                        fcvtms	w2, s3
+# CHECK-NEXT:  1      4     0.50                        fcvtms	x4, s5
+# CHECK-NEXT:  1      4     0.50                        fcvtmu	w6, s7
+# CHECK-NEXT:  1      4     0.50                        fcvtmu	x8, s9
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	w10, s11
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	x12, s13
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	w14, s15
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	x15, s16
+# CHECK-NEXT:  1      4     0.50                        scvtf	s17, w18
+# CHECK-NEXT:  1      4     0.50                        scvtf	s19, x20
+# CHECK-NEXT:  1      4     0.50                        ucvtf	s21, w22
+# CHECK-NEXT:  1      4     0.50                        scvtf	s23, x24
+# CHECK-NEXT:  1      4     0.50                        fcvtas	w25, s26
+# CHECK-NEXT:  1      4     0.50                        fcvtas	x27, s28
+# CHECK-NEXT:  1      4     0.50                        fcvtau	w29, s30
+# CHECK-NEXT:  1      4     0.50                        fcvtau	xzr, s0
+# CHECK-NEXT:  1      4     0.50                        fcvtns	w3, d31
+# CHECK-NEXT:  1      4     0.50                        fcvtns	xzr, d12
+# CHECK-NEXT:  1      4     0.50                        fcvtnu	wzr, d12
+# CHECK-NEXT:  1      4     0.50                        fcvtnu	x0, d0
+# CHECK-NEXT:  1      4     0.50                        fcvtps	wzr, d9
+# CHECK-NEXT:  1      4     0.50                        fcvtps	x12, d20
+# CHECK-NEXT:  1      4     0.50                        fcvtpu	w30, d23
+# CHECK-NEXT:  1      4     0.50                        fcvtpu	x29, d3
+# CHECK-NEXT:  1      4     0.50                        fcvtms	w2, d3
+# CHECK-NEXT:  1      4     0.50                        fcvtms	x4, d5
+# CHECK-NEXT:  1      4     0.50                        fcvtmu	w6, d7
+# CHECK-NEXT:  1      4     0.50                        fcvtmu	x8, d9
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	w10, d11
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	x12, d13
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	w14, d15
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	x15, d16
+# CHECK-NEXT:  1      4     0.50                        scvtf	d17, w18
+# CHECK-NEXT:  1      4     0.50                        scvtf	d19, x20
+# CHECK-NEXT:  1      4     0.50                        ucvtf	d21, w22
+# CHECK-NEXT:  1      4     0.50                        ucvtf	d23, x24
+# CHECK-NEXT:  1      4     0.50                        fcvtas	w25, d26
+# CHECK-NEXT:  1      4     0.50                        fcvtas	x27, d28
+# CHECK-NEXT:  1      4     0.50                        fcvtau	w29, d30
+# CHECK-NEXT:  1      4     0.50                        fcvtau	xzr, d0
+# CHECK-NEXT:  1      3     0.50                        fmov	w3, s9
+# CHECK-NEXT:  1      3     0.50                        fmov	s9, w3
+# CHECK-NEXT:  1      3     0.50                        fmov	x20, d31
+# CHECK-NEXT:  1      3     0.50                        fmov	d1, x15
+# CHECK-NEXT:  1      3     0.50                        fmov	x3, v12.d[1]
+# CHECK-NEXT:  1      3     0.50                        fmov	v1.d[1], x19
+# CHECK-NEXT:  1      3     0.50                        fmov	s2, #0.12500000
+# CHECK-NEXT:  1      3     0.50                        fmov	s3, #1.00000000
+# CHECK-NEXT:  1      3     0.50                        fmov	d30, #16.00000000
+# CHECK-NEXT:  1      3     0.50                        fmov	s4, #1.06250000
+# CHECK-NEXT:  1      3     0.50                        fmov	d10, #1.93750000
+# CHECK-NEXT:  1      3     0.50                        fmov	s12, #-1.00000000
+# CHECK-NEXT:  1      3     0.50                        fmov	d16, #8.50000000
+# CHECK-NEXT:  1      2     0.50    *                   ldr	w3, #0
+# CHECK-NEXT:  1      2     0.50    *                   ldr	x29, #4
+# CHECK-NEXT:  1      2     0.50    *                   ldrsw	xzr, #-4
+# CHECK-NEXT:  1      2     0.50    *                   ldr	s0, #8
+# CHECK-NEXT:  1      2     0.50    *                   ldr	d0, #1048572
+# CHECK-NEXT:  1      2     0.50    *                   ldr	q0, #-1048576
+# CHECK-NEXT:  1      2     0.50                  U     prfm	pldl1strm, #0
+# CHECK-NEXT:  1      2     0.50                  U     prfm	#22, #0
+# CHECK-NEXT:  2      3     1.00    *      *      U     stxrb	w18, w8, [sp]
+# CHECK-NEXT:  2      3     1.00    *      *      U     stxrh	w24, w15, [x16]
+# CHECK-NEXT:  2      3     1.00    *      *      U     stxr	w5, w6, [x17]
+# CHECK-NEXT:  2      3     1.00    *      *      U     stxr	w1, x10, [x21]
+# CHECK-NEXT:  1      2     0.50    *      *      U     ldxrb	w30, [x0]
+# CHECK-NEXT:  1      2     0.50    *      *      U     ldxrh	w17, [x4]
+# CHECK-NEXT:  1      2     0.50    *      *      U     ldxr	w22, [sp]
+# CHECK-NEXT:  1      2     0.50    *      *      U     ldxr	x11, [x29]
+# CHECK-NEXT:  1      2     0.50    *      *      U     ldxr	x11, [x29]
+# CHECK-NEXT:  1      2     0.50    *      *      U     ldxr	x11, [x29]
+# CHECK-NEXT:  2      3     1.00    *      *      U     stxp	w12, w11, w10, [sp]
+# CHECK-NEXT:  2      3     1.00    *      *      U     stxp	wzr, x27, x9, [x12]
+# CHECK-NEXT:  2      2     1.00    *      *      U     ldxp	w0, wzr, [sp]
+# CHECK-NEXT:  2      2     1.00    *      *      U     ldxp	x17, x0, [x18]
+# CHECK-NEXT:  2      2     1.00    *      *      U     ldxp	x17, x0, [x18]
+# CHECK-NEXT:  2      3     1.00    *      *      U     stlxrb	w12, w22, [x0]
+# CHECK-NEXT:  2      3     1.00    *      *      U     stlxrh	w10, w1, [x1]
+# CHECK-NEXT:  2      3     1.00    *      *      U     stlxr	w9, w2, [x2]
+# CHECK-NEXT:  2      3     1.00    *      *      U     stlxr	w9, x3, [sp]
+# CHECK-NEXT:  1      2     0.50    *      *      U     ldaxrb	w8, [x4]
+# CHECK-NEXT:  1      2     0.50    *      *      U     ldaxrh	w7, [x5]
+# CHECK-NEXT:  1      2     0.50    *      *      U     ldaxr	w6, [sp]
+# CHECK-NEXT:  1      2     0.50    *      *      U     ldaxr	x5, [x6]
+# CHECK-NEXT:  1      2     0.50    *      *      U     ldaxr	x5, [x6]
+# CHECK-NEXT:  1      2     0.50    *      *      U     ldaxr	x5, [x6]
+# CHECK-NEXT:  2      3     1.00    *      *      U     stlxp	w4, w5, w6, [sp]
+# CHECK-NEXT:  2      3     1.00    *      *      U     stlxp	wzr, x6, x7, [x1]
+# CHECK-NEXT:  2      2     1.00    *      *      U     ldaxp	w5, w18, [sp]
+# CHECK-NEXT:  2      2     1.00    *      *      U     ldaxp	x6, x19, [x22]
+# CHECK-NEXT:  2      2     1.00    *      *      U     ldaxp	x6, x19, [x22]
+# CHECK-NEXT:  1      1     1.00           *      U     stlrb	w24, [sp]
+# CHECK-NEXT:  1      1     1.00           *      U     stlrh	w25, [x30]
+# CHECK-NEXT:  1      1     1.00           *      U     stlr	w26, [x29]
+# CHECK-NEXT:  1      1     1.00           *      U     stlr	x27, [x28]
+# CHECK-NEXT:  1      1     1.00           *      U     stlr	x27, [x28]
+# CHECK-NEXT:  1      1     1.00           *      U     stlr	x27, [x28]
+# CHECK-NEXT:  1      2     0.50    *             U     ldarb	w23, [sp]
+# CHECK-NEXT:  1      2     0.50    *             U     ldarh	w22, [x30]
+# CHECK-NEXT:  1      2     0.50    *             U     ldar	wzr, [x29]
+# CHECK-NEXT:  1      2     0.50    *             U     ldar	x21, [x28]
+# CHECK-NEXT:  1      2     0.50    *             U     ldar	x21, [x28]
+# CHECK-NEXT:  1      2     0.50    *             U     ldar	x21, [x28]
+# CHECK-NEXT:  1      1     1.00           *            sturb	w9, [sp]
+# CHECK-NEXT:  1      1     1.00           *            sturh	wzr, [x12, #255]
+# CHECK-NEXT:  1      1     1.00           *            stur	w16, [x0, #-256]
+# CHECK-NEXT:  1      1     1.00           *            stur	x28, [x14, #1]
+# CHECK-NEXT:  1      2     0.50    *                   ldurb	w1, [x20, #255]
+# CHECK-NEXT:  1      2     0.50    *                   ldurh	w20, [x1, #255]
+# CHECK-NEXT:  1      2     0.50    *                   ldur	w12, [sp, #255]
+# CHECK-NEXT:  1      2     0.50    *                   ldur	xzr, [x12, #255]
+# CHECK-NEXT:  1      2     0.50    *                   ldursb	x9, [x7, #-256]
+# CHECK-NEXT:  1      2     0.50    *                   ldursh	x17, [x19, #-256]
+# CHECK-NEXT:  1      2     0.50    *                   ldursw	x20, [x15, #-256]
+# CHECK-NEXT:  1      2     0.50                  U     prfum	pldl2keep, [sp, #-256]
+# CHECK-NEXT:  1      2     0.50    *                   ldursb	w19, [x1, #-256]
+# CHECK-NEXT:  1      2     0.50    *                   ldursh	w15, [x21, #-256]
+# CHECK-NEXT:  1      1     1.00           *            stur	b0, [sp, #1]
+# CHECK-NEXT:  1      1     1.00           *            stur	h12, [x12, #-1]
+# CHECK-NEXT:  1      1     1.00           *            stur	s15, [x0, #255]
+# CHECK-NEXT:  1      1     1.00           *            stur	d31, [x5, #25]
+# CHECK-NEXT:  1      1     1.00           *            stur	q9, [x5]
+# CHECK-NEXT:  1      2     0.50    *                   ldur	b3, [sp]
+# CHECK-NEXT:  1      2     0.50    *                   ldur	h5, [x4, #-256]
+# CHECK-NEXT:  1      2     0.50    *                   ldur	s7, [x12, #-1]
+# CHECK-NEXT:  1      2     0.50    *                   ldur	d11, [x19, #4]
+# CHECK-NEXT:  1      2     0.50    *                   ldur	q13, [x1, #2]
+# CHECK-NEXT:  2      1     1.00           *            strb	w9, [x2], #255
+# CHECK-NEXT:  2      1     1.00           *            strb	w10, [x3], #1
+# CHECK-NEXT:  2      1     1.00           *            strb	w10, [x3], #-256
+# CHECK-NEXT:  2      1     1.00           *            strh	w9, [x2], #255
+# CHECK-NEXT:  2      1     1.00           *            strh	w9, [x2], #1
+# CHECK-NEXT:  2      1     1.00           *            strh	w10, [x3], #-256
+# CHECK-NEXT:  2      1     1.00           *            str	w19, [sp], #255
+# CHECK-NEXT:  2      1     1.00           *            str	w20, [x30], #1
+# CHECK-NEXT:  2      1     1.00           *            str	w21, [x12], #-256
+# CHECK-NEXT:  2      1     1.00           *            str	xzr, [x9], #255
+# CHECK-NEXT:  2      1     1.00           *            str	x2, [x3], #1
+# CHECK-NEXT:  2      1     1.00           *            str	x19, [x12], #-256
+# CHECK-NEXT:  2      2     0.50    *                   ldrb	w9, [x2], #255
+# CHECK-NEXT:  2      2     0.50    *                   ldrb	w10, [x3], #1
+# CHECK-NEXT:  2      2     0.50    *                   ldrb	w10, [x3], #-256
+# CHECK-NEXT:  2      2     0.50    *                   ldrh	w9, [x2], #255
+# CHECK-NEXT:  2      2     0.50    *                   ldrh	w9, [x2], #1
+# CHECK-NEXT:  2      2     0.50    *                   ldrh	w10, [x3], #-256
+# CHECK-NEXT:  2      2     0.50    *                   ldr	w19, [sp], #255
+# CHECK-NEXT:  2      2     0.50    *                   ldr	w20, [x30], #1
+# CHECK-NEXT:  2      2     0.50    *                   ldr	w21, [x12], #-256
+# CHECK-NEXT:  2      2     0.50    *                   ldr	xzr, [x9], #255
+# CHECK-NEXT:  2      2     0.50    *                   ldr	x2, [x3], #1
+# CHECK-NEXT:  2      2     0.50    *                   ldr	x19, [x12], #-256
+# CHECK-NEXT:  2      2     0.50    *                   ldrsb	xzr, [x9], #255
+# CHECK-NEXT:  2      2     0.50    *                   ldrsb	x2, [x3], #1
+# CHECK-NEXT:  2      2     0.50    *                   ldrsb	x19, [x12], #-256
+# CHECK-NEXT:  2      2     0.50    *                   ldrsh	xzr, [x9], #255
+# CHECK-NEXT:  2      2     0.50    *                   ldrsh	x2, [x3], #1
+# CHECK-NEXT:  2      2     0.50    *                   ldrsh	x19, [x12], #-256
+# CHECK-NEXT:  2      2     0.50    *                   ldrsw	xzr, [x9], #255
+# CHECK-NEXT:  2      2     0.50    *                   ldrsw	x2, [x3], #1
+# CHECK-NEXT:  2      2     0.50    *                   ldrsw	x19, [x12], #-256
+# CHECK-NEXT:  2      2     0.50    *                   ldrsb	wzr, [x9], #255
+# CHECK-NEXT:  2      2     0.50    *                   ldrsb	w2, [x3], #1
+# CHECK-NEXT:  2      2     0.50    *                   ldrsb	w19, [x12], #-256
+# CHECK-NEXT:  2      2     0.50    *                   ldrsh	wzr, [x9], #255
+# CHECK-NEXT:  2      2     0.50    *                   ldrsh	w2, [x3], #1
+# CHECK-NEXT:  2      2     0.50    *                   ldrsh	w19, [x12], #-256
+# CHECK-NEXT:  2      1     1.00           *            str	b0, [x0], #255
+# CHECK-NEXT:  2      1     1.00           *            str	b3, [x3], #1
+# CHECK-NEXT:  2      1     1.00           *            str	b5, [sp], #-256
+# CHECK-NEXT:  2      1     1.00           *            str	h10, [x10], #255
+# CHECK-NEXT:  2      1     1.00           *            str	h13, [x23], #1
+# CHECK-NEXT:  2      1     1.00           *            str	h15, [sp], #-256
+# CHECK-NEXT:  2      1     1.00           *            str	s20, [x20], #255
+# CHECK-NEXT:  2      1     1.00           *            str	s23, [x23], #1
+# CHECK-NEXT:  2      1     1.00           *            str	s25, [x0], #-256
+# CHECK-NEXT:  2      1     1.00           *            str	d20, [x20], #255
+# CHECK-NEXT:  2      1     1.00           *            str	d23, [x23], #1
+# CHECK-NEXT:  2      1     1.00           *            str	d25, [x0], #-256
+# CHECK-NEXT:  2      2     0.50    *                   ldr	b0, [x0], #255
+# CHECK-NEXT:  2      2     0.50    *                   ldr	b3, [x3], #1
+# CHECK-NEXT:  2      2     0.50    *                   ldr	b5, [sp], #-256
+# CHECK-NEXT:  2      2     0.50    *                   ldr	h10, [x10], #255
+# CHECK-NEXT:  2      2     0.50    *                   ldr	h13, [x23], #1
+# CHECK-NEXT:  2      2     0.50    *                   ldr	h15, [sp], #-256
+# CHECK-NEXT:  2      2     0.50    *                   ldr	s20, [x20], #255
+# CHECK-NEXT:  2      2     0.50    *                   ldr	s23, [x23], #1
+# CHECK-NEXT:  2      2     0.50    *                   ldr	s25, [x0], #-256
+# CHECK-NEXT:  2      2     0.50    *                   ldr	d20, [x20], #255
+# CHECK-NEXT:  2      2     0.50    *                   ldr	d23, [x23], #1
+# CHECK-NEXT:  2      2     0.50    *                   ldr	d25, [x0], #-256
+# CHECK-NEXT:  2      2     0.50    *                   ldr	q20, [x1], #255
+# CHECK-NEXT:  2      2     0.50    *                   ldr	q23, [x9], #1
+# CHECK-NEXT:  2      2     0.50    *                   ldr	q25, [x20], #-256
+# CHECK-NEXT:  2      1     1.00           *            str	q10, [x1], #255
+# CHECK-NEXT:  2      1     1.00           *            str	q22, [sp], #1
+# CHECK-NEXT:  2      1     1.00           *            str	q21, [x20], #-256
+# CHECK-NEXT:  2      2     0.50    *                   ldr	x3, [x4, #0]!
+# CHECK-NEXT:  2      1     1.00           *            strb	w9, [x2, #255]!
+# CHECK-NEXT:  2      1     1.00           *            strb	w10, [x3, #1]!
+# CHECK-NEXT:  2      1     1.00           *            strb	w10, [x3, #-256]!
+# CHECK-NEXT:  2      1     1.00           *            strh	w9, [x2, #255]!
+# CHECK-NEXT:  2      1     1.00           *            strh	w9, [x2, #1]!
+# CHECK-NEXT:  2      1     1.00           *            strh	w10, [x3, #-256]!
+# CHECK-NEXT:  2      1     1.00           *            str	w19, [sp, #255]!
+# CHECK-NEXT:  2      1     1.00           *            str	w20, [x30, #1]!
+# CHECK-NEXT:  2      1     1.00           *            str	w21, [x12, #-256]!
+# CHECK-NEXT:  2      1     1.00           *            str	xzr, [x9, #255]!
+# CHECK-NEXT:  2      1     1.00           *            str	x2, [x3, #1]!
+# CHECK-NEXT:  2      1     1.00           *            str	x19, [x12, #-256]!
+# CHECK-NEXT:  2      2     0.50    *                   ldrb	w9, [x2, #255]!
+# CHECK-NEXT:  2      2     0.50    *                   ldrb	w10, [x3, #1]!
+# CHECK-NEXT:  2      2     0.50    *                   ldrb	w10, [x3, #-256]!
+# CHECK-NEXT:  2      2     0.50    *                   ldrh	w9, [x2, #255]!
+# CHECK-NEXT:  2      2     0.50    *                   ldrh	w9, [x2, #1]!
+# CHECK-NEXT:  2      2     0.50    *                   ldrh	w10, [x3, #-256]!
+# CHECK-NEXT:  2      2     0.50    *                   ldr	w19, [sp, #255]!
+# CHECK-NEXT:  2      2     0.50    *                   ldr	w20, [x30, #1]!
+# CHECK-NEXT:  2      2     0.50    *                   ldr	w21, [x12, #-256]!
+# CHECK-NEXT:  2      2     0.50    *                   ldr	xzr, [x9, #255]!
+# CHECK-NEXT:  2      2     0.50    *                   ldr	x2, [x3, #1]!
+# CHECK-NEXT:  2      2     0.50    *                   ldr	x19, [x12, #-256]!
+# CHECK-NEXT:  2      2     0.50    *                   ldrsb	xzr, [x9, #255]!
+# CHECK-NEXT:  2      2     0.50    *                   ldrsb	x2, [x3, #1]!
+# CHECK-NEXT:  2      2     0.50    *                   ldrsb	x19, [x12, #-256]!
+# CHECK-NEXT:  2      2     0.50    *                   ldrsh	xzr, [x9, #255]!
+# CHECK-NEXT:  2      2     0.50    *                   ldrsh	x2, [x3, #1]!
+# CHECK-NEXT:  2      2     0.50    *                   ldrsh	x19, [x12, #-256]!
+# CHECK-NEXT:  2      2     0.50    *                   ldrsw	xzr, [x9, #255]!
+# CHECK-NEXT:  2      2     0.50    *                   ldrsw	x2, [x3, #1]!
+# CHECK-NEXT:  2      2     0.50    *                   ldrsw	x19, [x12, #-256]!
+# CHECK-NEXT:  2      2     0.50    *                   ldrsb	wzr, [x9, #255]!
+# CHECK-NEXT:  2      2     0.50    *                   ldrsb	w2, [x3, #1]!
+# CHECK-NEXT:  2      2     0.50    *                   ldrsb	w19, [x12, #-256]!
+# CHECK-NEXT:  2      2     0.50    *                   ldrsh	wzr, [x9, #255]!
+# CHECK-NEXT:  2      2     0.50    *                   ldrsh	w2, [x3, #1]!
+# CHECK-NEXT:  2      2     0.50    *                   ldrsh	w19, [x12, #-256]!
+# CHECK-NEXT:  2      1     1.00           *            str	b0, [x0, #255]!
+# CHECK-NEXT:  2      1     1.00           *            str	b3, [x3, #1]!
+# CHECK-NEXT:  2      1     1.00           *            str	b5, [sp, #-256]!
+# CHECK-NEXT:  2      1     1.00           *            str	h10, [x10, #255]!
+# CHECK-NEXT:  2      1     1.00           *            str	h13, [x23, #1]!
+# CHECK-NEXT:  2      1     1.00           *            str	h15, [sp, #-256]!
+# CHECK-NEXT:  2      1     1.00           *            str	s20, [x20, #255]!
+# CHECK-NEXT:  2      1     1.00           *            str	s23, [x23, #1]!
+# CHECK-NEXT:  2      1     1.00           *            str	s25, [x0, #-256]!
+# CHECK-NEXT:  2      1     1.00           *            str	d20, [x20, #255]!
+# CHECK-NEXT:  2      1     1.00           *            str	d23, [x23, #1]!
+# CHECK-NEXT:  2      1     1.00           *            str	d25, [x0, #-256]!
+# CHECK-NEXT:  2      2     0.50    *                   ldr	b0, [x0, #255]!
+# CHECK-NEXT:  2      2     0.50    *                   ldr	b3, [x3, #1]!
+# CHECK-NEXT:  2      2     0.50    *                   ldr	b5, [sp, #-256]!
+# CHECK-NEXT:  2      2     0.50    *                   ldr	h10, [x10, #255]!
+# CHECK-NEXT:  2      2     0.50    *                   ldr	h13, [x23, #1]!
+# CHECK-NEXT:  2      2     0.50    *                   ldr	h15, [sp, #-256]!
+# CHECK-NEXT:  2      2     0.50    *                   ldr	s20, [x20, #255]!
+# CHECK-NEXT:  2      2     0.50    *                   ldr	s23, [x23, #1]!
+# CHECK-NEXT:  2      2     0.50    *                   ldr	s25, [x0, #-256]!
+# CHECK-NEXT:  2      2     0.50    *                   ldr	d20, [x20, #255]!
+# CHECK-NEXT:  2      2     0.50    *                   ldr	d23, [x23, #1]!
+# CHECK-NEXT:  2      2     0.50    *                   ldr	d25, [x0, #-256]!
+# CHECK-NEXT:  2      2     0.50    *                   ldr	q20, [x1, #255]!
+# CHECK-NEXT:  2      2     0.50    *                   ldr	q23, [x9, #1]!
+# CHECK-NEXT:  2      2     0.50    *                   ldr	q25, [x20, #-256]!
+# CHECK-NEXT:  2      1     1.00           *            str	q10, [x1, #255]!
+# CHECK-NEXT:  2      1     1.00           *            str	q22, [sp, #1]!
+# CHECK-NEXT:  2      1     1.00           *            str	q21, [x20, #-256]!
+# CHECK-NEXT:  1      1     1.00           *            sttrb	w9, [sp]
+# CHECK-NEXT:  1      1     1.00           *            sttrh	wzr, [x12, #255]
+# CHECK-NEXT:  1      1     1.00           *            sttr	w16, [x0, #-256]
+# CHECK-NEXT:  1      1     1.00           *            sttr	x28, [x14, #1]
+# CHECK-NEXT:  1      2     0.50    *                   ldtrb	w1, [x20, #255]
+# CHECK-NEXT:  1      2     0.50    *                   ldtrh	w20, [x1, #255]
+# CHECK-NEXT:  1      2     0.50    *                   ldtr	w12, [sp, #255]
+# CHECK-NEXT:  1      2     0.50    *                   ldtr	xzr, [x12, #255]
+# CHECK-NEXT:  1      2     0.50    *                   ldtrsb	x9, [x7, #-256]
+# CHECK-NEXT:  1      2     0.50    *                   ldtrsh	x17, [x19, #-256]
+# CHECK-NEXT:  1      2     0.50    *                   ldtrsw	x20, [x15, #-256]
+# CHECK-NEXT:  1      2     0.50    *                   ldtrsb	w19, [x1, #-256]
+# CHECK-NEXT:  1      2     0.50    *                   ldtrsh	w15, [x21, #-256]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	x4, [x29]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	x30, [x12, #32760]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	x20, [sp, #8]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	xzr, [sp]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	w2, [sp]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	w17, [sp, #16380]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	w13, [x2, #4]
+# CHECK-NEXT:  1      2     0.50    *                   ldrsw	x2, [x5, #4]
+# CHECK-NEXT:  1      2     0.50    *                   ldrsw	x23, [sp, #16380]
+# CHECK-NEXT:  1      2     0.50    *                   ldrh	w2, [x4]
+# CHECK-NEXT:  1      2     0.50    *                   ldrsh	w23, [x6, #8190]
+# CHECK-NEXT:  1      2     0.50    *                   ldrsh	wzr, [sp, #2]
+# CHECK-NEXT:  1      2     0.50    *                   ldrsh	x29, [x2, #2]
+# CHECK-NEXT:  1      2     0.50    *                   ldrb	w26, [x3, #121]
+# CHECK-NEXT:  1      2     0.50    *                   ldrb	w12, [x2]
+# CHECK-NEXT:  1      2     0.50    *                   ldrsb	w27, [sp, #4095]
+# CHECK-NEXT:  1      2     0.50    *                   ldrsb	xzr, [x15]
+# CHECK-NEXT:  1      1     1.00           *            str	x30, [sp]
+# CHECK-NEXT:  1      1     1.00           *            str	w20, [x4, #16380]
+# CHECK-NEXT:  1      1     1.00           *            strh	w17, [sp, #8190]
+# CHECK-NEXT:  1      1     1.00           *            strb	w23, [x3, #4095]
+# CHECK-NEXT:  1      1     1.00           *            strb	wzr, [x2]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	b31, [sp, #4095]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	h20, [x2, #8190]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	s10, [x19, #16380]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	d3, [x10, #32760]
+# CHECK-NEXT:  1      1     1.00           *            str	q12, [sp, #65520]
+# CHECK-NEXT:  1      2     0.50    *                   ldrb	w3, [sp, x5]
+# CHECK-NEXT:  1      2     0.50    *                   ldrb	w9, [x27, x6]
+# CHECK-NEXT:  1      2     0.50    *                   ldrsb	w10, [x30, x7]
+# CHECK-NEXT:  1      2     0.50    *                   ldrb	w11, [x29, x3, sxtx]
+# CHECK-NEXT:  1      1     1.00           *            strb	w12, [x28, xzr, sxtx]
+# CHECK-NEXT:  1      2     0.50    *                   ldrb	w14, [x26, w6, uxtw]
+# CHECK-NEXT:  1      2     0.50    *                   ldrsb	w15, [x25, w7, uxtw]
+# CHECK-NEXT:  1      2     0.50    *                   ldrb	w17, [x23, w9, sxtw]
+# CHECK-NEXT:  1      2     0.50    *                   ldrsb	x18, [x22, w10, sxtw]
+# CHECK-NEXT:  1      2     0.50    *                   ldrsh	w3, [sp, x5]
+# CHECK-NEXT:  1      2     0.50    *                   ldrsh	w9, [x27, x6]
+# CHECK-NEXT:  1      2     0.50    *                   ldrh	w10, [x30, x7, lsl #1]
+# CHECK-NEXT:  1      1     1.00           *            strh	w11, [x29, x3, sxtx]
+# CHECK-NEXT:  1      2     0.50    *                   ldrh	w12, [x28, xzr, sxtx]
+# CHECK-NEXT:  1      2     0.50    *                   ldrsh	x13, [x27, x5, sxtx #1]
+# CHECK-NEXT:  1      2     0.50    *                   ldrh	w14, [x26, w6, uxtw]
+# CHECK-NEXT:  1      2     0.50    *                   ldrh	w15, [x25, w7, uxtw]
+# CHECK-NEXT:  1      2     0.50    *                   ldrsh	w16, [x24, w8, uxtw #1]
+# CHECK-NEXT:  1      2     0.50    *                   ldrh	w17, [x23, w9, sxtw]
+# CHECK-NEXT:  1      2     0.50    *                   ldrh	w18, [x22, w10, sxtw]
+# CHECK-NEXT:  1      1     1.00           *            strh	w19, [x21, wzr, sxtw #1]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	w3, [sp, x5]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	s9, [x27, x6]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	w10, [x30, x7, lsl #2]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	w11, [x29, x3, sxtx]
+# CHECK-NEXT:  1      1     1.00           *            str	s12, [x28, xzr, sxtx]
+# CHECK-NEXT:  1      1     1.00           *            str	w13, [x27, x5, sxtx #2]
+# CHECK-NEXT:  1      1     1.00           *            str	w14, [x26, w6, uxtw]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	w15, [x25, w7, uxtw]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	w16, [x24, w8, uxtw #2]
+# CHECK-NEXT:  1      2     0.50    *                   ldrsw	x17, [x23, w9, sxtw]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	w18, [x22, w10, sxtw]
+# CHECK-NEXT:  1      2     0.50    *                   ldrsw	x19, [x21, wzr, sxtw #2]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	x3, [sp, x5]
+# CHECK-NEXT:  1      1     1.00           *            str	x9, [x27, x6]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	d10, [x30, x7, lsl #3]
+# CHECK-NEXT:  1      1     1.00           *            str	x11, [x29, x3, sxtx]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	x12, [x28, xzr, sxtx]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	x13, [x27, x5, sxtx #3]
+# CHECK-NEXT:  1      2     0.50                  U     prfm	pldl1keep, [x26, w6, uxtw]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	x15, [x25, w7, uxtw]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	x16, [x24, w8, uxtw #3]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	x17, [x23, w9, sxtw]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	x18, [x22, w10, sxtw]
+# CHECK-NEXT:  1      1     1.00           *            str	d19, [x21, wzr, sxtw #3]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	q3, [sp, x5]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	q9, [x27, x6]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	q10, [x30, x7, lsl #4]
+# CHECK-NEXT:  1      1     1.00           *            str	q11, [x29, x3, sxtx]
+# CHECK-NEXT:  1      1     1.00           *            str	q12, [x28, xzr, sxtx]
+# CHECK-NEXT:  1      1     1.00           *            str	q13, [x27, x5, sxtx #4]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	q14, [x26, w6, uxtw]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	q15, [x25, w7, uxtw]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	q16, [x24, w8, uxtw #4]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	q17, [x23, w9, sxtw]
+# CHECK-NEXT:  1      1     1.00           *            str	q18, [x22, w10, sxtw]
+# CHECK-NEXT:  1      2     0.50    *                   ldr	q19, [x21, wzr, sxtw #4]
+# CHECK-NEXT:  2      3     1.00    *                   ldp	w3, w5, [sp]
+# CHECK-NEXT:  1      1     1.00           *            stp	wzr, w9, [sp, #252]
+# CHECK-NEXT:  2      3     1.00    *                   ldp	w2, wzr, [sp, #-256]
+# CHECK-NEXT:  2      3     1.00    *                   ldp	w9, w10, [sp, #4]
+# CHECK-NEXT:  2      3     1.00    *                   ldpsw	x9, x10, [sp, #4]
+# CHECK-NEXT:  2      3     1.00    *                   ldpsw	x9, x10, [x2, #-256]
+# CHECK-NEXT:  2      3     1.00    *                   ldpsw	x20, x30, [sp, #252]
+# CHECK-NEXT:  2      3     1.00    *                   ldp	x21, x29, [x2, #504]
+# CHECK-NEXT:  2      3     1.00    *                   ldp	x22, x23, [x3, #-512]
+# CHECK-NEXT:  2      3     1.00    *                   ldp	x24, x25, [x4, #8]
+# CHECK-NEXT:  2      3     1.00    *                   ldp	s29, s28, [sp, #252]
+# CHECK-NEXT:  1      1     1.00           *            stp	s27, s26, [sp, #-256]
+# CHECK-NEXT:  2      3     1.00    *                   ldp	s1, s2, [x3, #44]
+# CHECK-NEXT:  1      1     1.00           *            stp	d3, d5, [x9, #504]
+# CHECK-NEXT:  1      1     1.00           *            stp	d7, d11, [x10, #-512]
+# CHECK-NEXT:  2      3     1.00    *                   ldp	d2, d3, [x30, #-8]
+# CHECK-NEXT:  1      1     1.00           *            stp	q3, q5, [sp]
+# CHECK-NEXT:  1      1     1.00           *            stp	q17, q19, [sp, #1008]
+# CHECK-NEXT:  2      3     1.00    *                   ldp	q23, q29, [x1, #-1024]
+# CHECK-NEXT:  3      3     1.00    *                   ldp	w3, w5, [sp], #0
+# CHECK-NEXT:  2      1     1.00           *            stp	wzr, w9, [sp], #252
+# CHECK-NEXT:  3      3     1.00    *                   ldp	w2, wzr, [sp], #-256
+# CHECK-NEXT:  3      3     1.00    *                   ldp	w9, w10, [sp], #4
+# CHECK-NEXT:  3      3     1.00    *                   ldpsw	x9, x10, [sp], #4
+# CHECK-NEXT:  3      3     1.00    *                   ldpsw	x9, x10, [x2], #-256
+# CHECK-NEXT:  3      3     1.00    *                   ldpsw	x20, x30, [sp], #252
+# CHECK-NEXT:  3      3     1.00    *                   ldp	x21, x29, [x2], #504
+# CHECK-NEXT:  3      3     1.00    *                   ldp	x22, x23, [x3], #-512
+# CHECK-NEXT:  3      3     1.00    *                   ldp	x24, x25, [x4], #8
+# CHECK-NEXT:  3      3     1.00    *                   ldp	s29, s28, [sp], #252
+# CHECK-NEXT:  2      1     1.00           *            stp	s27, s26, [sp], #-256
+# CHECK-NEXT:  3      3     1.00    *                   ldp	s1, s2, [x3], #44
+# CHECK-NEXT:  2      1     1.00           *            stp	d3, d5, [x9], #504
+# CHECK-NEXT:  2      1     1.00           *            stp	d7, d11, [x10], #-512
+# CHECK-NEXT:  3      3     1.00    *                   ldp	d2, d3, [x30], #-8
+# CHECK-NEXT:  2      1     1.00           *            stp	q3, q5, [sp], #0
+# CHECK-NEXT:  2      1     1.00           *            stp	q17, q19, [sp], #1008
+# CHECK-NEXT:  3      3     1.00    *                   ldp	q23, q29, [x1], #-1024
+# CHECK-NEXT:  3      3     1.00    *                   ldp	w3, w5, [sp, #0]!
+# CHECK-NEXT:  2      1     1.00           *            stp	wzr, w9, [sp, #252]!
+# CHECK-NEXT:  3      3     1.00    *                   ldp	w2, wzr, [sp, #-256]!
+# CHECK-NEXT:  3      3     1.00    *                   ldp	w9, w10, [sp, #4]!
+# CHECK-NEXT:  3      3     1.00    *                   ldpsw	x9, x10, [sp, #4]!
+# CHECK-NEXT:  3      3     1.00    *                   ldpsw	x9, x10, [x2, #-256]!
+# CHECK-NEXT:  3      3     1.00    *                   ldpsw	x20, x30, [sp, #252]!
+# CHECK-NEXT:  3      3     1.00    *                   ldp	x21, x29, [x2, #504]!
+# CHECK-NEXT:  3      3     1.00    *                   ldp	x22, x23, [x3, #-512]!
+# CHECK-NEXT:  3      3     1.00    *                   ldp	x24, x25, [x4, #8]!
+# CHECK-NEXT:  3      3     1.00    *                   ldp	s29, s28, [sp, #252]!
+# CHECK-NEXT:  2      1     1.00           *            stp	s27, s26, [sp, #-256]!
+# CHECK-NEXT:  3      3     1.00    *                   ldp	s1, s2, [x3, #44]!
+# CHECK-NEXT:  2      1     1.00           *            stp	d3, d5, [x9, #504]!
+# CHECK-NEXT:  2      1     1.00           *            stp	d7, d11, [x10, #-512]!
+# CHECK-NEXT:  3      3     1.00    *                   ldp	d2, d3, [x30, #-8]!
+# CHECK-NEXT:  2      1     1.00           *            stp	q3, q5, [sp, #0]!
+# CHECK-NEXT:  2      1     1.00           *            stp	q17, q19, [sp, #1008]!
+# CHECK-NEXT:  3      3     1.00    *                   ldp	q23, q29, [x1, #-1024]!
+# CHECK-NEXT:  2      2     1.00    *                   ldnp	w3, w5, [sp]
+# CHECK-NEXT:  1      1     1.00           *            stnp	wzr, w9, [sp, #252]
+# CHECK-NEXT:  2      2     1.00    *                   ldnp	w2, wzr, [sp, #-256]
+# CHECK-NEXT:  2      2     1.00    *                   ldnp	w9, w10, [sp, #4]
+# CHECK-NEXT:  2      2     1.00    *                   ldnp	x21, x29, [x2, #504]
+# CHECK-NEXT:  2      2     1.00    *                   ldnp	x22, x23, [x3, #-512]
+# CHECK-NEXT:  2      2     1.00    *                   ldnp	x24, x25, [x4, #8]
+# CHECK-NEXT:  2      2     1.00    *                   ldnp	s29, s28, [sp, #252]
+# CHECK-NEXT:  1      1     1.00           *            stnp	s27, s26, [sp, #-256]
+# CHECK-NEXT:  2      2     1.00    *                   ldnp	s1, s2, [x3, #44]
+# CHECK-NEXT:  1      1     1.00           *            stnp	d3, d5, [x9, #504]
+# CHECK-NEXT:  1      1     1.00           *            stnp	d7, d11, [x10, #-512]
+# CHECK-NEXT:  2      2     1.00    *                   ldnp	d2, d3, [x30, #-8]
+# CHECK-NEXT:  1      1     1.00           *            stnp	q3, q5, [sp]
+# CHECK-NEXT:  1      1     1.00           *            stnp	q17, q19, [sp, #1008]
+# CHECK-NEXT:  2      2     1.00    *                   ldnp	q23, q29, [x1, #-1024]
+# CHECK-NEXT:  1      1     0.33                        mov	w3, #983055
+# CHECK-NEXT:  1      1     0.33                        mov	x10, #-6148914691236517206
+# CHECK-NEXT:  1      1     0.33                        and	w12, w23, w21
+# CHECK-NEXT:  1      2     0.33                        and	w16, w15, w1, lsl #1
+# CHECK-NEXT:  1      2     0.33                        and	w9, w4, w10, lsl #31
+# CHECK-NEXT:  1      1     0.33                        and	w3, w30, w11
+# CHECK-NEXT:  1      2     0.33                        and	x3, x5, x7, lsl #63
+# CHECK-NEXT:  1      2     0.33                        and	x5, x14, x19, asr #4
+# CHECK-NEXT:  1      2     0.33                        and	w3, w17, w19, ror #31
+# CHECK-NEXT:  1      2     0.33                        and	w0, w2, wzr, lsr #17
+# CHECK-NEXT:  1      2     0.33                        and	w3, w30, w11, asr #2
+# CHECK-NEXT:  1      1     0.33                        and	xzr, x4, x26
+# CHECK-NEXT:  1      2     0.33                        and	w3, wzr, w20, ror #2
+# CHECK-NEXT:  1      2     0.33                        and	x7, x20, xzr, asr #63
+# CHECK-NEXT:  1      2     0.33                        bic	x13, x20, x14, lsl #47
+# CHECK-NEXT:  1      1     0.33                        bic	w2, w7, w9
+# CHECK-NEXT:  1      2     0.33                        orr	w2, w7, w0, asr #31
+# CHECK-NEXT:  1      2     0.33                        orr	x8, x9, x10, lsl #12
+# CHECK-NEXT:  1      2     0.33                        orn	x3, x5, x7, asr #2
+# CHECK-NEXT:  1      1     0.33                        orn	w2, w5, w29
+# CHECK-NEXT:  1      2     0.33                        ands	w7, wzr, w9, lsl #1
+# CHECK-NEXT:  1      2     0.33                        ands	x3, x5, x20, ror #63
+# CHECK-NEXT:  1      1     0.33                        bics	w3, w5, w7
+# CHECK-NEXT:  1      2     0.33                        bics	x3, xzr, x3, lsl #1
+# CHECK-NEXT:  1      2     0.33                        tst	w3, w7, lsl #31
+# CHECK-NEXT:  1      2     0.33                        tst	x2, x20, asr #2
+# CHECK-NEXT:  1      1     0.33                        mov	x3, x6
+# CHECK-NEXT:  1      1     0.33                        mov	x3, xzr
+# CHECK-NEXT:  1      1     0.33                        mov	wzr, w2
+# CHECK-NEXT:  1      1     0.33                        mov	w3, w5
+# CHECK-NEXT:  1      1     0.33                        movz	w2, #0, lsl #16
+# CHECK-NEXT:  1      1     0.33                        mov	w2, #-1235
+# CHECK-NEXT:  1      1     0.33                        mov	x2, #5299989643264
+# CHECK-NEXT:  1      1     0.33                        mov	x2, #0
+# CHECK-NEXT:  1      1     0.33                        movk	w3, #0
+# CHECK-NEXT:  1      1     0.33                        movz	x4, #0, lsl #16
+# CHECK-NEXT:  1      1     0.33                        movk	w5, #0, lsl #16
+# CHECK-NEXT:  1      1     0.33                        movz	x6, #0, lsl #32
+# CHECK-NEXT:  1      1     0.33                        movk	x7, #0, lsl #32
+# CHECK-NEXT:  1      1     0.33                        movz	x8, #0, lsl #48
+# CHECK-NEXT:  1      1     0.33                        movk	x9, #0, lsl #48
+# CHECK-NEXT:  1      1     0.33                        adr	x2, #1600
+# CHECK-NEXT:  1      1     0.33                        adrp	x21, #6553600
+# CHECK-NEXT:  1      1     0.33                        adr	x0, #262144
+# CHECK-NEXT:  1      1     1.00                        tbz	x12, #62, #0
+# CHECK-NEXT:  1      1     1.00                        tbz	x12, #62, #4
+# CHECK-NEXT:  1      1     1.00                        tbz	x12, #62, #-32768
+# CHECK-NEXT:  1      1     1.00                        tbnz	x12, #60, #32764
+# CHECK-NEXT:  1      1     1.00                        b	#4
+# CHECK-NEXT:  1      1     1.00                        b	#-4
+# CHECK-NEXT:  1      1     1.00                        b	#134217724
+# CHECK-NEXT:  1      1     1.00                        br	x20
+# CHECK-NEXT:  1      1     1.00                        blr	xzr
+# CHECK-NEXT:  1      1     1.00                  U     ret	x10
+# CHECK-NEXT:  1      1     1.00                  U     ret
+# CHECK-NEXT:  1      1     1.00                  U     eret
+# CHECK-NEXT:  1      1     1.00                  U     drps
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - CortexA510UnitALU0
+# CHECK-NEXT: [1.0] - CortexA510UnitALU12
+# CHECK-NEXT: [1.1] - CortexA510UnitALU12
+# CHECK-NEXT: [2]   - CortexA510UnitB
+# CHECK-NEXT: [3]   - CortexA510UnitDiv
+# CHECK-NEXT: [4]   - CortexA510UnitLd1
+# CHECK-NEXT: [5]   - CortexA510UnitLdSt
+# CHECK-NEXT: [6]   - CortexA510UnitMAC
+# CHECK-NEXT: [7]   - CortexA510UnitPAC
+# CHECK-NEXT: [8]   - CortexA510UnitVALU0
+# CHECK-NEXT: [9]   - CortexA510UnitVALU1
+# CHECK-NEXT: [10.0] - CortexA510UnitVMAC
+# CHECK-NEXT: [10.1] - CortexA510UnitVMAC
+# CHECK-NEXT: [11]  - CortexA510UnitVMC
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1.0]  [1.1]  [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10.0] [10.1] [11]
+# CHECK-NEXT: 155.00 155.00 155.00 22.00  48.00  161.00 289.00 69.00   -     110.50 110.50 6.00   6.00   87.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1.0]  [1.1]  [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10.0] [10.1] [11]   Instructions:
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	w2, w3, #4095
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	w30, w29, #1, lsl #12
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	w13, w5, #4095, lsl #12
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	x5, x7, #1638
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	w20, wsp, #801
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	wsp, wsp, #1104
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	wsp, w30, #4084
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	x0, x24, #291
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	x3, x24, #4095, lsl #12
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	x8, sp, #1074
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	sp, x29, #3816
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	w0, wsp, #4077
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	w4, w20, #546, lsl #12
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	sp, sp, #288
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	wsp, w19, #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adds	w13, w23, #291, lsl #12
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	w2, #4095
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adds	w20, wsp, #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	x3, #1, lsl #12
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	sp, #20, lsl #12
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	x30, #4095
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     subs	x4, sp, #3822
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	w3, #291, lsl #12
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	wsp, #1365
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	sp, #1092, lsl #12
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     mov	sp, x30
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     mov	wsp, w20
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     mov	x11, sp
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     mov	w24, wsp
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	w3, w5, w7
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	wzr, w3, w5
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	w20, wzr, w4
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	w4, w6, wzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	w11, w13, w15
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	w9, w3, wzr, lsl #10
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	w17, w29, w20, lsl #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	w21, w22, w23, lsr #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	w24, w25, w26, lsr #18
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	w27, w28, w29, lsr #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	w2, w3, w4, asr #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	w5, w6, w7, asr #21
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	w8, w9, w10, asr #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	x3, x5, x7
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	xzr, x3, x5
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	x20, xzr, x4
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	x4, x6, xzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	x11, x13, x15
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	x9, x3, xzr, lsl #10
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	x17, x29, x20, lsl #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	x21, x22, x23, lsr #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	x24, x25, x26, lsr #18
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	x27, x28, x29, lsr #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	x2, x3, x4, asr #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	x5, x6, x7, asr #21
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     add	x8, x9, x10, asr #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adds	w3, w5, w7
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	w3, w5
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adds	w20, wzr, w4
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adds	w4, w6, wzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adds	w11, w13, w15
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adds	w9, w3, wzr, lsl #10
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adds	w17, w29, w20, lsl #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adds	w21, w22, w23, lsr #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adds	w24, w25, w26, lsr #18
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adds	w27, w28, w29, lsr #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adds	w2, w3, w4, asr #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adds	w5, w6, w7, asr #21
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adds	w8, w9, w10, asr #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adds	x3, x5, x7
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	x3, x5
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adds	x20, xzr, x4
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adds	x4, x6, xzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adds	x11, x13, x15
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adds	x9, x3, xzr, lsl #10
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adds	x17, x29, x20, lsl #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adds	x21, x22, x23, lsr #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adds	x24, x25, x26, lsr #18
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adds	x27, x28, x29, lsr #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adds	x2, x3, x4, asr #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adds	x5, x6, x7, asr #21
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adds	x8, x9, x10, asr #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	w3, w5, w7
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	wzr, w3, w5
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	w4, w6, wzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	w11, w13, w15
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	w9, w3, wzr, lsl #10
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	w17, w29, w20, lsl #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	w21, w22, w23, lsr #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	w24, w25, w26, lsr #18
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	w27, w28, w29, lsr #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	w2, w3, w4, asr #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	w5, w6, w7, asr #21
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	w8, w9, w10, asr #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	x3, x5, x7
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	xzr, x3, x5
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	x4, x6, xzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	x11, x13, x15
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	x9, x3, xzr, lsl #10
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	x17, x29, x20, lsl #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	x21, x22, x23, lsr #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	x24, x25, x26, lsr #18
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	x27, x28, x29, lsr #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	x2, x3, x4, asr #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	x5, x6, x7, asr #21
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sub	x8, x9, x10, asr #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     subs	w3, w5, w7
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	w3, w5
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     subs	w4, w6, wzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     subs	w11, w13, w15
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     subs	w9, w3, wzr, lsl #10
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     subs	w17, w29, w20, lsl #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     subs	w21, w22, w23, lsr #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     subs	w24, w25, w26, lsr #18
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     subs	w27, w28, w29, lsr #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     subs	w2, w3, w4, asr #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     subs	w5, w6, w7, asr #21
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     subs	w8, w9, w10, asr #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     subs	x3, x5, x7
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	x3, x5
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     subs	x4, x6, xzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     subs	x11, x13, x15
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     subs	x9, x3, xzr, lsl #10
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     subs	x17, x29, x20, lsl #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     subs	x21, x22, x23, lsr #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     subs	x24, x25, x26, lsr #18
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     subs	x27, x28, x29, lsr #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     subs	x2, x3, x4, asr #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     subs	x5, x6, x7, asr #21
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     subs	x8, x9, x10, asr #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	wzr, w4
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	w5, wzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	w6, w7
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	w8, w9, lsl #15
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	w10, w11, lsl #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	w12, w13, lsr #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	w14, w15, lsr #21
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	w16, w17, lsr #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	w18, w19, asr #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	w20, w21, asr #22
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	w22, w23, asr #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	x0, x3
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	xzr, x4
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	x5, xzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	x6, x7
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	x8, x9, lsl #15
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	x10, x11, lsl #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	x12, x13, lsr #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	x14, x15, lsr #41
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	x16, x17, lsr #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	x18, x19, asr #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	x20, x21, asr #55
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmn	x22, x23, asr #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	w0, w3
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	wzr, w4
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	w5, wzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	w6, w7
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	w8, w9, lsl #15
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	w10, w11, lsl #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	w12, w13, lsr #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	w14, w15, lsr #21
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	w18, w19, asr #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	w20, w21, asr #22
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	w22, w23, asr #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	x0, x3
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	xzr, x4
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	x5, xzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	x6, x7
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	x8, x9, lsl #15
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	x10, x11, lsl #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	x12, x13, lsr #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	x14, x15, lsr #41
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	x16, x17, lsr #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	x18, x19, asr #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	x20, x21, asr #55
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	x22, x23, asr #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	wzr, w0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cmp	xzr, x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adc	w29, w27, w25
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adc	wzr, w3, w4
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adc	w9, wzr, w10
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adc	w20, w0, wzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adc	x29, x27, x25
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adc	xzr, x3, x4
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adc	x9, xzr, x10
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adc	x20, x0, xzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adcs	w29, w27, w25
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adcs	wzr, w3, w4
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adcs	w9, wzr, w10
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adcs	w20, w0, wzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adcs	x29, x27, x25
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adcs	xzr, x3, x4
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adcs	x9, xzr, x10
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adcs	x20, x0, xzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sbc	w29, w27, w25
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sbc	wzr, w3, w4
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ngc	w9, w10
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sbc	w20, w0, wzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sbc	x29, x27, x25
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sbc	xzr, x3, x4
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ngc	x9, x10
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sbc	x20, x0, xzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sbcs	w29, w27, w25
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sbcs	wzr, w3, w4
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ngcs	w9, w10
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sbcs	w20, w0, wzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sbcs	x29, x27, x25
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sbcs	xzr, x3, x4
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ngcs	x9, x10
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sbcs	x20, x0, xzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ngc	w3, w12
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ngc	wzr, w9
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ngc	w23, wzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ngc	x29, x30
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ngc	xzr, x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ngc	x0, xzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ngcs	w3, w12
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ngcs	wzr, w9
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ngcs	w23, wzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ngcs	x29, x30
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ngcs	xzr, x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ngcs	x0, xzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sbfx	x1, x2, #3, #2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     asr	x3, x4, #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     asr	wzr, wzr, #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sbfx	w12, w9, #0, #1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ubfiz	x4, x5, #52, #11
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ubfx	xzr, x4, #0, #1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ubfiz	x4, xzr, #1, #6
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsr	x5, x6, #12
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     bfi	x4, x5, #52, #11
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     bfxil	xzr, x4, #0, #1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     bfc	x4, #1, #6
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     bfxil	x5, x6, #12, #52
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sxtb	w1, w2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sxtb	xzr, w3
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sxth	w9, w10
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sxth	x0, w1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sxtw	x3, w30
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uxtb	w1, w2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uxth	w9, w10
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ubfx	x3, x30, #0, #32
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     asr	w3, w2, #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     asr	w9, w10, #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     asr	x20, x21, #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     asr	w1, wzr, #3
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsr	w3, w2, #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsr	w9, w10, #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsr	x20, x21, #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsr	wzr, wzr, #3
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsr	w3, w2, #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsl	w9, w10, #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsl	x20, x21, #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsl	w1, wzr, #3
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sbfx	w9, w10, #0, #1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sbfiz	x2, x3, #63, #1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     asr	x19, x20, #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sbfiz	x9, x10, #5, #59
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     asr	w9, w10, #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sbfiz	w11, w12, #31, #1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sbfiz	w13, w14, #29, #3
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sbfiz	xzr, xzr, #10, #11
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sbfx	w9, w10, #0, #1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     asr	x2, x3, #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     asr	x19, x20, #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     asr	x9, x10, #5
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     asr	w9, w10, #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     asr	w11, w12, #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     asr	w13, w14, #29
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sbfx	xzr, xzr, #10, #11
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     bfxil	w9, w10, #0, #1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     bfi	x2, x3, #63, #1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     bfxil	x19, x20, #0, #64
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     bfi	x9, x10, #5, #59
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     bfxil	w9, w10, #0, #32
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     bfi	w11, w12, #31, #1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     bfi	w13, w14, #29, #3
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     bfc	xzr, #10, #11
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     bfxil	w9, w10, #0, #1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     bfxil	x2, x3, #63, #1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     bfxil	x19, x20, #0, #64
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     bfxil	x9, x10, #5, #59
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     bfxil	w9, w10, #0, #32
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     bfxil	w11, w12, #31, #1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     bfxil	w13, w14, #29, #3
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     bfxil	xzr, xzr, #10, #11
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ubfx	w9, w10, #0, #1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsl	x2, x3, #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsr	x19, x20, #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsl	x9, x10, #5
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsr	w9, w10, #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsl	w11, w12, #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsl	w13, w14, #29
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ubfiz	xzr, xzr, #10, #11
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ubfx	w9, w10, #0, #1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsr	x2, x3, #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsr	x19, x20, #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsr	x9, x10, #5
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsr	w9, w10, #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsr	w11, w12, #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsr	w13, w14, #29
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ubfx	xzr, xzr, #10, #11
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     cbz	w5, #4
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     cbz	x5, #0
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     cbnz	x2, #-4
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     cbnz	x26, #1048572
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     cbz	wzr, #0
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     cbnz	xzr, #0
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     b.ne	#4
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     b.ge	#1048572
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     b.ge	#-4
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ccmp	w1, #31, #0, eq
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ccmp	w3, #0, #15, hs
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ccmp	wzr, #15, #13, hs
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ccmp	x9, #31, #0, le
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ccmp	x3, #0, #15, gt
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ccmp	xzr, #5, #7, ne
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ccmn	w1, #31, #0, eq
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ccmn	w3, #0, #15, hs
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ccmn	wzr, #15, #13, hs
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ccmn	x9, #31, #0, le
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ccmn	x3, #0, #15, gt
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ccmn	xzr, #5, #7, ne
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ccmp	w1, wzr, #0, eq
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ccmp	w3, w0, #15, hs
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ccmp	wzr, w15, #13, hs
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ccmp	x9, xzr, #0, le
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ccmp	x3, x0, #15, gt
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ccmp	xzr, x5, #7, ne
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ccmn	w1, wzr, #0, eq
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ccmn	w3, w0, #15, hs
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ccmn	wzr, w15, #13, hs
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ccmn	x9, xzr, #0, le
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ccmn	x3, x0, #15, gt
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ccmn	xzr, x5, #7, ne
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csel	w1, w0, w19, ne
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csel	wzr, w5, w9, eq
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csel	w9, wzr, w30, gt
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csel	w1, w28, wzr, mi
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csel	x19, x23, x29, lt
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csel	xzr, x3, x4, ge
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csel	x5, xzr, x6, hs
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csel	x7, x8, xzr, lo
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csinc	w1, w0, w19, ne
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csinc	wzr, w5, w9, eq
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csinc	w9, wzr, w30, gt
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csinc	w1, w28, wzr, mi
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csinc	x19, x23, x29, lt
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csinc	xzr, x3, x4, ge
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csinc	x5, xzr, x6, hs
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csinc	x7, x8, xzr, lo
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csinv	w1, w0, w19, ne
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csinv	wzr, w5, w9, eq
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csinv	w9, wzr, w30, gt
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csinv	w1, w28, wzr, mi
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csinv	x19, x23, x29, lt
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csinv	xzr, x3, x4, ge
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csinv	x5, xzr, x6, hs
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csinv	x7, x8, xzr, lo
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csneg	w1, w0, w19, ne
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csneg	wzr, w5, w9, eq
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csneg	w9, wzr, w30, gt
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csneg	w1, w28, wzr, mi
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csneg	x19, x23, x29, lt
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csneg	xzr, x3, x4, ge
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csneg	x5, xzr, x6, hs
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csneg	x7, x8, xzr, lo
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cset	w3, eq
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cset	x9, pl
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csetm	w20, ne
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csetm	x30, ge
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csinc	w2, wzr, wzr, al
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csinv	x3, xzr, xzr, nv
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cinc	w3, w5, gt
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cinc	wzr, w4, le
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cset	w9, lt
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cinc	x3, x5, gt
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cinc	xzr, x4, le
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cset	x9, lt
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csinc	w5, w6, w6, nv
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csinc	x1, x2, x2, al
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cinv	w3, w5, gt
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cinv	wzr, w4, le
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csetm	w9, lt
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cinv	x3, x5, gt
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cinv	xzr, x4, le
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csetm	x9, lt
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csinv	x1, x0, x0, al
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csinv	w9, w8, w8, nv
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cneg	w3, w5, gt
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cneg	wzr, w4, le
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cneg	w9, wzr, lt
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cneg	x3, x5, gt
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cneg	xzr, x4, le
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cneg	x9, xzr, lt
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csneg	x4, x8, x8, al
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     csinv	w9, w8, w8, nv
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     rbit	w0, w7
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     rbit	x18, x3
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     rev16	w17, w1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     rev16	x5, x2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     rev	w18, w0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     rev32	x20, x1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     rev	x22, x2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     clz	w24, w3
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     clz	x26, x4
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cls	w3, w5
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cls	x20, x5
+# CHECK-NEXT:  -      -      -      -     8.00    -      -      -      -      -      -      -      -      -     udiv	w0, w7, w10
+# CHECK-NEXT:  -      -      -      -     16.00   -      -      -      -      -      -      -      -      -     udiv	x9, x22, x4
+# CHECK-NEXT:  -      -      -      -     8.00    -      -      -      -      -      -      -      -      -     sdiv	w12, w21, w0
+# CHECK-NEXT:  -      -      -      -     16.00   -      -      -      -      -      -      -      -      -     sdiv	x13, x2, x1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsl	w11, w12, w13
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsl	x14, x15, x16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsr	w17, w18, w19
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsr	x20, x21, x22
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     asr	w23, w24, w25
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     asr	x26, x27, x28
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ror	w0, w1, w2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ror	x3, x4, x5
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsl	w6, w7, w8
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsl	x9, x10, x11
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsr	w12, w13, w14
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     lsr	x15, x16, x17
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     asr	w18, w19, w20
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     asr	x21, x22, x23
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ror	w24, w25, w26
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ror	x27, x28, x29
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -     smulh	x30, x29, x28
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -     smulh	xzr, x27, x26
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -     umulh	x30, x29, x28
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -     umulh	x23, x30, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     madd	w1, w3, w7, w4
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     madd	wzr, w0, w9, w11
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     madd	w13, wzr, w4, w4
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     madd	w19, w30, wzr, w29
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     mul	w4, w5, w6
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -     madd	x1, x3, x7, x4
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -     madd	xzr, x0, x9, x11
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -     madd	x13, xzr, x4, x4
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -     madd	x19, x30, xzr, x29
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -     mul	x4, x5, x6
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     msub	w1, w3, w7, w4
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     msub	wzr, w0, w9, w11
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     msub	w13, wzr, w4, w4
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     msub	w19, w30, wzr, w29
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     mneg	w4, w5, w6
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -     msub	x1, x3, x7, x4
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -     msub	xzr, x0, x9, x11
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -     msub	x13, xzr, x4, x4
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -     msub	x19, x30, xzr, x29
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -     mneg	x4, x5, x6
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     smaddl	x3, w5, w2, x9
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     smaddl	xzr, w10, w11, x12
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     smaddl	x13, wzr, w14, x15
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     smaddl	x16, w17, wzr, x18
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     smull	x19, w20, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     smsubl	x3, w5, w2, x9
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     smsubl	xzr, w10, w11, x12
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     smsubl	x13, wzr, w14, x15
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     smsubl	x16, w17, wzr, x18
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     smnegl	x19, w20, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     umaddl	x3, w5, w2, x9
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     umaddl	xzr, w10, w11, x12
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     umaddl	x13, wzr, w14, x15
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     umaddl	x16, w17, wzr, x18
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     umull	x19, w20, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     umsubl	x3, w5, w2, x9
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     umsubl	x16, w17, wzr, x18
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     umnegl	x19, w20, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -     smulh	x30, x29, x28
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -     smulh	x23, x22, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -     umulh	x23, x22, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -     2.00    -      -      -      -      -      -     mul	x19, x20, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     mneg	w21, w22, w23
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     smull	x11, w13, w17
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     umull	x11, w13, w17
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     smnegl	x11, w13, w17
+# CHECK-NEXT:  -      -      -      -      -      -      -     1.00    -      -      -      -      -      -     umnegl	x11, w13, w17
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     extr	w3, w5, w7, #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     extr	w11, w13, w17, #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     extr	x3, x5, x7, #15
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     extr	x11, x13, x17, #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ror	x19, x23, #24
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ror	x29, xzr, #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ror	w9, w13, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmp	s3, s5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmp	s31, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmp	s31, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmpe	s29, s30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmpe	s15, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmpe	s15, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmp	d4, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmp	d23, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmp	d23, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmpe	d26, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmpe	d29, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmpe	d29, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fccmp	s1, s31, #0, eq
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fccmp	s3, s0, #15, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fccmp	s31, s15, #13, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fccmp	d9, d31, #0, le
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fccmp	d3, d0, #15, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fccmp	d31, d5, #7, ne
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fccmpe	s1, s31, #0, eq
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fccmpe	s3, s0, #15, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fccmpe	s31, s15, #13, hs
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fccmpe	d9, d31, #0, le
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fccmpe	d3, d0, #15, gt
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fccmpe	d31, d5, #7, ne
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcsel	s3, s20, s9, pl
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcsel	d9, d10, d11, mi
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmov	s0, s1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fabs	s2, s3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fneg	s4, s5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     29.00  fsqrt	s6, s7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvt	d8, s9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvt	h10, s11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintn	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintp	s14, s15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintm	s16, s17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintz	s18, s19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frinta	s20, s21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintx	s22, s23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frinti	s24, s25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmov	d0, d1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fabs	d2, d3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fneg	d4, d5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     29.00  fsqrt	d6, d7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvt	s8, d9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvt	h10, d11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintn	d12, d13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintp	d14, d15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintm	d16, d17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintz	d18, d19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frinta	d20, d21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintx	d22, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frinti	d24, d25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvt	s26, h27
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvt	d28, h29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmul	s20, s19, s17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     10.00  fdiv	s1, s2, s3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fadd	s4, s5, s6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fsub	s7, s8, s9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmax	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmin	s13, s14, s15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmaxnm	s16, s17, s18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fminnm	s19, s20, s21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fnmul	s22, s23, s2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmul	d20, d19, d17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     19.00  fdiv	d1, d2, d3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fadd	d4, d5, d6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fsub	d7, d8, d9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmax	d10, d11, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmin	d13, d14, d15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmaxnm	d16, d17, d18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fminnm	d19, d20, d21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fnmul	d22, d23, d24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmadd	s3, s5, s6, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmadd	d3, d13, d0, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmsub	s3, s5, s6, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmsub	d3, d13, d0, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fnmadd	s3, s5, s6, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fnmadd	d3, d13, d0, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fnmsub	s3, s5, s6, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fnmsub	d3, d13, d0, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	w3, h5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	wzr, h20, #13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	w19, h0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	x3, h5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	x12, h30, #45
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	x19, h0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	w3, s5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	wzr, s20, #13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	w19, s0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	x3, s5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	x12, s30, #45
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	x19, s0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	w3, d5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	wzr, d20, #13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	w19, d0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	x3, d5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	x12, d30, #45
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	x19, d0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	w3, h5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	wzr, h20, #13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	w19, h0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	x3, h5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	x12, h30, #45
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	x19, h0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	w3, s5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	wzr, s20, #13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	w19, s0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	x3, s5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	x12, s30, #45
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	x19, s0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	w3, d5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	wzr, d20, #13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	w19, d0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	x3, d5, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	x12, d30, #45
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	x19, d0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	h23, w19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	h31, wzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	h14, w0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	h23, x19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	h31, xzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	h14, x0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	s23, w19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	s31, wzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	s14, w0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	s23, x19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	s31, xzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	s14, x0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	d23, w19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	d31, wzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	d14, w0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	d23, x19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	d31, xzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	d14, x0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	h23, w19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	h31, wzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	h14, w0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	h23, x19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	h31, xzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	h14, x0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	s23, w19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	s31, wzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	s14, w0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	s23, x19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	s31, xzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	s14, x0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	d23, w19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	d31, wzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	d14, w0, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	d23, x19, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	d31, xzr, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	d14, x0, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtns	w3, h31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtns	xzr, h12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtnu	wzr, h12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtnu	x0, h0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtps	wzr, h9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtps	x12, h20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtpu	w30, h23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtpu	x29, h3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtms	w2, h3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtms	x4, h5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtmu	w6, h7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtmu	x8, h9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	w10, h11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	x12, h13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	w14, h15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	x15, h16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	h17, w18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	h19, x20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	h21, w22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	h23, x24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtas	w25, h26
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtas	x27, h28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtau	w29, h30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtau	xzr, h0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtns	w3, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtns	xzr, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtnu	wzr, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtnu	x0, s0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtps	wzr, s9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtps	x12, s20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtpu	w30, s23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtpu	x29, s3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtms	w2, s3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtms	x4, s5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtmu	w6, s7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtmu	x8, s9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	w10, s11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	x12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	w14, s15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	x15, s16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	s17, w18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	s19, x20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	s21, w22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	s23, x24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtas	w25, s26
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtas	x27, s28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtau	w29, s30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtau	xzr, s0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtns	w3, d31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtns	xzr, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtnu	wzr, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtnu	x0, d0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtps	wzr, d9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtps	x12, d20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtpu	w30, d23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtpu	x29, d3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtms	w2, d3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtms	x4, d5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtmu	w6, d7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtmu	x8, d9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	w10, d11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	x12, d13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	w14, d15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	x15, d16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	d17, w18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	d19, x20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	d21, w22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	d23, x24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtas	w25, d26
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtas	x27, d28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtau	w29, d30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtau	xzr, d0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmov	w3, s9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmov	s9, w3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmov	x20, d31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmov	d1, x15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmov	x3, v12.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmov	v1.d[1], x19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmov	s2, #0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmov	s3, #1.00000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmov	d30, #16.00000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmov	s4, #1.06250000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmov	d10, #1.93750000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmov	s12, #-1.00000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmov	d16, #8.50000000
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	w3, #0
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	x29, #4
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsw	xzr, #-4
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	s0, #8
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	d0, #1048572
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	q0, #-1048576
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     prfm	pldl1strm, #0
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     prfm	#22, #0
+# CHECK-NEXT:  -      -      -      -      -     0.50   1.50    -      -      -      -      -      -      -     stxrb	w18, w8, [sp]
+# CHECK-NEXT:  -      -      -      -      -     0.50   1.50    -      -      -      -      -      -      -     stxrh	w24, w15, [x16]
+# CHECK-NEXT:  -      -      -      -      -     0.50   1.50    -      -      -      -      -      -      -     stxr	w5, w6, [x17]
+# CHECK-NEXT:  -      -      -      -      -     0.50   1.50    -      -      -      -      -      -      -     stxr	w1, x10, [x21]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldxrb	w30, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldxrh	w17, [x4]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldxr	w22, [sp]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldxr	x11, [x29]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldxr	x11, [x29]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldxr	x11, [x29]
+# CHECK-NEXT:  -      -      -      -      -     0.50   1.50    -      -      -      -      -      -      -     stxp	w12, w11, w10, [sp]
+# CHECK-NEXT:  -      -      -      -      -     0.50   1.50    -      -      -      -      -      -      -     stxp	wzr, x27, x9, [x12]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldxp	w0, wzr, [sp]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldxp	x17, x0, [x18]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldxp	x17, x0, [x18]
+# CHECK-NEXT:  -      -      -      -      -     0.50   1.50    -      -      -      -      -      -      -     stlxrb	w12, w22, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   1.50    -      -      -      -      -      -      -     stlxrh	w10, w1, [x1]
+# CHECK-NEXT:  -      -      -      -      -     0.50   1.50    -      -      -      -      -      -      -     stlxr	w9, w2, [x2]
+# CHECK-NEXT:  -      -      -      -      -     0.50   1.50    -      -      -      -      -      -      -     stlxr	w9, x3, [sp]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldaxrb	w8, [x4]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldaxrh	w7, [x5]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldaxr	w6, [sp]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldaxr	x5, [x6]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldaxr	x5, [x6]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldaxr	x5, [x6]
+# CHECK-NEXT:  -      -      -      -      -     0.50   1.50    -      -      -      -      -      -      -     stlxp	w4, w5, w6, [sp]
+# CHECK-NEXT:  -      -      -      -      -     0.50   1.50    -      -      -      -      -      -      -     stlxp	wzr, x6, x7, [x1]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldaxp	w5, w18, [sp]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldaxp	x6, x19, [x22]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldaxp	x6, x19, [x22]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stlrb	w24, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stlrh	w25, [x30]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stlr	w26, [x29]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stlr	x27, [x28]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stlr	x27, [x28]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stlr	x27, [x28]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldarb	w23, [sp]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldarh	w22, [x30]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldar	wzr, [x29]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldar	x21, [x28]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldar	x21, [x28]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldar	x21, [x28]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sturb	w9, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sturh	wzr, [x12, #255]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stur	w16, [x0, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stur	x28, [x14, #1]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldurb	w1, [x20, #255]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldurh	w20, [x1, #255]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldur	w12, [sp, #255]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldur	xzr, [x12, #255]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldursb	x9, [x7, #-256]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldursh	x17, [x19, #-256]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldursw	x20, [x15, #-256]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     prfum	pldl2keep, [sp, #-256]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldursb	w19, [x1, #-256]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldursh	w15, [x21, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stur	b0, [sp, #1]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stur	h12, [x12, #-1]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stur	s15, [x0, #255]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stur	d31, [x5, #25]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stur	q9, [x5]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldur	b3, [sp]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldur	h5, [x4, #-256]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldur	s7, [x12, #-1]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldur	d11, [x19, #4]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldur	q13, [x1, #2]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     strb	w9, [x2], #255
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     strb	w10, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     strb	w10, [x3], #-256
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     strh	w9, [x2], #255
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     strh	w9, [x2], #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     strh	w10, [x3], #-256
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	w19, [sp], #255
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	w20, [x30], #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	w21, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	xzr, [x9], #255
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	x2, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	x19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrb	w9, [x2], #255
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrb	w10, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrb	w10, [x3], #-256
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrh	w9, [x2], #255
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrh	w9, [x2], #1
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrh	w10, [x3], #-256
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	w19, [sp], #255
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	w20, [x30], #1
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	w21, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	xzr, [x9], #255
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	x2, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	x19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsb	xzr, [x9], #255
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsb	x2, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsb	x19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsh	xzr, [x9], #255
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsh	x2, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsh	x19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsw	xzr, [x9], #255
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsw	x2, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsw	x19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsb	wzr, [x9], #255
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsb	w2, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsb	w19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsh	wzr, [x9], #255
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsh	w2, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsh	w19, [x12], #-256
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	b0, [x0], #255
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	b3, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	b5, [sp], #-256
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	h10, [x10], #255
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	h13, [x23], #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	h15, [sp], #-256
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	s20, [x20], #255
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	s23, [x23], #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	s25, [x0], #-256
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	d20, [x20], #255
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	d23, [x23], #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	d25, [x0], #-256
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	b0, [x0], #255
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	b3, [x3], #1
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	b5, [sp], #-256
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	h10, [x10], #255
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	h13, [x23], #1
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	h15, [sp], #-256
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	s20, [x20], #255
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	s23, [x23], #1
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	s25, [x0], #-256
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	d20, [x20], #255
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	d23, [x23], #1
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	d25, [x0], #-256
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	q20, [x1], #255
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	q23, [x9], #1
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	q25, [x20], #-256
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	q10, [x1], #255
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	q22, [sp], #1
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	q21, [x20], #-256
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	x3, [x4, #0]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     strb	w9, [x2, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     strb	w10, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     strb	w10, [x3, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     strh	w9, [x2, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     strh	w9, [x2, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     strh	w10, [x3, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	w19, [sp, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	w20, [x30, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	w21, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	xzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	x2, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	x19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrb	w9, [x2, #255]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrb	w10, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrb	w10, [x3, #-256]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrh	w9, [x2, #255]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrh	w9, [x2, #1]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrh	w10, [x3, #-256]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	w19, [sp, #255]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	w20, [x30, #1]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	w21, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	xzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	x2, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	x19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsb	xzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsb	x2, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsb	x19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsh	xzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsh	x2, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsh	x19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsw	xzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsw	x2, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsw	x19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsb	wzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsb	w2, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsb	w19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsh	wzr, [x9, #255]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsh	w2, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsh	w19, [x12, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	b0, [x0, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	b3, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	b5, [sp, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	h10, [x10, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	h13, [x23, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	h15, [sp, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	s20, [x20, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	s23, [x23, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	s25, [x0, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	d20, [x20, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	d23, [x23, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	d25, [x0, #-256]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	b0, [x0, #255]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	b3, [x3, #1]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	b5, [sp, #-256]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	h10, [x10, #255]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	h13, [x23, #1]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	h15, [sp, #-256]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	s20, [x20, #255]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	s23, [x23, #1]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	s25, [x0, #-256]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	d20, [x20, #255]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	d23, [x23, #1]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	d25, [x0, #-256]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	q20, [x1, #255]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	q23, [x9, #1]!
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	q25, [x20, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	q10, [x1, #255]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	q22, [sp, #1]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	q21, [x20, #-256]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sttrb	w9, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sttrh	wzr, [x12, #255]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sttr	w16, [x0, #-256]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     sttr	x28, [x14, #1]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldtrb	w1, [x20, #255]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldtrh	w20, [x1, #255]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldtr	w12, [sp, #255]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldtr	xzr, [x12, #255]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldtrsb	x9, [x7, #-256]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldtrsh	x17, [x19, #-256]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldtrsw	x20, [x15, #-256]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldtrsb	w19, [x1, #-256]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldtrsh	w15, [x21, #-256]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	x4, [x29]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	x30, [x12, #32760]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	x20, [sp, #8]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	xzr, [sp]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	w2, [sp]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	w17, [sp, #16380]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	w13, [x2, #4]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsw	x2, [x5, #4]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsw	x23, [sp, #16380]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrh	w2, [x4]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsh	w23, [x6, #8190]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsh	wzr, [sp, #2]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsh	x29, [x2, #2]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrb	w26, [x3, #121]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrb	w12, [x2]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsb	w27, [sp, #4095]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsb	xzr, [x15]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	x30, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	w20, [x4, #16380]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     strh	w17, [sp, #8190]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     strb	w23, [x3, #4095]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     strb	wzr, [x2]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	b31, [sp, #4095]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	h20, [x2, #8190]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	s10, [x19, #16380]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	d3, [x10, #32760]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	q12, [sp, #65520]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrb	w3, [sp, x5]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrb	w9, [x27, x6]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsb	w10, [x30, x7]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrb	w11, [x29, x3, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     strb	w12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrb	w14, [x26, w6, uxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsb	w15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrb	w17, [x23, w9, sxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsb	x18, [x22, w10, sxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsh	w3, [sp, x5]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsh	w9, [x27, x6]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrh	w10, [x30, x7, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     strh	w11, [x29, x3, sxtx]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrh	w12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsh	x13, [x27, x5, sxtx #1]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrh	w14, [x26, w6, uxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrh	w15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsh	w16, [x24, w8, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrh	w17, [x23, w9, sxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrh	w18, [x22, w10, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     strh	w19, [x21, wzr, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	w3, [sp, x5]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	s9, [x27, x6]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	w10, [x30, x7, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	w11, [x29, x3, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	s12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	w13, [x27, x5, sxtx #2]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	w14, [x26, w6, uxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	w15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	w16, [x24, w8, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsw	x17, [x23, w9, sxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	w18, [x22, w10, sxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldrsw	x19, [x21, wzr, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	x3, [sp, x5]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	x9, [x27, x6]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	d10, [x30, x7, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	x11, [x29, x3, sxtx]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	x12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	x13, [x27, x5, sxtx #3]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     prfm	pldl1keep, [x26, w6, uxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	x15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	x16, [x24, w8, uxtw #3]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	x17, [x23, w9, sxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	x18, [x22, w10, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	d19, [x21, wzr, sxtw #3]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	q3, [sp, x5]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	q9, [x27, x6]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	q10, [x30, x7, lsl #4]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	q11, [x29, x3, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	q12, [x28, xzr, sxtx]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	q13, [x27, x5, sxtx #4]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	q14, [x26, w6, uxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	q15, [x25, w7, uxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	q16, [x24, w8, uxtw #4]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	q17, [x23, w9, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	q18, [x22, w10, sxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	q19, [x21, wzr, sxtw #4]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	w3, w5, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stp	wzr, w9, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	w2, wzr, [sp, #-256]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	w9, w10, [sp, #4]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldpsw	x9, x10, [sp, #4]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldpsw	x9, x10, [x2, #-256]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldpsw	x20, x30, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	x21, x29, [x2, #504]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	x22, x23, [x3, #-512]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	x24, x25, [x4, #8]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	s29, s28, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stp	s27, s26, [sp, #-256]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	s1, s2, [x3, #44]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stp	d3, d5, [x9, #504]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stp	d7, d11, [x10, #-512]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	d2, d3, [x30, #-8]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stp	q3, q5, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stp	q17, q19, [sp, #1008]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	q23, q29, [x1, #-1024]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	w3, w5, [sp], #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stp	wzr, w9, [sp], #252
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	w2, wzr, [sp], #-256
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	w9, w10, [sp], #4
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldpsw	x9, x10, [sp], #4
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldpsw	x9, x10, [x2], #-256
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldpsw	x20, x30, [sp], #252
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	x21, x29, [x2], #504
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	x22, x23, [x3], #-512
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	x24, x25, [x4], #8
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	s29, s28, [sp], #252
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stp	s27, s26, [sp], #-256
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	s1, s2, [x3], #44
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stp	d3, d5, [x9], #504
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stp	d7, d11, [x10], #-512
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	d2, d3, [x30], #-8
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stp	q3, q5, [sp], #0
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stp	q17, q19, [sp], #1008
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	q23, q29, [x1], #-1024
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	w3, w5, [sp, #0]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stp	wzr, w9, [sp, #252]!
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	w2, wzr, [sp, #-256]!
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	w9, w10, [sp, #4]!
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldpsw	x9, x10, [sp, #4]!
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldpsw	x9, x10, [x2, #-256]!
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldpsw	x20, x30, [sp, #252]!
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	x21, x29, [x2, #504]!
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	x22, x23, [x3, #-512]!
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	x24, x25, [x4, #8]!
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	s29, s28, [sp, #252]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stp	s27, s26, [sp, #-256]!
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	s1, s2, [x3, #44]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stp	d3, d5, [x9, #504]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stp	d7, d11, [x10, #-512]!
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	d2, d3, [x30, #-8]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stp	q3, q5, [sp, #0]!
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stp	q17, q19, [sp, #1008]!
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldp	q23, q29, [x1, #-1024]!
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldnp	w3, w5, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stnp	wzr, w9, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldnp	w2, wzr, [sp, #-256]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldnp	w9, w10, [sp, #4]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldnp	x21, x29, [x2, #504]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldnp	x22, x23, [x3, #-512]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldnp	x24, x25, [x4, #8]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldnp	s29, s28, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stnp	s27, s26, [sp, #-256]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldnp	s1, s2, [x3, #44]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stnp	d3, d5, [x9, #504]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stnp	d7, d11, [x10, #-512]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldnp	d2, d3, [x30, #-8]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stnp	q3, q5, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stnp	q17, q19, [sp, #1008]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ldnp	q23, q29, [x1, #-1024]
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     mov	w3, #983055
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     mov	x10, #-6148914691236517206
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     and	w12, w23, w21
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     and	w16, w15, w1, lsl #1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     and	w9, w4, w10, lsl #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     and	w3, w30, w11
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     and	x3, x5, x7, lsl #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     and	x5, x14, x19, asr #4
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     and	w3, w17, w19, ror #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     and	w0, w2, wzr, lsr #17
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     and	w3, w30, w11, asr #2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     and	xzr, x4, x26
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     and	w3, wzr, w20, ror #2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     and	x7, x20, xzr, asr #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     bic	x13, x20, x14, lsl #47
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     bic	w2, w7, w9
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     orr	w2, w7, w0, asr #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     orr	x8, x9, x10, lsl #12
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     orn	x3, x5, x7, asr #2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     orn	w2, w5, w29
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ands	w7, wzr, w9, lsl #1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ands	x3, x5, x20, ror #63
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     bics	w3, w5, w7
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     bics	x3, xzr, x3, lsl #1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     tst	w3, w7, lsl #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     tst	x2, x20, asr #2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     mov	x3, x6
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     mov	x3, xzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     mov	wzr, w2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     mov	w3, w5
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     movz	w2, #0, lsl #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     mov	w2, #-1235
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     mov	x2, #5299989643264
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     mov	x2, #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     movk	w3, #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     movz	x4, #0, lsl #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     movk	w5, #0, lsl #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     movz	x6, #0, lsl #32
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     movk	x7, #0, lsl #32
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     movz	x8, #0, lsl #48
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     movk	x9, #0, lsl #48
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adr	x2, #1600
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adrp	x21, #6553600
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     adr	x0, #262144
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     tbz	x12, #62, #0
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     tbz	x12, #62, #4
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     tbz	x12, #62, #-32768
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     tbnz	x12, #60, #32764
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     b	#4
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     b	#-4
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     b	#134217724
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     br	x20
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     blr	xzr
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     ret	x10
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     ret
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     eret
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     drps
diff --git a/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-neon-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-neon-instructions.s
new file mode 100644
index 0000000000000..73eadc268bf26
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-neon-instructions.s
@@ -0,0 +1,3213 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=aarch64 -mcpu=cortex-a510 -instruction-tables < %s | FileCheck %s
+
+abs d29, d24
+abs v0.16b, v0.16b
+abs v0.2d, v0.2d
+abs v0.2s, v0.2s
+abs v0.4h, v0.4h
+abs v0.4s, v0.4s
+abs v0.8b, v0.8b
+abs v0.8h, v0.8h
+add d17, d31, d29
+add v0.8b, v0.8b, v0.8b
+addhn v0.2s, v0.2d, v0.2d
+addhn v0.4h, v0.4s, v0.4s
+addhn v0.8b, v0.8h, v0.8h
+addhn2 v0.16b, v0.8h, v0.8h
+addhn2 v0.4s, v0.2d, v0.2d
+addhn2 v0.8h, v0.4s, v0.4s
+addp v0.2d, v0.2d, v0.2d
+addp v0.8b, v0.8b, v0.8b
+and v0.8b, v0.8b, v0.8b
+bic v0.4h, #15, lsl #8
+bic v0.8b, v0.8b, v0.8b
+bif v0.16b, v0.16b, v0.16b
+bit v0.16b, v0.16b, v0.16b
+bsl v0.8b, v0.8b, v0.8b
+cls v0.16b, v0.16b
+cls v0.2s, v0.2s
+cls v0.4h, v0.4h
+cls v0.4s, v0.4s
+cls v0.8b, v0.8b
+cls v0.8h, v0.8h
+clz v0.16b, v0.16b
+clz v0.2s, v0.2s
+clz v0.4h, v0.4h
+clz v0.4s, v0.4s
+clz v0.8b, v0.8b
+clz v0.8h, v0.8h
+cmeq d20, d21, 0
+cmeq d20, d21, d22
+cmeq v0.16b, v0.16b, 0
+cmeq v0.16b, v0.16b, v0.16b
+cmge d20, d21, 0
+cmge d20, d21, d22
+cmge v0.4h, v0.4h, v0.4h
+cmge v0.8b, v0.8b, 0
+cmgt d20, d21, 0
+cmgt d20, d21, d22
+cmgt v0.2s, v0.2s, 0
+cmgt v0.4s, v0.4s, v0.4s
+cmhi d20, d21, d22
+cmhi v0.8h, v0.8h, v0.8h
+cmhs d20, d21, d22
+cmhs v0.8b, v0.8b, v0.8b
+cmle d20, d21, 0
+cmle v0.2d, v0.2d, 0
+cmlt d20, d21, 0
+cmlt v0.8h, v0.8h, 0
+cmtst d20, d21, d22
+cmtst v0.2s, v0.2s, v0.2s
+cnt v0.16b, v0.16b
+cnt v0.8b, v0.8b
+dup v0.16b,w28
+dup v0.2d,x28
+dup v0.2s,w28
+dup v0.4h,w28
+dup v0.4s,w28
+dup v0.8b,w28
+dup v0.8h,w28
+eor v0.16b, v0.16b, v0.16b
+ext v0.16b, v0.16b, v0.16b, #3
+ext v0.8b, v0.8b, v0.8b, #3
+fabd d29, d24, d20
+fabd s29, s24, s20
+fabd v0.4s, v0.4s, v0.4s
+fabs v0.2d, v0.2d
+fabs v0.2s, v0.2s
+fabs v0.4h, v0.4h
+fabs v0.4s, v0.4s
+fabs v0.8h, v0.8h
+facge d20, d21, d22
+facge s10, s11, s12
+facge v0.4s, v0.4s, v0.4s
+facgt d20, d21, d22
+facgt s10, s11, s12
+facgt v0.2d, v0.2d, v0.2d
+fadd v0.4s, v0.4s, v0.4s
+faddp v0.2s, v0.2s, v0.2s
+faddp v0.4s, v0.4s, v0.4s
+fcmeq d20, d21, #0.0
+fcmeq d20, d21, d22
+fcmeq s10, s11, #0.0
+fcmeq s10, s11, s12
+fcmeq v0.2s, v0.2s, #0.0
+fcmeq v0.2s, v0.2s, v0.2s
+fcmge d20, d21, #0.0
+fcmge d20, d21, d22
+fcmge s10, s11, #0.0
+fcmge s10, s11, s12
+fcmge v0.2d, v0.2d, #0.0
+fcmge v0.4s, v0.4s, v0.4s
+fcmgt d20, d21, #0.0
+fcmgt d20, d21, d22
+fcmgt s10, s11, #0.0
+fcmgt s10, s11, s12
+fcmgt v0.4s, v0.4s, #0.0
+fcmgt v0.4s, v0.4s, v0.4s
+fcmle d20, d21, #0.0
+fcmle s10, s11, #0.0
+fcmle v0.2d, v0.2d, #0.0
+fcmlt d20, d21, #0.0
+fcmlt s10, s11, #0.0
+fcmlt v0.4s, v0.4s, #0.0
+fcvtas d21, d14
+fcvtas s12, s13
+fcvtas v0.2d, v0.2d
+fcvtas v0.2s, v0.2s
+fcvtas v0.4h, v0.4h
+fcvtas v0.4s, v0.4s
+fcvtas v0.8h, v0.8h
+fcvtau d21, d14
+fcvtau s12, s13
+fcvtau v0.2d, v0.2d
+fcvtau v0.2s, v0.2s
+fcvtau v0.4h, v0.4h
+fcvtau v0.4s, v0.4s
+fcvtau v0.8h, v0.8h
+fcvtl v0.2d, v0.2s
+fcvtl v0.4s, v0.4h
+fcvtl2 v0.2d, v0.4s
+fcvtl2 v0.4s, v0.8h
+fcvtms d21, d14
+fcvtms s22, s13
+fcvtms v0.2d, v0.2d
+fcvtms v0.2s, v0.2s
+fcvtms v0.4h, v0.4h
+fcvtms v0.4s, v0.4s
+fcvtms v0.8h, v0.8h
+fcvtmu d21, d14
+fcvtmu s12, s13
+fcvtmu v0.2d, v0.2d
+fcvtmu v0.2s, v0.2s
+fcvtmu v0.4h, v0.4h
+fcvtmu v0.4s, v0.4s
+fcvtmu v0.8h, v0.8h
+fcvtn v0.2s, v0.2d
+fcvtn v0.4h, v0.4s
+fcvtn2 v0.4s, v0.2d
+fcvtn2 v0.8h, v0.4s
+fcvtns d21, d14
+fcvtns s22, s13
+fcvtns v0.2d, v0.2d
+fcvtns v0.2s, v0.2s
+fcvtns v0.4h, v0.4h
+fcvtns v0.4s, v0.4s
+fcvtns v0.8h, v0.8h
+fcvtnu d21, d14
+fcvtnu s12, s13
+fcvtnu v0.2d, v0.2d
+fcvtnu v0.2s, v0.2s
+fcvtnu v0.4h, v0.4h
+fcvtnu v0.4s, v0.4s
+fcvtnu v0.8h, v0.8h
+fcvtps d21, d14
+fcvtps s22, s13
+fcvtps v0.2d, v0.2d
+fcvtps v0.2s, v0.2s
+fcvtps v0.4h, v0.4h
+fcvtps v0.4s, v0.4s
+fcvtps v0.8h, v0.8h
+fcvtpu d21, d14
+fcvtpu s12, s13
+fcvtpu v0.2d, v0.2d
+fcvtpu v0.2s, v0.2s
+fcvtpu v0.4h, v0.4h
+fcvtpu v0.4s, v0.4s
+fcvtpu v0.8h, v0.8h
+fcvtxn s22, d13
+fcvtxn v0.2s, v0.2d
+fcvtxn2 v0.4s, v0.2d
+fcvtzs d21, d12, #1
+fcvtzs d21, d14
+fcvtzs s12, s13
+fcvtzs s21, s12, #1
+fcvtzs v0.2d, v0.2d
+fcvtzs v0.2d, v0.2d, #3
+fcvtzs v0.2s, v0.2s
+fcvtzs v0.2s, v0.2s, #3
+fcvtzs v0.4h, v0.4h
+fcvtzs v0.4s, v0.4s
+fcvtzs v0.4s, v0.4s, #3
+fcvtzs v0.8h, v0.8h
+fcvtzu d21, d12, #1
+fcvtzu d21, d14
+fcvtzu s12, s13
+fcvtzu s21, s12, #1
+fcvtzu v0.2d, v0.2d
+fcvtzu v0.2d, v0.2d, #3
+fcvtzu v0.2s, v0.2s
+fcvtzu v0.2s, v0.2s, #3
+fcvtzu v0.4h, v0.4h
+fcvtzu v0.4s, v0.4s
+fcvtzu v0.4s, v0.4s, #3
+fcvtzu v0.8h, v0.8h
+fdiv v0.2s, v0.2s, v0.2s
+fmax v0.2d, v0.2d, v0.2d
+fmax v0.2s, v0.2s, v0.2s
+fmax v0.4s, v0.4s, v0.4s
+fmaxnm v0.2d, v0.2d, v0.2d
+fmaxnm v0.2s, v0.2s, v0.2s
+fmaxnm v0.4s, v0.4s, v0.4s
+fmaxnmp v0.2d, v0.2d, v0.2d
+fmaxnmp v0.2s, v0.2s, v0.2s
+fmaxnmp v0.4s, v0.4s, v0.4s
+fmaxp v0.2d, v0.2d, v0.2d
+fmaxp v0.2s, v0.2s, v0.2s
+fmaxp v0.4s, v0.4s, v0.4s
+fmin v0.2d, v0.2d, v0.2d
+fmin v0.2s, v0.2s, v0.2s
+fmin v0.4s, v0.4s, v0.4s
+fminnm v0.2d, v0.2d, v0.2d
+fminnm v0.2s, v0.2s, v0.2s
+fminnm v0.4s, v0.4s, v0.4s
+fminnmp v0.2d, v0.2d, v0.2d
+fminnmp v0.2s, v0.2s, v0.2s
+fminnmp v0.4s, v0.4s, v0.4s
+fminp v0.2d, v0.2d, v0.2d
+fminp v0.2s, v0.2s, v0.2s
+fminp v0.4s, v0.4s, v0.4s
+fmla d0, d1, v0.d[1]
+fmla s0, s1, v0.s[3]
+fmla v0.2s, v0.2s, v0.2s
+fmls d0, d4, v0.d[1]
+fmls s3, s5, v0.s[3]
+fmls v0.2s, v0.2s, v0.2s
+fmov v0.2d, #-1.25
+fmov v0.2s, #13.0
+fmov v0.4s, #1.0
+fmul d0, d1, v0.d[1]
+fmul s0, s1, v0.s[3]
+fmul v0.2s, v0.2s, v0.2s
+fmulx d0, d4, v0.d[1]
+fmulx d23, d11, d1
+fmulx s20, s22, s15
+fmulx s3, s5, v0.s[3]
+fmulx v0.2d, v0.2d, v0.2d
+fmulx v0.2s, v0.2s, v0.2s
+fmulx v0.4s, v0.4s, v0.4s
+fneg v0.2d, v0.2d
+fneg v0.2s, v0.2s
+fneg v0.4h, v0.4h
+fneg v0.4s, v0.4s
+fneg v0.8h, v0.8h
+frecpe d13, d13
+frecpe s19, s14
+frecpe v0.2d, v0.2d
+frecpe v0.2s, v0.2s
+frecpe v0.4h, v0.4h
+frecpe v0.4s, v0.4s
+frecpe v0.8h, v0.8h
+frecps  v0.4s, v0.4s, v0.4s
+frecps d22, d30, d21
+frecps s21, s16, s13
+frecpx d16, d19
+frecpx s18, s10
+frinta v0.2d, v0.2d
+frinta v0.2s, v0.2s
+frinta v0.4h, v0.4h
+frinta v0.4s, v0.4s
+frinta v0.8h, v0.8h
+frinti v0.2d, v0.2d
+frinti v0.2s, v0.2s
+frinti v0.4h, v0.4h
+frinti v0.4s, v0.4s
+frinti v0.8h, v0.8h
+frintm v0.2d, v0.2d
+frintm v0.2s, v0.2s
+frintm v0.4h, v0.4h
+frintm v0.4s, v0.4s
+frintm v0.8h, v0.8h
+frintn v0.2d, v0.2d
+frintn v0.2s, v0.2s
+frintn v0.4h, v0.4h
+frintn v0.4s, v0.4s
+frintn v0.8h, v0.8h
+frintp v0.2d, v0.2d
+frintp v0.2s, v0.2s
+frintp v0.4h, v0.4h
+frintp v0.4s, v0.4s
+frintp v0.8h, v0.8h
+frintx v0.2d, v0.2d
+frintx v0.2s, v0.2s
+frintx v0.4h, v0.4h
+frintx v0.4s, v0.4s
+frintx v0.8h, v0.8h
+frintz v0.2d, v0.2d
+frintz v0.2s, v0.2s
+frintz v0.4h, v0.4h
+frintz v0.4s, v0.4s
+frintz v0.8h, v0.8h
+frsqrte d21, d12
+frsqrte s22, s13
+frsqrte v0.2d, v0.2d
+frsqrte v0.2s, v0.2s
+frsqrte v0.4h, v0.4h
+frsqrte v0.4s, v0.4s
+frsqrte v0.8h, v0.8h
+frsqrts d8, d22, d18
+frsqrts s21, s5, s12
+frsqrts v0.2d, v0.2d, v0.2d
+fsqrt v0.2d, v0.2d
+fsqrt v0.2s, v0.2s
+fsqrt v0.4h, v0.4h
+fsqrt v0.4s, v0.4s
+fsqrt v0.8h, v0.8h
+fsub v0.2s, v0.2s, v0.2s
+ld1 { v0.16b }, [x0]
+ld1 { v0.2d, v1.2d, v2.2d }, [x0], #48
+ld1 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+ld1 { v0.4s, v1.4s }, [sp], #32
+ld1 { v0.4s, v1.4s, v2.4s }, [sp]
+ld1 { v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
+ld1 { v0.8h }, [x15], x2
+ld1 { v0.8h, v1.8h }, [x15]
+ld1 { v0.b }[9], [x0]
+ld1 { v0.b }[9], [x0], #1
+ld1r { v0.16b }, [x0]
+ld1r { v0.16b }, [x0], #1
+ld1r { v0.8h }, [x15]
+ld1r { v0.8h }, [x15], #2
+ld2 { v0.16b, v1.16b }, [x0], x1
+ld2 { v0.8b, v1.8b }, [x0]
+ld2 { v0.h, v1.h }[7], [x15]
+ld2 { v0.h, v1.h }[7], [x15], #4
+ld2r { v0.2d, v1.2d }, [x0]
+ld2r { v0.2d, v1.2d }, [x0], #16
+ld2r { v0.4s, v1.4s }, [sp]
+ld2r { v0.4s, v1.4s }, [sp], #8
+ld3 { v0.4h, v1.4h, v2.4h }, [x15]
+ld3 { v0.8h, v1.8h, v2.8h }, [x15], x2
+ld3 { v0.s, v1.s, v2.s }[3], [sp]
+ld3 { v0.s, v1.s, v2.s }[3], [sp], x3
+ld3r { v0.4h, v1.4h, v2.4h }, [x15]
+ld3r { v0.4h, v1.4h, v2.4h }, [x15], #6
+ld3r { v0.8b, v1.8b, v2.8b }, [x0]
+ld3r { v0.8b, v1.8b, v2.8b }, [x0], #3
+ld4 { v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+ld4 { v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
+ld4 { v0.d, v1.d, v2.d, v3.d }[1], [x0]
+ld4 { v0.d, v1.d, v2.d, v3.d }[1], [x0], #32
+ld4 { v0.h, v1.h, v2.h, v3.h }[7], [x0], x0
+ld4r { v0.1d, v1.1d, v2.1d, v3.1d }, [sp]
+ld4r { v0.1d, v1.1d, v2.1d, v3.1d }, [sp], x7
+ld4r { v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+ld4r { v0.2s, v1.2s, v2.2s, v3.2s }, [sp], x30
+mla v0.8b, v0.8b, v0.8b
+mls v0.4h, v0.4h, v0.4h
+mov b0, v0.b[15]
+mov d6, v0.d[1]
+mov h2, v0.h[5]
+mov s17, v0.s[2]
+mov v0.16b, v0.16b
+mov v0.8b, v0.8b
+movi d15, #0xff00ff00ff00ff
+movi v0.16b, #31
+movi v0.2d, #0xff0000ff0000ffff
+movi v0.2s, #8, msl #8
+movi v0.4s, #255, lsl #24
+movi v0.8b, #255
+mul v0.8b, v0.8b, v0.8b
+mvni v0.2s, 0
+mvni v0.4s, #16, msl #16
+neg d29, d24
+neg v0.16b, v0.16b
+neg v0.2d, v0.2d
+neg v0.2s, v0.2s
+neg v0.4h, v0.4h
+neg v0.4s, v0.4s
+neg v0.8b, v0.8b
+neg v0.8h, v0.8h
+not v0.16b, v0.16b
+not v0.8b, v0.8b
+orn v0.16b, v0.16b, v0.16b
+orr v0.16b, v0.16b, v0.16b
+orr v0.8h, #31
+pmul v0.16b, v0.16b, v0.16b
+pmul v0.8b, v0.8b, v0.8b
+pmull v0.8h, v0.8b, v0.8b
+pmull2 v0.8h, v0.16b, v0.16b
+raddhn v0.2s, v0.2d, v0.2d
+raddhn v0.4h, v0.4s, v0.4s
+raddhn v0.8b, v0.8h, v0.8h
+raddhn2 v0.16b, v0.8h, v0.8h
+raddhn2 v0.4s, v0.2d, v0.2d
+raddhn2 v0.8h, v0.4s, v0.4s
+rbit v0.16b, v0.16b
+rbit v0.8b, v0.8b
+rev16 v21.8b, v1.8b
+rev16 v30.16b, v31.16b
+rev32 v0.4h, v9.4h
+rev32 v21.8b, v1.8b
+rev32 v30.16b, v31.16b
+rev32 v4.8h, v7.8h
+rev64 v0.16b, v31.16b
+rev64 v1.8b, v9.8b
+rev64 v13.4h, v21.4h
+rev64 v2.8h, v4.8h
+rev64 v4.2s, v0.2s
+rev64 v6.4s, v8.4s
+rshrn v0.2s, v0.2d, #3
+rshrn v0.4h, v0.4s, #3
+rshrn v0.8b, v0.8h, #3
+rshrn2 v0.16b, v0.8h, #3
+rshrn2 v0.4s, v0.2d, #3
+rshrn2 v0.8h, v0.4s, #3
+rsubhn v0.2s, v0.2d, v0.2d
+rsubhn v0.4h, v0.4s, v0.4s
+rsubhn v0.8b, v0.8h, v0.8h
+rsubhn2 v0.16b, v0.8h, v0.8h
+rsubhn2 v0.4s, v0.2d, v0.2d
+rsubhn2 v0.8h, v0.4s, v0.4s
+saba v0.16b, v0.16b, v0.16b
+sabal v0.2d, v0.2s, v0.2s
+sabal v0.4s, v0.4h, v0.4h
+sabal v0.8h, v0.8b, v0.8b
+sabal2 v0.2d, v0.4s, v0.4s
+sabal2 v0.4s, v0.8h, v0.8h
+sabal2 v0.8h, v0.16b, v0.16b
+sabd v0.4h, v0.4h, v0.4h
+sabdl v0.2d, v0.2s, v0.2s
+sabdl v0.4s, v0.4h, v0.4h
+sabdl v0.8h, v0.8b, v0.8b
+sabdl2 v0.2d, v0.4s, v0.4s
+sabdl2 v0.4s, v0.8h, v0.8h
+sabdl2 v0.8h, v0.16b, v0.16b
+sadalp v0.1d, v0.2s
+sadalp v0.2d, v0.4s
+sadalp v0.2s, v0.4h
+sadalp v0.4h, v0.8b
+sadalp v0.4s, v0.8h
+sadalp v0.8h, v0.16b
+saddl v0.2d, v0.2s, v0.2s
+saddl v0.4s, v0.4h, v0.4h
+saddl v0.8h, v0.8b, v0.8b
+saddl2 v0.2d, v0.4s, v0.4s
+saddl2 v0.4s, v0.8h, v0.8h
+saddl2 v0.8h, v0.16b, v0.16b
+saddlp v0.1d, v0.2s
+saddlp v0.2d, v0.4s
+saddlp v0.2s, v0.4h
+saddlp v0.4h, v0.8b
+saddlp v0.4s, v0.8h
+saddlp v0.8h, v0.16b
+saddw v0.2d, v0.2d, v0.2s
+saddw v0.4s, v0.4s, v0.4h
+saddw v0.8h, v0.8h, v0.8b
+saddw2 v0.2d, v0.2d, v0.4s
+saddw2 v0.4s, v0.4s, v0.8h
+saddw2 v0.8h, v0.8h, v0.16b
+scvtf d21, d12
+scvtf d21, d12, #64
+scvtf s22, s13
+scvtf s22, s13, #32
+scvtf v0.2d, v0.2d
+scvtf v0.2d, v0.2d, #3
+scvtf v0.2s, v0.2s
+scvtf v0.2s, v0.2s, #3
+scvtf v0.4h, v0.4h
+scvtf v0.4s, v0.4s
+scvtf v0.4s, v0.4s, #3
+scvtf v0.8h, v0.8h
+shadd v0.8b, v0.8b, v0.8b
+shl d7, d10, #12
+shl v0.16b, v0.16b, #3
+shl v0.2d, v0.2d, #3
+shl v0.4h, v0.4h, #3
+shl v0.4s, v0.4s, #3
+shll	v0.2d, v0.2s, #32
+shll	v0.4s, v0.4h, #16
+shll	v0.8h, v0.8b, #8
+shll v0.2d, v0.2s, #32
+shll v0.4s, v0.4h, #16
+shll v0.8h, v0.8b, #8
+shll2	v0.2d, v0.4s, #32
+shll2	v0.4s, v0.8h, #16
+shll2	v0.8h, v0.16b, #8
+shll2 v0.2d, v0.4s, #32
+shll2 v0.4s, v0.8h, #16
+shll2 v0.8h, v0.16b, #8
+shrn v0.2s, v0.2d, #3
+shrn v0.4h, v0.4s, #3
+shrn v0.8b, v0.8h, #3
+shrn2 v0.16b, v0.8h, #3
+shrn2 v0.4s, v0.2d, #3
+shrn2 v0.8h, v0.4s, #3
+shsub v0.2s, v0.2s, v0.2s
+shsub v0.4h, v0.4h, v0.4h
+sli d10, d14, #12
+sli v0.16b, v0.16b, #3
+sli v0.2d, v0.2d, #3
+sli v0.2s, v0.2s, #3
+sli v0.4h, v0.4h, #3
+sli v0.4s, v0.4s, #3
+sli v0.8b, v0.8b, #3
+sli v0.8h, v0.8h, #3
+smax v0.2s, v0.2s, v0.2s
+smax v0.4h, v0.4h, v0.4h
+smax v0.8b, v0.8b, v0.8b
+smaxp v0.2s, v0.2s, v0.2s
+smaxp v0.4h, v0.4h, v0.4h
+smaxp v0.8b, v0.8b, v0.8b
+smin v0.16b, v0.16b, v0.16b
+smin v0.4s, v0.4s, v0.4s
+smin v0.8h, v0.8h, v0.8h
+sminp v0.16b, v0.16b, v0.16b
+sminp v0.4s, v0.4s, v0.4s
+sminp v0.8h, v0.8h, v0.8h
+smlal v0.2d, v0.2s, v0.2s
+smlal v0.4s, v0.4h, v0.4h
+smlal v0.8h, v0.8b, v0.8b
+smlal2 v0.2d, v0.4s, v0.4s
+smlal2 v0.4s, v0.8h, v0.8h
+smlal2 v0.8h, v0.16b, v0.16b
+smlsl v0.2d, v0.2s, v0.2s
+smlsl v0.4s, v0.4h, v0.4h
+smlsl v0.8h, v0.8b, v0.8b
+smlsl2 v0.2d, v0.4s, v0.4s
+smlsl2 v0.4s, v0.8h, v0.8h
+smlsl2 v0.8h, v0.16b, v0.16b
+smull v0.2d, v0.2s, v0.2s
+smull v0.4s, v0.4h, v0.4h
+smull v0.8h, v0.8b, v0.8b
+smull2 v0.2d, v0.4s, v0.4s
+smull2 v0.4s, v0.8h, v0.8h
+smull2 v0.8h, v0.16b, v0.16b
+sqabs b19, b14
+sqabs d18, d12
+sqabs h21, h15
+sqabs s20, s12
+sqabs v0.16b, v0.16b
+sqabs v0.2d, v0.2d
+sqabs v0.2s, v0.2s
+sqabs v0.4h, v0.4h
+sqabs v0.4s, v0.4s
+sqabs v0.8b, v0.8b
+sqabs v0.8h, v0.8h
+sqadd b20, b11, b15
+sqadd v0.16b, v0.16b, v0.16b
+sqadd v0.2s, v0.2s, v0.2s
+sqdmlal d19, s24, s12
+sqdmlal d8, s9, v0.s[1]
+sqdmlal s0, h0, v0.h[3]
+sqdmlal s17, h27, h12
+sqdmlal v0.2d, v0.2s, v0.2s
+sqdmlal v0.4s, v0.4h, v0.4h
+sqdmlal2 v0.2d, v0.4s, v0.4s
+sqdmlal2 v0.4s, v0.8h, v0.8h
+sqdmlsl d12, s23, s13
+sqdmlsl d8, s9, v0.s[1]
+sqdmlsl s0, h0, v0.h[3]
+sqdmlsl s14, h12, h25
+sqdmlsl v0.2d, v0.2s, v0.2s
+sqdmlsl v0.4s, v0.4h, v0.4h
+sqdmlsl2 v0.2d, v0.4s, v0.4s
+sqdmlsl2 v0.4s, v0.8h, v0.8h
+sqdmulh h10, h11, h12
+sqdmulh h7, h15, v0.h[3]
+sqdmulh s15, s14, v0.s[1]
+sqdmulh s20, s21, s2
+sqdmulh v0.2s, v0.2s, v0.2s
+sqdmulh v0.4s, v0.4s, v0.4s
+sqdmull d1, s1, v0.s[1]
+sqdmull d15, s22, s12
+sqdmull s1, h1, v0.h[3]
+sqdmull s12, h22, h12
+sqdmull v0.2d, v0.2s, v0.2s
+sqdmull v0.4s, v0.4h, v0.4h
+sqdmull2 v0.2d, v0.4s, v0.4s
+sqdmull2 v0.4s, v0.8h, v0.8h
+sqneg b19, b14
+sqneg d18, d12
+sqneg h21, h15
+sqneg s20, s12
+sqneg v0.16b, v0.16b
+sqneg v0.2d, v0.2d
+sqneg v0.2s, v0.2s
+sqneg v0.4h, v0.4h
+sqneg v0.4s, v0.4s
+sqneg v0.8b, v0.8b
+sqneg v0.8h, v0.8h
+sqrdmulh h10, h11, h12
+sqrdmulh h7, h15, v0.h[3]
+sqrdmulh s15, s14, v0.s[1]
+sqrdmulh s20, s21, s2
+sqrdmulh v0.4h, v0.4h, v0.4h
+sqrdmulh v0.8h, v0.8h, v0.8h
+sqrshl d31, d31, d31
+sqrshl h3, h4, h15
+sqrshl v0.2s, v0.2s, v0.2s
+sqrshl v0.4h, v0.4h, v0.4h
+sqrshl v0.8b, v0.8b, v0.8b
+sqrshrn b10, h13, #2
+sqrshrn h15, s10, #6
+sqrshrn s15, d12, #9
+sqrshrn v0.2s, v0.2d, #3
+sqrshrn v0.4h, v0.4s, #3
+sqrshrn v0.8b, v0.8h, #3
+sqrshrn2 v0.16b, v0.8h, #3
+sqrshrn2 v0.4s, v0.2d, #3
+sqrshrn2 v0.8h, v0.4s, #3
+sqrshrun b17, h10, #6
+sqrshrun h10, s13, #15
+sqrshrun s22, d16, #31
+sqrshrun v0.2s, v0.2d, #3
+sqrshrun v0.4h, v0.4s, #3
+sqrshrun v0.8b, v0.8h, #3
+sqrshrun2 v0.16b, v0.8h, #3
+sqrshrun2 v0.4s, v0.2d, #3
+sqrshrun2 v0.8h, v0.4s, #3
+sqshl b11, b19, #7
+sqshl d15, d16, #51
+sqshl d31, d31, d31
+sqshl h13, h18, #11
+sqshl h3, h4, h15
+sqshl s14, s17, #22
+sqshl v0.16b, v0.16b, #3
+sqshl v0.2d, v0.2d, #3
+sqshl v0.2s, v0.2s, #3
+sqshl v0.2s, v0.2s, v0.2s
+sqshl v0.4h, v0.4h, #3
+sqshl v0.4h, v0.4h, v0.4h
+sqshl v0.4s, v0.4s, #3
+sqshl v0.8b, v0.8b, #3
+sqshl v0.8b, v0.8b, v0.8b
+sqshl v0.8h, v0.8h, #3
+sqshlu b15, b18, #6
+sqshlu d11, d13, #32
+sqshlu h19, h17, #6
+sqshlu s16, s14, #25
+sqshlu v0.16b, v0.16b, #3
+sqshlu v0.2d, v0.2d, #3
+sqshlu v0.2s, v0.2s, #3
+sqshlu v0.4h, v0.4h, #3
+sqshlu v0.4s, v0.4s, #3
+sqshlu v0.8b, v0.8b, #3
+sqshlu v0.8h, v0.8h, #3
+sqshrn b10, h15, #5
+sqshrn h17, s10, #4
+sqshrn s18, d10, #31
+sqshrn v0.2s, v0.2d, #3
+sqshrn v0.4h, v0.4s, #3
+sqshrn v0.8b, v0.8h, #3
+sqshrn2 v0.16b, v0.8h, #3
+sqshrn2 v0.4s, v0.2d, #3
+sqshrn2 v0.8h, v0.4s, #3
+sqshrun b15, h10, #7
+sqshrun h20, s14, #3
+sqshrun s10, d15, #15
+sqshrun v0.2s, v0.2d, #3
+sqshrun v0.4h, v0.4s, #3
+sqshrun v0.8b, v0.8h, #3
+sqshrun2 v0.16b, v0.8h, #3
+sqshrun2 v0.4s, v0.2d, #3
+sqshrun2 v0.8h, v0.4s, #3
+sqsub s20, s10, s7
+sqsub v0.2d, v0.2d, v0.2d
+sqsub v0.4s, v0.4s, v0.4s
+sqsub v0.8b, v0.8b, v0.8b
+sqxtn b18, h18
+sqxtn h20, s17
+sqxtn s19, d14
+sqxtn v0.2s, v0.2d
+sqxtn v0.4h, v0.4s
+sqxtn v0.8b, v0.8h
+sqxtn2 v0.16b, v0.8h
+sqxtn2 v0.4s, v0.2d
+sqxtn2 v0.8h, v0.4s
+sqxtun b19, h14
+sqxtun h21, s15
+sqxtun s20, d12
+sqxtun v0.2s, v0.2d
+sqxtun v0.4h, v0.4s
+sqxtun v0.8b, v0.8h
+sqxtun2 v0.16b, v0.8h
+sqxtun2 v0.4s, v0.2d
+sqxtun2 v0.8h, v0.4s
+srhadd v0.2s, v0.2s, v0.2s
+srhadd v0.4h, v0.4h, v0.4h
+srhadd v0.8b, v0.8b, v0.8b
+sri d10, d12, #14
+sri v0.16b, v0.16b, #3
+sri v0.2d, v0.2d, #3
+sri v0.2s, v0.2s, #3
+sri v0.4h, v0.4h, #3
+sri v0.4s, v0.4s, #3
+sri v0.8b, v0.8b, #3
+sri v0.8h, v0.8h, #3
+srshl d16, d16, d16
+srshl v0.2s, v0.2s, v0.2s
+srshl v0.4h, v0.4h, v0.4h
+srshl v0.8b, v0.8b, v0.8b
+srshr d19, d18, #7
+srshr v0.16b, v0.16b, #3
+srshr v0.2d, v0.2d, #3
+srshr v0.2s, v0.2s, #3
+srshr v0.4h, v0.4h, #3
+srshr v0.4s, v0.4s, #3
+srshr v0.8b, v0.8b, #3
+srshr v0.8h, v0.8h, #3
+srsra d15, d11, #19
+srsra v0.16b, v0.16b, #3
+srsra v0.2d, v0.2d, #3
+srsra v0.2s, v0.2s, #3
+srsra v0.4h, v0.4h, #3
+srsra v0.4s, v0.4s, #3
+srsra v0.8b, v0.8b, #3
+srsra v0.8h, v0.8h, #3
+sshl d31, d31, d31
+sshl v0.2d, v0.2d, v0.2d
+sshl v0.2s, v0.2s, v0.2s
+sshl v0.4h, v0.4h, v0.4h
+sshl v0.8b, v0.8b, v0.8b
+sshll v0.2d, v0.2s, #3
+sshll2 v0.4s, v0.8h, #3
+sshr d15, d16, #12
+sshr v0.16b, v0.16b, #3
+sshr v0.2d, v0.2d, #3
+sshr v0.2s, v0.2s, #3
+sshr v0.4h, v0.4h, #3
+sshr v0.4s, v0.4s, #3
+sshr v0.8b, v0.8b, #3
+sshr v0.8h, v0.8h, #3
+ssra d18, d12, #21
+ssra v0.16b, v0.16b, #3
+ssra v0.2d, v0.2d, #3
+ssra v0.2s, v0.2s, #3
+ssra v0.4h, v0.4h, #3
+ssra v0.4s, v0.4s, #3
+ssra v0.8b, v0.8b, #3
+ssra v0.8h, v0.8h, #3
+ssubl v0.2d, v0.2s, v0.2s
+ssubl v0.4s, v0.4h, v0.4h
+ssubl v0.8h, v0.8b, v0.8b
+ssubl2 v0.2d, v0.4s, v0.4s
+ssubl2 v0.4s, v0.8h, v0.8h
+ssubl2 v0.8h, v0.16b, v0.16b
+ssubw v0.2d, v0.2d, v0.2s
+ssubw v0.4s, v0.4s, v0.4h
+ssubw v0.8h, v0.8h, v0.8b
+ssubw2 v0.2d, v0.2d, v0.4s
+ssubw2 v0.4s, v0.4s, v0.8h
+ssubw2 v0.8h, v0.8h, v0.16b
+st1 { v0.16b }, [x0]
+st1 { v0.2d, v1.2d, v2.2d }, [x0], #48
+st1 { v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+st1 { v0.4s, v1.4s }, [sp], #32
+st1 { v0.4s, v1.4s, v2.4s }, [sp]
+st1 { v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
+st1 { v0.8h }, [x15], x2
+st1 { v0.8h, v1.8h }, [x15]
+st1 { v0.d }[1], [x0]
+st1 { v0.d }[1], [x0], #8
+st2 { v0.16b, v1.16b }, [x0], x1
+st2 { v0.8b, v1.8b }, [x0]
+st2 { v0.s, v1.s }[3], [sp]
+st2 { v0.s, v1.s }[3], [sp], #8
+st3 { v0.4h, v1.4h, v2.4h }, [x15]
+st3 { v0.8h, v1.8h, v2.8h }, [x15], x2
+st3 { v0.h, v1.h, v2.h }[7], [x15]
+st3 { v0.h, v1.h, v2.h }[7], [x15], #6
+st4 { v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+st4 { v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
+st4 { v0.b, v1.b, v2.b, v3.b }[9], [x0]
+st4 { v0.b, v1.b, v2.b, v3.b }[9], [x0], x5
+sub d15, d5, d16
+sub v0.2d, v0.2d, v0.2d
+suqadd b19, b14
+suqadd d18, d22
+suqadd h20, h15
+suqadd s21, s12
+suqadd v0.16b, v0.16b
+suqadd v0.2d, v0.2d
+suqadd v0.2s, v0.2s
+suqadd v0.4h, v0.4h
+suqadd v0.4s, v0.4s
+suqadd v0.8b, v0.8b
+suqadd v0.8h, v0.8h
+tbl v0.16b, { v0.16b }, v0.16b
+tbl v0.16b, { v0.16b, v1.16b }, v0.16b
+tbl v0.16b, { v0.16b, v1.16b, v2.16b }, v0.16b
+tbl v0.16b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.16b
+tbl v0.8b, { v0.16b }, v0.8b
+tbl v0.8b, { v0.16b, v1.16b }, v0.8b
+tbl v0.8b, { v0.16b, v1.16b, v2.16b }, v0.8b
+tbl v0.8b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.8b
+tbx v0.16b, { v0.16b }, v0.16b
+tbx v0.16b, { v0.16b, v1.16b }, v0.16b
+tbx v0.16b, { v0.16b, v1.16b, v2.16b }, v0.16b
+tbx v0.16b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.16b
+tbx v0.8b, { v0.16b }, v0.8b
+tbx v0.8b, { v0.16b, v1.16b }, v0.8b
+tbx v0.8b, { v0.16b, v1.16b, v2.16b }, v0.8b
+tbx v0.8b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.8b
+trn1	v0.16b, v0.16b, v0.16b
+trn1	v0.2d, v0.2d, v0.2d
+trn1	v0.2s, v0.2s, v0.2s
+trn1	v0.4h, v0.4h, v0.4h
+trn1	v0.4s, v0.4s, v0.4s
+trn1	v0.8b, v0.8b, v0.8b
+trn1	v0.8h, v0.8h, v0.8h
+trn2	v0.16b, v0.16b, v0.16b
+trn2	v0.2d, v0.2d, v0.2d
+trn2	v0.2s, v0.2s, v0.2s
+trn2	v0.4h, v0.4h, v0.4h
+trn2	v0.4s, v0.4s, v0.4s
+trn2	v0.8b, v0.8b, v0.8b
+trn2	v0.8h, v0.8h, v0.8h
+uaba v0.8b, v0.8b, v0.8b
+uabal v0.2d, v0.2s, v0.2s
+uabal v0.4s, v0.4h, v0.4h
+uabal v0.8h, v0.8b, v0.8b
+uabal2 v0.2d, v0.4s, v0.4s
+uabal2 v0.4s, v0.8h, v0.8h
+uabal2 v0.8h, v0.16b, v0.16b
+uabd v0.4h, v0.4h, v0.4h
+uabdl v0.2d, v0.2s, v0.2s
+uabdl v0.4s, v0.4h, v0.4h
+uabdl v0.8h, v0.8b, v0.8b
+uabdl2 v0.2d, v0.4s, v0.4s
+uabdl2 v0.4s, v0.8h, v0.8h
+uabdl2 v0.8h, v0.16b, v0.16b
+uadalp v0.1d, v0.2s
+uadalp v0.2d, v0.4s
+uadalp v0.2s, v0.4h
+uadalp v0.4h, v0.8b
+uadalp v0.4s, v0.8h
+uadalp v0.8h, v0.16b
+uaddl v0.2d, v0.2s, v0.2s
+uaddl v0.4s, v0.4h, v0.4h
+uaddl v0.8h, v0.8b, v0.8b
+uaddl2 v0.2d, v0.4s, v0.4s
+uaddl2 v0.4s, v0.8h, v0.8h
+uaddl2 v0.8h, v0.16b, v0.16b
+uaddlp v0.1d, v0.2s
+uaddlp v0.2d, v0.4s
+uaddlp v0.2s, v0.4h
+uaddlp v0.4h, v0.8b
+uaddlp v0.4s, v0.8h
+uaddlp v0.8h, v0.16b
+uaddw v0.2d, v0.2d, v0.2s
+uaddw v0.4s, v0.4s, v0.4h
+uaddw v0.8h, v0.8h, v0.8b
+uaddw2 v0.2d, v0.2d, v0.4s
+uaddw2 v0.4s, v0.4s, v0.8h
+uaddw2 v0.8h, v0.8h, v0.16b
+ucvtf d21, d14
+ucvtf d21, d14, #64
+ucvtf s22, s13
+ucvtf s22, s13, #32
+ucvtf v0.2d, v0.2d
+ucvtf v0.2d, v0.2d, #3
+ucvtf v0.2s, v0.2s
+ucvtf v0.2s, v0.2s, #3
+ucvtf v0.4h, v0.4h
+ucvtf v0.4s, v0.4s
+ucvtf v0.4s, v0.4s, #3
+ucvtf v0.8h, v0.8h
+uhadd v0.16b, v0.16b, v0.16b
+uhadd v0.8h, v0.8h, v0.8h
+uhsub v0.4s, v0.4s, v0.4s
+umax v0.16b, v0.16b, v0.16b
+umax v0.4s, v0.4s, v0.4s
+umax v0.8h, v0.8h, v0.8h
+umaxp v0.16b, v0.16b, v0.16b
+umaxp v0.4s, v0.4s, v0.4s
+umaxp v0.8h, v0.8h, v0.8h
+umin v0.2s, v0.2s, v0.2s
+umin v0.4h, v0.4h, v0.4h
+umin v0.8b, v0.8b, v0.8b
+uminp v0.2s, v0.2s, v0.2s
+uminp v0.4h, v0.4h, v0.4h
+uminp v0.8b, v0.8b, v0.8b
+umlal v0.2d, v0.2s, v0.2s
+umlal v0.4s, v0.4h, v0.4h
+umlal v0.8h, v0.8b, v0.8b
+umlal2 v0.2d, v0.4s, v0.4s
+umlal2 v0.4s, v0.8h, v0.8h
+umlal2 v0.8h, v0.16b, v0.16b
+umlsl v0.2d, v0.2s, v0.2s
+umlsl v0.4s, v0.4h, v0.4h
+umlsl v0.8h, v0.8b, v0.8b
+umlsl2 v0.2d, v0.4s, v0.4s
+umlsl2 v0.4s, v0.8h, v0.8h
+umlsl2 v0.8h, v0.16b, v0.16b
+umull v0.2d, v0.2s, v0.2s
+umull v0.4s, v0.4h, v0.4h
+umull v0.8h, v0.8b, v0.8b
+umull2 v0.2d, v0.4s, v0.4s
+umull2 v0.4s, v0.8h, v0.8h
+umull2 v0.8h, v0.16b, v0.16b
+uqadd h0, h1, h5
+uqadd v0.8h, v0.8h, v0.8h
+uqrshl b11, b20, b30
+uqrshl s23, s20, s16
+uqrshl v0.16b, v0.16b, v0.16b
+uqrshl v0.4s, v0.4s, v0.4s
+uqrshl v0.4s, v0.4s, v0.4s
+uqrshl v0.8h, v0.8h, v0.8h
+uqrshrn b10, h12, #5
+uqrshrn h12, s10, #14
+uqrshrn s10, d10, #25
+uqrshrn v0.2s, v0.2d, #3
+uqrshrn v0.4h, v0.4s, #3
+uqrshrn v0.8b, v0.8h, #3
+uqrshrn2 v0.16b, v0.8h, #3
+uqrshrn2 v0.4s, v0.2d, #3
+uqrshrn2 v0.8h, v0.4s, #3
+uqshl b11, b20, b30
+uqshl b18, b15, #6
+uqshl d15, d12, #19
+uqshl h11, h18, #7
+uqshl s14, s19, #18
+uqshl s23, s20, s16
+uqshl v0.16b, v0.16b, #3
+uqshl v0.16b, v0.16b, v0.16b
+uqshl v0.2d, v0.2d, #3
+uqshl v0.2d, v0.2d, v0.2d
+uqshl v0.2s, v0.2s, #3
+uqshl v0.4h, v0.4h, #3
+uqshl v0.4s, v0.4s, #3
+uqshl v0.4s, v0.4s, v0.4s
+uqshl v0.8b, v0.8b, #3
+uqshl v0.8h, v0.8h, #3
+uqshl v0.8h, v0.8h, v0.8h
+uqshrn b12, h10, #7
+uqshrn h10, s14, #5
+uqshrn s10, d12, #13
+uqshrn v0.2s, v0.2d, #3
+uqshrn v0.4h, v0.4s, #3
+uqshrn v0.8b, v0.8h, #3
+uqshrn2 v0.16b, v0.8h, #3
+uqshrn2 v0.4s, v0.2d, #3
+uqshrn2 v0.8h, v0.4s, #3
+uqsub d16, d16, d16
+uqsub v0.4h, v0.4h, v0.4h
+uqxtn b18, h18
+uqxtn h20, s17
+uqxtn s19, d14
+uqxtn v0.2s, v0.2d
+uqxtn v0.4h, v0.4s
+uqxtn v0.8b, v0.8h
+uqxtn2 v0.16b, v0.8h
+uqxtn2 v0.4s, v0.2d
+uqxtn2 v0.8h, v0.4s
+urecpe v0.2s, v0.2s
+urecpe v0.4s, v0.4s
+urhadd v0.16b, v0.16b, v0.16b
+urhadd v0.4s, v0.4s, v0.4s
+urhadd v0.8h, v0.8h, v0.8h
+urshl d8, d7, d4
+urshl v0.16b, v0.16b, v0.16b
+urshl v0.2d, v0.2d, v0.2d
+urshl v0.4s, v0.4s, v0.4s
+urshl v0.8h, v0.8h, v0.8h
+urshr d20, d23, #31
+urshr v0.16b, v0.16b, #3
+urshr v0.2d, v0.2d, #3
+urshr v0.2s, v0.2s, #3
+urshr v0.4h, v0.4h, #3
+urshr v0.4s, v0.4s, #3
+urshr v0.8b, v0.8b, #3
+urshr v0.8h, v0.8h, #3
+ursqrte v0.2s, v0.2s
+ursqrte v0.4s, v0.4s
+ursra d18, d10, #13
+ursra v0.16b, v0.16b, #3
+ursra v0.2d, v0.2d, #3
+ursra v0.2s, v0.2s, #3
+ursra v0.4h, v0.4h, #3
+ursra v0.4s, v0.4s, #3
+ursra v0.8b, v0.8b, #3
+ursra v0.8h, v0.8h, #3
+ushl d0, d0, d0
+ushl v0.16b, v0.16b, v0.16b
+ushl v0.4s, v0.4s, v0.4s
+ushl v0.8h, v0.8h, v0.8h
+ushll v0.4s, v0.4h, #3
+ushll2 v0.8h, v0.16b, #3
+ushr d10, d17, #18
+ushr v0.16b, v0.16b, #3
+ushr v0.2d, v0.2d, #3
+ushr v0.2s, v0.2s, #3
+ushr v0.4h, v0.4h, #3
+ushr v0.4s, v0.4s, #3
+ushr v0.8b, v0.8b, #3
+ushr v0.8h, v0.8h, #3
+usqadd b19, b14
+usqadd d18, d22
+usqadd h20, h15
+usqadd s21, s12
+usqadd v0.16b, v0.16b
+usqadd v0.2d, v0.2d
+usqadd v0.2s, v0.2s
+usqadd v0.4h, v0.4h
+usqadd v0.4s, v0.4s
+usqadd v0.8b, v0.8b
+usqadd v0.8h, v0.8h
+usra d20, d13, #61
+usra v0.16b, v0.16b, #3
+usra v0.2d, v0.2d, #3
+usra v0.2s, v0.2s, #3
+usra v0.4h, v0.4h, #3
+usra v0.4s, v0.4s, #3
+usra v0.8b, v0.8b, #3
+usra v0.8h, v0.8h, #3
+usubl v0.2d, v0.2s, v0.2s
+usubl v0.4s, v0.4h, v0.4h
+usubl v0.8h, v0.8b, v0.8b
+usubl2 v0.2d, v0.4s, v0.4s
+usubl2 v0.4s, v0.8h, v0.8h
+usubl2 v0.8h, v0.16b, v0.16b
+usubw v0.2d, v0.2d, v0.2s
+usubw v0.4s, v0.4s, v0.4h
+usubw v0.8h, v0.8h, v0.8b
+usubw2 v0.2d, v0.2d, v0.4s
+usubw2 v0.4s, v0.4s, v0.8h
+usubw2 v0.8h, v0.8h, v0.16b
+uzp1	v0.16b, v0.16b, v0.16b
+uzp1	v0.2d, v0.2d, v0.2d
+uzp1	v0.2s, v0.2s, v0.2s
+uzp1	v0.4h, v0.4h, v0.4h
+uzp1	v0.4s, v0.4s, v0.4s
+uzp1	v0.8b, v0.8b, v0.8b
+uzp1	v0.8h, v0.8h, v0.8h
+uzp2	v0.16b, v0.16b, v0.16b
+uzp2	v0.2d, v0.2d, v0.2d
+uzp2	v0.2s, v0.2s, v0.2s
+uzp2	v0.4h, v0.4h, v0.4h
+uzp2	v0.4s, v0.4s, v0.4s
+uzp2	v0.8b, v0.8b, v0.8b
+uzp2	v0.8h, v0.8h, v0.8h
+xtn v0.2s, v0.2d
+xtn v0.4h, v0.4s
+xtn v0.8b, v0.8h
+xtn2 v0.16b, v0.8h
+xtn2 v0.4s, v0.2d
+xtn2 v0.8h, v0.4s
+zip1	v0.16b, v0.16b, v0.16b
+zip1	v0.2d, v0.2d, v0.2d
+zip1	v0.2s, v0.2s, v0.2s
+zip1	v0.4h, v0.4h, v0.4h
+zip1	v0.4s, v0.4s, v0.4s
+zip1	v0.8b, v0.8b, v0.8b
+zip1	v0.8h, v0.8h, v0.8h
+zip2	v0.16b, v0.16b, v0.16b
+zip2	v0.2d, v0.2d, v0.2d
+zip2	v0.2s, v0.2s, v0.2s
+zip2	v0.4h, v0.4h, v0.4h
+zip2	v0.4s, v0.4s, v0.4s
+zip2	v0.8b, v0.8b, v0.8b
+zip2	v0.8h, v0.8h, v0.8h
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      4     0.50                        abs	d29, d24
+# CHECK-NEXT:  1      4     0.50                        abs	v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        abs	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        abs	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        abs	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        abs	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        abs	v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        abs	v0.8h, v0.8h
+# CHECK-NEXT:  1      3     0.50                        add	d17, d31, d29
+# CHECK-NEXT:  1      3     0.50                        add	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        addhn	v0.2s, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        addhn	v0.4h, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        addhn	v0.8b, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        addhn2	v0.16b, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        addhn2	v0.4s, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        addhn2	v0.8h, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        addp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        addp	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     0.50                        and	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     0.50                        bic	v0.4h, #15, lsl #8
+# CHECK-NEXT:  1      3     0.50                        bic	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        bif	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        bit	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        bsl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        cls	v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        cls	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        cls	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        cls	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        cls	v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        cls	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        clz	v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        clz	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        clz	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        clz	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        clz	v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        clz	v0.8h, v0.8h
+# CHECK-NEXT:  1      3     0.50                        cmeq	d20, d21, #0
+# CHECK-NEXT:  1      3     0.50                        cmeq	d20, d21, d22
+# CHECK-NEXT:  1      3     0.50                        cmeq	v0.16b, v0.16b, #0
+# CHECK-NEXT:  1      3     0.50                        cmeq	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      3     0.50                        cmge	d20, d21, #0
+# CHECK-NEXT:  1      3     0.50                        cmge	d20, d21, d22
+# CHECK-NEXT:  1      3     0.50                        cmge	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     0.50                        cmge	v0.8b, v0.8b, #0
+# CHECK-NEXT:  1      3     0.50                        cmgt	d20, d21, #0
+# CHECK-NEXT:  1      3     0.50                        cmgt	d20, d21, d22
+# CHECK-NEXT:  1      3     0.50                        cmgt	v0.2s, v0.2s, #0
+# CHECK-NEXT:  1      3     0.50                        cmgt	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      3     0.50                        cmhi	d20, d21, d22
+# CHECK-NEXT:  1      3     0.50                        cmhi	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      3     0.50                        cmhs	d20, d21, d22
+# CHECK-NEXT:  1      3     0.50                        cmhs	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     0.50                        cmle	d20, d21, #0
+# CHECK-NEXT:  1      3     0.50                        cmle	v0.2d, v0.2d, #0
+# CHECK-NEXT:  1      3     0.50                        cmlt	d20, d21, #0
+# CHECK-NEXT:  1      3     0.50                        cmlt	v0.8h, v0.8h, #0
+# CHECK-NEXT:  1      4     0.50                        cmtst	d20, d21, d22
+# CHECK-NEXT:  1      4     0.50                        cmtst	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        cnt	v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        cnt	v0.8b, v0.8b
+# CHECK-NEXT:  1      3     0.50                        dup	v0.16b, w28
+# CHECK-NEXT:  1      3     0.50                        dup	v0.2d, x28
+# CHECK-NEXT:  1      4     0.50                        dup	v0.2s, w28
+# CHECK-NEXT:  1      4     0.50                        dup	v0.4h, w28
+# CHECK-NEXT:  1      3     0.50                        dup	v0.4s, w28
+# CHECK-NEXT:  1      4     0.50                        dup	v0.8b, w28
+# CHECK-NEXT:  1      3     0.50                        dup	v0.8h, w28
+# CHECK-NEXT:  1      3     0.50                        eor	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        ext	v0.16b, v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      4     0.50                        ext	v0.8b, v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      4     0.50                        fabd	d29, d24, d20
+# CHECK-NEXT:  1      4     0.50                        fabd	s29, s24, s20
+# CHECK-NEXT:  1      4     0.50                        fabd	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fabs	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        fabs	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fabs	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        fabs	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fabs	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        facge	d20, d21, d22
+# CHECK-NEXT:  1      4     0.50                        facge	s10, s11, s12
+# CHECK-NEXT:  1      4     0.50                        facge	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        facgt	d20, d21, d22
+# CHECK-NEXT:  1      4     0.50                        facgt	s10, s11, s12
+# CHECK-NEXT:  1      4     0.50                        facgt	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        fadd	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        faddp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        faddp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fcmeq	d20, d21, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmeq	d20, d21, d22
+# CHECK-NEXT:  1      4     0.50                        fcmeq	s10, s11, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmeq	s10, s11, s12
+# CHECK-NEXT:  1      4     0.50                        fcmeq	v0.2s, v0.2s, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmeq	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fcmge	d20, d21, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmge	d20, d21, d22
+# CHECK-NEXT:  1      4     0.50                        fcmge	s10, s11, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmge	s10, s11, s12
+# CHECK-NEXT:  1      4     0.50                        fcmge	v0.2d, v0.2d, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmge	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fcmgt	d20, d21, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmgt	d20, d21, d22
+# CHECK-NEXT:  1      4     0.50                        fcmgt	s10, s11, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmgt	s10, s11, s12
+# CHECK-NEXT:  1      4     0.50                        fcmgt	v0.4s, v0.4s, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmgt	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fcmle	d20, d21, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmle	s10, s11, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmle	v0.2d, v0.2d, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmlt	d20, d21, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmlt	s10, s11, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmlt	v0.4s, v0.4s, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcvtas	d21, d14
+# CHECK-NEXT:  1      4     0.50                        fcvtas	s12, s13
+# CHECK-NEXT:  1      4     0.50                        fcvtas	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        fcvtas	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fcvtas	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        fcvtas	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fcvtas	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        fcvtau	d21, d14
+# CHECK-NEXT:  1      4     0.50                        fcvtau	s12, s13
+# CHECK-NEXT:  1      4     0.50                        fcvtau	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        fcvtau	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fcvtau	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        fcvtau	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fcvtau	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        fcvtl	v0.2d, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fcvtl	v0.4s, v0.4h
+# CHECK-NEXT:  1      4     0.50                        fcvtl2	v0.2d, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fcvtl2	v0.4s, v0.8h
+# CHECK-NEXT:  1      4     0.50                        fcvtms	d21, d14
+# CHECK-NEXT:  1      4     0.50                        fcvtms	s22, s13
+# CHECK-NEXT:  1      4     0.50                        fcvtms	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        fcvtms	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fcvtms	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        fcvtms	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fcvtms	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        fcvtmu	d21, d14
+# CHECK-NEXT:  1      4     0.50                        fcvtmu	s12, s13
+# CHECK-NEXT:  1      4     0.50                        fcvtmu	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        fcvtmu	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fcvtmu	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        fcvtmu	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fcvtmu	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        fcvtn	v0.2s, v0.2d
+# CHECK-NEXT:  1      4     0.50                        fcvtn	v0.4h, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fcvtn2	v0.4s, v0.2d
+# CHECK-NEXT:  1      4     0.50                        fcvtn2	v0.8h, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fcvtns	d21, d14
+# CHECK-NEXT:  1      4     0.50                        fcvtns	s22, s13
+# CHECK-NEXT:  1      4     0.50                        fcvtns	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        fcvtns	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fcvtns	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        fcvtns	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fcvtns	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        fcvtnu	d21, d14
+# CHECK-NEXT:  1      4     0.50                        fcvtnu	s12, s13
+# CHECK-NEXT:  1      4     0.50                        fcvtnu	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        fcvtnu	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fcvtnu	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        fcvtnu	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fcvtnu	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        fcvtps	d21, d14
+# CHECK-NEXT:  1      4     0.50                        fcvtps	s22, s13
+# CHECK-NEXT:  1      4     0.50                        fcvtps	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        fcvtps	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fcvtps	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        fcvtps	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fcvtps	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        fcvtpu	d21, d14
+# CHECK-NEXT:  1      4     0.50                        fcvtpu	s12, s13
+# CHECK-NEXT:  1      4     0.50                        fcvtpu	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        fcvtpu	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fcvtpu	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        fcvtpu	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fcvtpu	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        fcvtxn	s22, d13
+# CHECK-NEXT:  1      4     0.50                        fcvtxn	v0.2s, v0.2d
+# CHECK-NEXT:  1      4     0.50                        fcvtxn2	v0.4s, v0.2d
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	d21, d12, #1
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	d21, d14
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	s12, s13
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	s21, s12, #1
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	d21, d12, #1
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	d21, d14
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	s12, s13
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	s21, s12, #1
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	v0.8h, v0.8h
+# CHECK-NEXT:  1      13    10.00                       fdiv	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fmax	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        fmax	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fmax	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fmaxnm	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        fmaxnm	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fmaxnm	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fmaxnmp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        fmaxnmp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fmaxnmp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fmaxp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        fmaxp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fmaxp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fmin	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        fmin	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fmin	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fminnm	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        fminnm	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fminnm	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fminnmp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        fminnmp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fminnmp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fminp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        fminp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fminp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fmla	d0, d1, v0.d[1]
+# CHECK-NEXT:  1      4     0.50                        fmla	s0, s1, v0.s[3]
+# CHECK-NEXT:  1      4     0.50                        fmla	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fmls	d0, d4, v0.d[1]
+# CHECK-NEXT:  1      4     0.50                        fmls	s3, s5, v0.s[3]
+# CHECK-NEXT:  1      4     0.50                        fmls	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fmov	v0.2d, #-1.25000000
+# CHECK-NEXT:  1      4     0.50                        fmov	v0.2s, #13.00000000
+# CHECK-NEXT:  1      4     0.50                        fmov	v0.4s, #1.00000000
+# CHECK-NEXT:  1      4     0.50                        fmul	d0, d1, v0.d[1]
+# CHECK-NEXT:  1      4     0.50                        fmul	s0, s1, v0.s[3]
+# CHECK-NEXT:  1      4     0.50                        fmul	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fmulx	d0, d4, v0.d[1]
+# CHECK-NEXT:  1      4     0.50                        fmulx	d23, d11, d1
+# CHECK-NEXT:  1      4     0.50                        fmulx	s20, s22, s15
+# CHECK-NEXT:  1      4     0.50                        fmulx	s3, s5, v0.s[3]
+# CHECK-NEXT:  1      4     0.50                        fmulx	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        fmulx	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fmulx	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fneg	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        fneg	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        fneg	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        fneg	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        fneg	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        frecpe	d13, d13
+# CHECK-NEXT:  1      4     0.50                        frecpe	s19, s14
+# CHECK-NEXT:  1      4     0.50                        frecpe	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        frecpe	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        frecpe	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        frecpe	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        frecpe	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        frecps	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        frecps	d22, d30, d21
+# CHECK-NEXT:  1      4     0.50                        frecps	s21, s16, s13
+# CHECK-NEXT:  1      4     0.50                        frecpx	d16, d19
+# CHECK-NEXT:  1      4     0.50                        frecpx	s18, s10
+# CHECK-NEXT:  1      4     0.50                        frinta	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        frinta	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        frinta	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        frinta	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        frinta	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        frinti	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        frinti	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        frinti	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        frinti	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        frinti	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        frintm	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        frintm	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        frintm	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        frintm	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        frintm	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        frintn	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        frintn	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        frintn	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        frintn	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        frintn	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        frintp	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        frintp	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        frintp	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        frintp	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        frintp	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        frintx	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        frintx	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        frintx	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        frintx	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        frintx	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        frintz	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        frintz	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        frintz	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        frintz	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        frintz	v0.8h, v0.8h
+# CHECK-NEXT:  1      22    19.00                       frsqrte	d21, d12
+# CHECK-NEXT:  1      12    9.00                        frsqrte	s22, s13
+# CHECK-NEXT:  1      22    19.00                       frsqrte	v0.2d, v0.2d
+# CHECK-NEXT:  1      12    9.00                        frsqrte	v0.2s, v0.2s
+# CHECK-NEXT:  1      8     5.00                        frsqrte	v0.4h, v0.4h
+# CHECK-NEXT:  1      12    9.00                        frsqrte	v0.4s, v0.4s
+# CHECK-NEXT:  1      8     5.00                        frsqrte	v0.8h, v0.8h
+# CHECK-NEXT:  1      22    19.00                       frsqrts	d8, d22, d18
+# CHECK-NEXT:  1      12    9.00                        frsqrts	s21, s5, s12
+# CHECK-NEXT:  1      22    19.00                       frsqrts	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      22    19.00                       fsqrt	v0.2d, v0.2d
+# CHECK-NEXT:  1      12    9.00                        fsqrt	v0.2s, v0.2s
+# CHECK-NEXT:  1      8     5.00                        fsqrt	v0.4h, v0.4h
+# CHECK-NEXT:  1      12    9.00                        fsqrt	v0.4s, v0.4s
+# CHECK-NEXT:  1      8     5.00                        fsqrt	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        fsub	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      3     0.50    *                   ld1	{ v0.16b }, [x0]
+# CHECK-NEXT:  2      4     1.00    *                   ld1	{ v0.2d, v1.2d, v2.2d }, [x0], #48
+# CHECK-NEXT:  1      4     1.00    *                   ld1	{ v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+# CHECK-NEXT:  2      4     1.00    *                   ld1	{ v0.4s, v1.4s }, [sp], #32
+# CHECK-NEXT:  1      4     1.00    *                   ld1	{ v0.4s, v1.4s, v2.4s }, [sp]
+# CHECK-NEXT:  2      4     1.00    *                   ld1	{ v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
+# CHECK-NEXT:  2      4     1.00    *                   ld1	{ v0.8h }, [x15], x2
+# CHECK-NEXT:  1      3     0.50    *                   ld1	{ v0.8h, v1.8h }, [x15]
+# CHECK-NEXT:  1      3     0.50    *                   ld1	{ v0.b }[9], [x0]
+# CHECK-NEXT:  2      3     0.50    *                   ld1	{ v0.b }[9], [x0], #1
+# CHECK-NEXT:  1      3     0.50    *                   ld1r	{ v0.16b }, [x0]
+# CHECK-NEXT:  2      3     0.50    *                   ld1r	{ v0.16b }, [x0], #1
+# CHECK-NEXT:  1      3     0.50    *                   ld1r	{ v0.8h }, [x15]
+# CHECK-NEXT:  2      3     0.50    *                   ld1r	{ v0.8h }, [x15], #2
+# CHECK-NEXT:  2      6     2.00    *                   ld2	{ v0.16b, v1.16b }, [x0], x1
+# CHECK-NEXT:  1      4     1.00    *                   ld2	{ v0.8b, v1.8b }, [x0]
+# CHECK-NEXT:  1      4     1.00    *                   ld2	{ v0.h, v1.h }[7], [x15]
+# CHECK-NEXT:  2      4     1.00    *                   ld2	{ v0.h, v1.h }[7], [x15], #4
+# CHECK-NEXT:  1      4     1.00    *                   ld2r	{ v0.2d, v1.2d }, [x0]
+# CHECK-NEXT:  2      4     1.00    *                   ld2r	{ v0.2d, v1.2d }, [x0], #16
+# CHECK-NEXT:  1      4     1.00    *                   ld2r	{ v0.4s, v1.4s }, [sp]
+# CHECK-NEXT:  2      4     1.00    *                   ld2r	{ v0.4s, v1.4s }, [sp], #8
+# CHECK-NEXT:  1      5     1.50    *                   ld3	{ v0.4h, v1.4h, v2.4h }, [x15]
+# CHECK-NEXT:  2      5     1.50    *                   ld3	{ v0.8h, v1.8h, v2.8h }, [x15], x2
+# CHECK-NEXT:  1      4     1.00    *                   ld3	{ v0.s, v1.s, v2.s }[3], [sp]
+# CHECK-NEXT:  2      4     1.00    *                   ld3	{ v0.s, v1.s, v2.s }[3], [sp], x3
+# CHECK-NEXT:  1      4     1.00    *                   ld3r	{ v0.4h, v1.4h, v2.4h }, [x15]
+# CHECK-NEXT:  2      4     1.00    *                   ld3r	{ v0.4h, v1.4h, v2.4h }, [x15], #6
+# CHECK-NEXT:  1      4     1.00    *                   ld3r	{ v0.8b, v1.8b, v2.8b }, [x0]
+# CHECK-NEXT:  2      4     1.00    *                   ld3r	{ v0.8b, v1.8b, v2.8b }, [x0], #3
+# CHECK-NEXT:  1      6     2.00    *                   ld4	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+# CHECK-NEXT:  2      6     2.00    *                   ld4	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
+# CHECK-NEXT:  1      4     1.00    *                   ld4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0]
+# CHECK-NEXT:  2      4     1.00    *                   ld4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0], #32
+# CHECK-NEXT:  2      4     1.00    *                   ld4	{ v0.h, v1.h, v2.h, v3.h }[7], [x0], x0
+# CHECK-NEXT:  1      4     1.00    *                   ld4r	{ v0.1d, v1.1d, v2.1d, v3.1d }, [sp]
+# CHECK-NEXT:  2      4     1.00    *                   ld4r	{ v0.1d, v1.1d, v2.1d, v3.1d }, [sp], x7
+# CHECK-NEXT:  1      4     1.00    *                   ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+# CHECK-NEXT:  2      4     1.00    *                   ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp], x30
+# CHECK-NEXT:  1      4     0.50                        mla	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        mls	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        mov	b0, v0.b[15]
+# CHECK-NEXT:  1      4     0.50                        mov	d6, v0.d[1]
+# CHECK-NEXT:  1      4     0.50                        mov	h2, v0.h[5]
+# CHECK-NEXT:  1      4     0.50                        mov	s17, v0.s[2]
+# CHECK-NEXT:  1      3     0.50                        mov	v0.16b, v0.16b
+# CHECK-NEXT:  1      3     0.50                        mov	v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        movi	d15, #0xff00ff00ff00ff
+# CHECK-NEXT:  1      4     0.50                        movi	v0.16b, #31
+# CHECK-NEXT:  1      4     0.50                        movi	v0.2d, #0xff0000ff0000ffff
+# CHECK-NEXT:  1      4     0.50                        movi	v0.2s, #8, msl #8
+# CHECK-NEXT:  1      4     0.50                        movi	v0.4s, #255, lsl #24
+# CHECK-NEXT:  1      4     0.50                        movi	v0.8b, #255
+# CHECK-NEXT:  1      4     0.50                        mul	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     0.50                        mvni	v0.2s, #0
+# CHECK-NEXT:  1      3     0.50                        mvni	v0.4s, #16, msl #16
+# CHECK-NEXT:  1      3     0.50                        neg	d29, d24
+# CHECK-NEXT:  1      4     0.50                        neg	v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        neg	v0.2d, v0.2d
+# CHECK-NEXT:  1      3     0.50                        neg	v0.2s, v0.2s
+# CHECK-NEXT:  1      3     0.50                        neg	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        neg	v0.4s, v0.4s
+# CHECK-NEXT:  1      3     0.50                        neg	v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        neg	v0.8h, v0.8h
+# CHECK-NEXT:  1      3     0.50                        mvn	v0.16b, v0.16b
+# CHECK-NEXT:  1      3     0.50                        mvn	v0.8b, v0.8b
+# CHECK-NEXT:  1      3     0.50                        orn	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      3     0.50                        mov	v0.16b, v0.16b
+# CHECK-NEXT:  1      3     0.50                        orr	v0.8h, #31
+# CHECK-NEXT:  1      4     0.50                        pmul	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        pmul	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        pmull	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        pmull2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        raddhn	v0.2s, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        raddhn	v0.4h, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        raddhn	v0.8b, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        raddhn2	v0.16b, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        raddhn2	v0.4s, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        raddhn2	v0.8h, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        rbit	v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        rbit	v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        rev16	v21.8b, v1.8b
+# CHECK-NEXT:  1      4     0.50                        rev16	v30.16b, v31.16b
+# CHECK-NEXT:  1      4     0.50                        rev32	v0.4h, v9.4h
+# CHECK-NEXT:  1      4     0.50                        rev32	v21.8b, v1.8b
+# CHECK-NEXT:  1      4     0.50                        rev32	v30.16b, v31.16b
+# CHECK-NEXT:  1      4     0.50                        rev32	v4.8h, v7.8h
+# CHECK-NEXT:  1      4     0.50                        rev64	v0.16b, v31.16b
+# CHECK-NEXT:  1      4     0.50                        rev64	v1.8b, v9.8b
+# CHECK-NEXT:  1      4     0.50                        rev64	v13.4h, v21.4h
+# CHECK-NEXT:  1      4     0.50                        rev64	v2.8h, v4.8h
+# CHECK-NEXT:  1      4     0.50                        rev64	v4.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        rev64	v6.4s, v8.4s
+# CHECK-NEXT:  1      4     0.50                        rshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  1      4     0.50                        rshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  1      4     0.50                        rshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  1      4     0.50                        rshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  1      4     0.50                        rshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  1      4     0.50                        rshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  1      4     0.50                        rsubhn	v0.2s, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        rsubhn	v0.4h, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        rsubhn	v0.8b, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        rsubhn2	v0.16b, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        rsubhn2	v0.4s, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        rsubhn2	v0.8h, v0.4s, v0.4s
+# CHECK-NEXT:  1      8     0.50                        saba	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      8     0.50                        sabal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      8     0.50                        sabal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      8     0.50                        sabal	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      8     0.50                        sabal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      8     0.50                        sabal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      8     0.50                        sabal2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      3     0.50                        sabd	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     0.50                        sabdl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      3     0.50                        sabdl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     0.50                        sabdl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     0.50                        sabdl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      3     0.50                        sabdl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      3     0.50                        sabdl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      8     1.00                        sadalp	v0.1d, v0.2s
+# CHECK-NEXT:  1      8     1.00                        sadalp	v0.2d, v0.4s
+# CHECK-NEXT:  1      8     1.00                        sadalp	v0.2s, v0.4h
+# CHECK-NEXT:  1      8     1.00                        sadalp	v0.4h, v0.8b
+# CHECK-NEXT:  1      8     1.00                        sadalp	v0.4s, v0.8h
+# CHECK-NEXT:  1      8     1.00                        sadalp	v0.8h, v0.16b
+# CHECK-NEXT:  1      4     0.50                        saddl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        saddl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        saddl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        saddl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        saddl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        saddl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        saddlp	v0.1d, v0.2s
+# CHECK-NEXT:  1      4     0.50                        saddlp	v0.2d, v0.4s
+# CHECK-NEXT:  1      4     0.50                        saddlp	v0.2s, v0.4h
+# CHECK-NEXT:  1      4     0.50                        saddlp	v0.4h, v0.8b
+# CHECK-NEXT:  1      4     0.50                        saddlp	v0.4s, v0.8h
+# CHECK-NEXT:  1      4     0.50                        saddlp	v0.8h, v0.16b
+# CHECK-NEXT:  1      4     0.50                        saddw	v0.2d, v0.2d, v0.2s
+# CHECK-NEXT:  1      4     0.50                        saddw	v0.4s, v0.4s, v0.4h
+# CHECK-NEXT:  1      4     0.50                        saddw	v0.8h, v0.8h, v0.8b
+# CHECK-NEXT:  1      4     0.50                        saddw2	v0.2d, v0.2d, v0.4s
+# CHECK-NEXT:  1      4     0.50                        saddw2	v0.4s, v0.4s, v0.8h
+# CHECK-NEXT:  1      4     0.50                        saddw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  1      4     0.50                        scvtf	d21, d12
+# CHECK-NEXT:  1      4     0.50                        scvtf	d21, d12, #64
+# CHECK-NEXT:  1      4     0.50                        scvtf	s22, s13
+# CHECK-NEXT:  1      4     0.50                        scvtf	s22, s13, #32
+# CHECK-NEXT:  1      4     0.50                        scvtf	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        scvtf	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      4     0.50                        scvtf	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        scvtf	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      4     0.50                        scvtf	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        scvtf	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        scvtf	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      4     0.50                        scvtf	v0.8h, v0.8h
+# CHECK-NEXT:  1      3     0.50                        shadd	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     0.50                        shl	d7, d10, #12
+# CHECK-NEXT:  1      3     0.50                        shl	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      3     0.50                        shl	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      3     0.50                        shl	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      3     0.50                        shl	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      3     0.50                        shll	v0.2d, v0.2s, #32
+# CHECK-NEXT:  1      3     0.50                        shll	v0.4s, v0.4h, #16
+# CHECK-NEXT:  1      3     0.50                        shll	v0.8h, v0.8b, #8
+# CHECK-NEXT:  1      3     0.50                        shll	v0.2d, v0.2s, #32
+# CHECK-NEXT:  1      3     0.50                        shll	v0.4s, v0.4h, #16
+# CHECK-NEXT:  1      3     0.50                        shll	v0.8h, v0.8b, #8
+# CHECK-NEXT:  1      3     0.50                        shll2	v0.2d, v0.4s, #32
+# CHECK-NEXT:  1      3     0.50                        shll2	v0.4s, v0.8h, #16
+# CHECK-NEXT:  1      3     0.50                        shll2	v0.8h, v0.16b, #8
+# CHECK-NEXT:  1      3     0.50                        shll2	v0.2d, v0.4s, #32
+# CHECK-NEXT:  1      3     0.50                        shll2	v0.4s, v0.8h, #16
+# CHECK-NEXT:  1      3     0.50                        shll2	v0.8h, v0.16b, #8
+# CHECK-NEXT:  1      3     0.50                        shrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  1      3     0.50                        shrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  1      3     0.50                        shrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  1      3     0.50                        shrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  1      3     0.50                        shrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  1      3     0.50                        shrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  1      3     0.50                        shsub	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      3     0.50                        shsub	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     0.50                        sli	d10, d14, #12
+# CHECK-NEXT:  1      4     0.50                        sli	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      4     0.50                        sli	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      4     0.50                        sli	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      4     0.50                        sli	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      4     0.50                        sli	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      4     0.50                        sli	v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      4     0.50                        sli	v0.8h, v0.8h, #3
+# CHECK-NEXT:  1      4     0.50                        smax	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        smax	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        smax	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        smaxp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        smaxp	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        smaxp	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        smin	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        smin	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        smin	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        sminp	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        sminp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        sminp	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        smlal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        smlal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        smlal	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        smlal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        smlal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        smlal2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        smlsl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        smlsl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        smlsl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        smlsl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        smlsl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        smlsl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        smull	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        smull	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        smull	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        smull2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        smull2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        smull2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        sqabs	b19, b14
+# CHECK-NEXT:  1      4     0.50                        sqabs	d18, d12
+# CHECK-NEXT:  1      4     0.50                        sqabs	h21, h15
+# CHECK-NEXT:  1      4     0.50                        sqabs	s20, s12
+# CHECK-NEXT:  1      4     0.50                        sqabs	v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        sqabs	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        sqabs	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        sqabs	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        sqabs	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        sqabs	v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        sqabs	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        sqadd	b20, b11, b15
+# CHECK-NEXT:  1      4     0.50                        sqadd	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        sqadd	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        sqdmlal	d19, s24, s12
+# CHECK-NEXT:  1      4     0.50                        sqdmlal	d8, s9, v0.s[1]
+# CHECK-NEXT:  1      4     0.50                        sqdmlal	s0, h0, v0.h[3]
+# CHECK-NEXT:  1      4     0.50                        sqdmlal	s17, h27, h12
+# CHECK-NEXT:  1      4     0.50                        sqdmlal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        sqdmlal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        sqdmlal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        sqdmlal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        sqdmlsl	d12, s23, s13
+# CHECK-NEXT:  1      4     0.50                        sqdmlsl	d8, s9, v0.s[1]
+# CHECK-NEXT:  1      4     0.50                        sqdmlsl	s0, h0, v0.h[3]
+# CHECK-NEXT:  1      4     0.50                        sqdmlsl	s14, h12, h25
+# CHECK-NEXT:  1      4     0.50                        sqdmlsl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        sqdmlsl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        sqdmlsl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        sqdmlsl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        sqdmulh	h10, h11, h12
+# CHECK-NEXT:  1      4     0.50                        sqdmulh	h7, h15, v0.h[3]
+# CHECK-NEXT:  1      4     0.50                        sqdmulh	s15, s14, v0.s[1]
+# CHECK-NEXT:  1      4     0.50                        sqdmulh	s20, s21, s2
+# CHECK-NEXT:  1      4     0.50                        sqdmulh	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        sqdmulh	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        sqdmull	d1, s1, v0.s[1]
+# CHECK-NEXT:  1      4     0.50                        sqdmull	d15, s22, s12
+# CHECK-NEXT:  1      4     0.50                        sqdmull	s1, h1, v0.h[3]
+# CHECK-NEXT:  1      4     0.50                        sqdmull	s12, h22, h12
+# CHECK-NEXT:  1      4     0.50                        sqdmull	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        sqdmull	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        sqdmull2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        sqdmull2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        sqneg	b19, b14
+# CHECK-NEXT:  1      4     0.50                        sqneg	d18, d12
+# CHECK-NEXT:  1      4     0.50                        sqneg	h21, h15
+# CHECK-NEXT:  1      4     0.50                        sqneg	s20, s12
+# CHECK-NEXT:  1      4     0.50                        sqneg	v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        sqneg	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        sqneg	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        sqneg	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        sqneg	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        sqneg	v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        sqneg	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        sqrdmulh	h10, h11, h12
+# CHECK-NEXT:  1      4     0.50                        sqrdmulh	h7, h15, v0.h[3]
+# CHECK-NEXT:  1      4     0.50                        sqrdmulh	s15, s14, v0.s[1]
+# CHECK-NEXT:  1      4     0.50                        sqrdmulh	s20, s21, s2
+# CHECK-NEXT:  1      4     0.50                        sqrdmulh	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        sqrdmulh	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        sqrshl	d31, d31, d31
+# CHECK-NEXT:  1      4     0.50                        sqrshl	h3, h4, h15
+# CHECK-NEXT:  1      4     0.50                        sqrshl	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        sqrshl	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        sqrshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        sqrshrn	b10, h13, #2
+# CHECK-NEXT:  1      4     0.50                        sqrshrn	h15, s10, #6
+# CHECK-NEXT:  1      4     0.50                        sqrshrn	s15, d12, #9
+# CHECK-NEXT:  1      4     0.50                        sqrshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  1      4     0.50                        sqrshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  1      4     0.50                        sqrshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  1      4     0.50                        sqrshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  1      4     0.50                        sqrshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  1      4     0.50                        sqrshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  1      4     0.50                        sqrshrun	b17, h10, #6
+# CHECK-NEXT:  1      4     0.50                        sqrshrun	h10, s13, #15
+# CHECK-NEXT:  1      4     0.50                        sqrshrun	s22, d16, #31
+# CHECK-NEXT:  1      4     0.50                        sqrshrun	v0.2s, v0.2d, #3
+# CHECK-NEXT:  1      4     0.50                        sqrshrun	v0.4h, v0.4s, #3
+# CHECK-NEXT:  1      4     0.50                        sqrshrun	v0.8b, v0.8h, #3
+# CHECK-NEXT:  1      4     0.50                        sqrshrun2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  1      4     0.50                        sqrshrun2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  1      4     0.50                        sqrshrun2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  1      4     0.50                        sqshl	b11, b19, #7
+# CHECK-NEXT:  1      4     0.50                        sqshl	d15, d16, #51
+# CHECK-NEXT:  1      4     0.50                        sqshl	d31, d31, d31
+# CHECK-NEXT:  1      4     0.50                        sqshl	h13, h18, #11
+# CHECK-NEXT:  1      4     0.50                        sqshl	h3, h4, h15
+# CHECK-NEXT:  1      4     0.50                        sqshl	s14, s17, #22
+# CHECK-NEXT:  1      4     0.50                        sqshl	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      4     0.50                        sqshl	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      4     0.50                        sqshl	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      4     0.50                        sqshl	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        sqshl	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      4     0.50                        sqshl	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        sqshl	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      4     0.50                        sqshl	v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      4     0.50                        sqshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        sqshl	v0.8h, v0.8h, #3
+# CHECK-NEXT:  1      4     0.50                        sqshlu	b15, b18, #6
+# CHECK-NEXT:  1      4     0.50                        sqshlu	d11, d13, #32
+# CHECK-NEXT:  1      4     0.50                        sqshlu	h19, h17, #6
+# CHECK-NEXT:  1      4     0.50                        sqshlu	s16, s14, #25
+# CHECK-NEXT:  1      4     0.50                        sqshlu	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      4     0.50                        sqshlu	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      4     0.50                        sqshlu	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      4     0.50                        sqshlu	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      4     0.50                        sqshlu	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      4     0.50                        sqshlu	v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      4     0.50                        sqshlu	v0.8h, v0.8h, #3
+# CHECK-NEXT:  1      4     0.50                        sqshrn	b10, h15, #5
+# CHECK-NEXT:  1      4     0.50                        sqshrn	h17, s10, #4
+# CHECK-NEXT:  1      4     0.50                        sqshrn	s18, d10, #31
+# CHECK-NEXT:  1      4     0.50                        sqshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  1      4     0.50                        sqshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  1      4     0.50                        sqshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  1      4     0.50                        sqshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  1      4     0.50                        sqshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  1      4     0.50                        sqshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  1      4     0.50                        sqshrun	b15, h10, #7
+# CHECK-NEXT:  1      4     0.50                        sqshrun	h20, s14, #3
+# CHECK-NEXT:  1      4     0.50                        sqshrun	s10, d15, #15
+# CHECK-NEXT:  1      4     0.50                        sqshrun	v0.2s, v0.2d, #3
+# CHECK-NEXT:  1      4     0.50                        sqshrun	v0.4h, v0.4s, #3
+# CHECK-NEXT:  1      4     0.50                        sqshrun	v0.8b, v0.8h, #3
+# CHECK-NEXT:  1      4     0.50                        sqshrun2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  1      4     0.50                        sqshrun2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  1      4     0.50                        sqshrun2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  1      4     0.50                        sqsub	s20, s10, s7
+# CHECK-NEXT:  1      4     0.50                        sqsub	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        sqsub	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        sqsub	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        sqxtn	b18, h18
+# CHECK-NEXT:  1      4     0.50                        sqxtn	h20, s17
+# CHECK-NEXT:  1      4     0.50                        sqxtn	s19, d14
+# CHECK-NEXT:  1      4     0.50                        sqxtn	v0.2s, v0.2d
+# CHECK-NEXT:  1      4     0.50                        sqxtn	v0.4h, v0.4s
+# CHECK-NEXT:  1      4     0.50                        sqxtn	v0.8b, v0.8h
+# CHECK-NEXT:  1      4     0.50                        sqxtn2	v0.16b, v0.8h
+# CHECK-NEXT:  1      4     0.50                        sqxtn2	v0.4s, v0.2d
+# CHECK-NEXT:  1      4     0.50                        sqxtn2	v0.8h, v0.4s
+# CHECK-NEXT:  1      4     0.50                        sqxtun	b19, h14
+# CHECK-NEXT:  1      4     0.50                        sqxtun	h21, s15
+# CHECK-NEXT:  1      4     0.50                        sqxtun	s20, d12
+# CHECK-NEXT:  1      4     0.50                        sqxtun	v0.2s, v0.2d
+# CHECK-NEXT:  1      4     0.50                        sqxtun	v0.4h, v0.4s
+# CHECK-NEXT:  1      4     0.50                        sqxtun	v0.8b, v0.8h
+# CHECK-NEXT:  1      4     0.50                        sqxtun2	v0.16b, v0.8h
+# CHECK-NEXT:  1      4     0.50                        sqxtun2	v0.4s, v0.2d
+# CHECK-NEXT:  1      4     0.50                        sqxtun2	v0.8h, v0.4s
+# CHECK-NEXT:  1      3     0.50                        srhadd	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      3     0.50                        srhadd	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     0.50                        srhadd	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     0.50                        sri	d10, d12, #14
+# CHECK-NEXT:  1      4     0.50                        sri	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      4     0.50                        sri	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      4     0.50                        sri	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      4     0.50                        sri	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      4     0.50                        sri	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      4     0.50                        sri	v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      4     0.50                        sri	v0.8h, v0.8h, #3
+# CHECK-NEXT:  1      3     0.50                        srshl	d16, d16, d16
+# CHECK-NEXT:  1      3     0.50                        srshl	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      3     0.50                        srshl	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     0.50                        srshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     0.50                        srshr	d19, d18, #7
+# CHECK-NEXT:  1      3     0.50                        srshr	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      3     0.50                        srshr	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      3     0.50                        srshr	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      3     0.50                        srshr	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      3     0.50                        srshr	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      3     0.50                        srshr	v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      3     0.50                        srshr	v0.8h, v0.8h, #3
+# CHECK-NEXT:  1      8     1.00                        srsra	d15, d11, #19
+# CHECK-NEXT:  1      8     1.00                        srsra	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      8     1.00                        srsra	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      8     1.00                        srsra	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      8     1.00                        srsra	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      8     1.00                        srsra	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      8     1.00                        srsra	v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      8     1.00                        srsra	v0.8h, v0.8h, #3
+# CHECK-NEXT:  1      3     0.50                        sshl	d31, d31, d31
+# CHECK-NEXT:  1      3     0.50                        sshl	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      3     0.50                        sshl	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      3     0.50                        sshl	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     0.50                        sshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     0.50                        sshll	v0.2d, v0.2s, #3
+# CHECK-NEXT:  1      3     0.50                        sshll2	v0.4s, v0.8h, #3
+# CHECK-NEXT:  1      3     0.50                        sshr	d15, d16, #12
+# CHECK-NEXT:  1      3     0.50                        sshr	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      3     0.50                        sshr	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      3     0.50                        sshr	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      3     0.50                        sshr	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      3     0.50                        sshr	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      3     0.50                        sshr	v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      3     0.50                        sshr	v0.8h, v0.8h, #3
+# CHECK-NEXT:  1      8     1.00                        ssra	d18, d12, #21
+# CHECK-NEXT:  1      8     1.00                        ssra	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      8     1.00                        ssra	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      8     1.00                        ssra	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      8     1.00                        ssra	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      8     1.00                        ssra	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      8     1.00                        ssra	v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      8     1.00                        ssra	v0.8h, v0.8h, #3
+# CHECK-NEXT:  1      4     0.50                        ssubl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        ssubl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        ssubl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        ssubl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        ssubl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        ssubl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        ssubw	v0.2d, v0.2d, v0.2s
+# CHECK-NEXT:  1      4     0.50                        ssubw	v0.4s, v0.4s, v0.4h
+# CHECK-NEXT:  1      4     0.50                        ssubw	v0.8h, v0.8h, v0.8b
+# CHECK-NEXT:  1      4     0.50                        ssubw2	v0.2d, v0.2d, v0.4s
+# CHECK-NEXT:  1      4     0.50                        ssubw2	v0.4s, v0.4s, v0.8h
+# CHECK-NEXT:  1      4     0.50                        ssubw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  1      4     1.00           *            st1	{ v0.16b }, [x0]
+# CHECK-NEXT:  2      5     2.00           *            st1	{ v0.2d, v1.2d, v2.2d }, [x0], #48
+# CHECK-NEXT:  1      5     4.00           *            st1	{ v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+# CHECK-NEXT:  2      4     1.00           *            st1	{ v0.4s, v1.4s }, [sp], #32
+# CHECK-NEXT:  1      5     2.00           *            st1	{ v0.4s, v1.4s, v2.4s }, [sp]
+# CHECK-NEXT:  2      5     4.00           *            st1	{ v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
+# CHECK-NEXT:  2      4     1.00           *            st1	{ v0.8h }, [x15], x2
+# CHECK-NEXT:  1      4     1.00           *            st1	{ v0.8h, v1.8h }, [x15]
+# CHECK-NEXT:  1      4     1.00           *            st1	{ v0.d }[1], [x0]
+# CHECK-NEXT:  2      4     1.00           *            st1	{ v0.d }[1], [x0], #8
+# CHECK-NEXT:  2      5     4.00           *            st2	{ v0.16b, v1.16b }, [x0], x1
+# CHECK-NEXT:  1      5     2.00           *            st2	{ v0.8b, v1.8b }, [x0]
+# CHECK-NEXT:  1      5     2.00           *            st2	{ v0.s, v1.s }[3], [sp]
+# CHECK-NEXT:  2      5     2.00           *            st2	{ v0.s, v1.s }[3], [sp], #8
+# CHECK-NEXT:  1      5     4.00           *            st3	{ v0.4h, v1.4h, v2.4h }, [x15]
+# CHECK-NEXT:  2      5     4.00           *            st3	{ v0.8h, v1.8h, v2.8h }, [x15], x2
+# CHECK-NEXT:  1      5     2.00           *            st3	{ v0.h, v1.h, v2.h }[7], [x15]
+# CHECK-NEXT:  2      5     2.00           *            st3	{ v0.h, v1.h, v2.h }[7], [x15], #6
+# CHECK-NEXT:  1      5     4.00           *            st4	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+# CHECK-NEXT:  2      5     4.00           *            st4	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
+# CHECK-NEXT:  1      5     2.00           *            st4	{ v0.b, v1.b, v2.b, v3.b }[9], [x0]
+# CHECK-NEXT:  2      5     2.00           *            st4	{ v0.b, v1.b, v2.b, v3.b }[9], [x0], x5
+# CHECK-NEXT:  1      3     0.50                        sub	d15, d5, d16
+# CHECK-NEXT:  1      4     0.50                        sub	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        suqadd	b19, b14
+# CHECK-NEXT:  1      4     0.50                        suqadd	d18, d22
+# CHECK-NEXT:  1      4     0.50                        suqadd	h20, h15
+# CHECK-NEXT:  1      4     0.50                        suqadd	s21, s12
+# CHECK-NEXT:  1      4     0.50                        suqadd	v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        suqadd	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        suqadd	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        suqadd	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        suqadd	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        suqadd	v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        suqadd	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        tbl	v0.16b, { v0.16b }, v0.16b
+# CHECK-NEXT:  1      4     0.50                        tbl	v0.16b, { v0.16b, v1.16b }, v0.16b
+# CHECK-NEXT:  1      4     0.50                        tbl	v0.16b, { v0.16b, v1.16b, v2.16b }, v0.16b
+# CHECK-NEXT:  1      4     0.50                        tbl	v0.16b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.16b
+# CHECK-NEXT:  1      4     0.50                        tbl	v0.8b, { v0.16b }, v0.8b
+# CHECK-NEXT:  1      4     0.50                        tbl	v0.8b, { v0.16b, v1.16b }, v0.8b
+# CHECK-NEXT:  1      4     0.50                        tbl	v0.8b, { v0.16b, v1.16b, v2.16b }, v0.8b
+# CHECK-NEXT:  1      4     0.50                        tbl	v0.8b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.8b
+# CHECK-NEXT:  1      4     0.50                        tbx	v0.16b, { v0.16b }, v0.16b
+# CHECK-NEXT:  1      4     0.50                        tbx	v0.16b, { v0.16b, v1.16b }, v0.16b
+# CHECK-NEXT:  1      4     0.50                        tbx	v0.16b, { v0.16b, v1.16b, v2.16b }, v0.16b
+# CHECK-NEXT:  1      4     0.50                        tbx	v0.16b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.16b
+# CHECK-NEXT:  1      4     0.50                        tbx	v0.8b, { v0.16b }, v0.8b
+# CHECK-NEXT:  1      4     0.50                        tbx	v0.8b, { v0.16b, v1.16b }, v0.8b
+# CHECK-NEXT:  1      4     0.50                        tbx	v0.8b, { v0.16b, v1.16b, v2.16b }, v0.8b
+# CHECK-NEXT:  1      4     0.50                        tbx	v0.8b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.8b
+# CHECK-NEXT:  1      4     0.50                        trn1	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        trn1	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        trn1	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        trn1	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        trn1	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        trn1	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        trn1	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        trn2	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        trn2	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        trn2	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        trn2	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        trn2	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        trn2	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        trn2	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      8     0.50                        uaba	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      8     0.50                        uabal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      8     0.50                        uabal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      8     0.50                        uabal	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      8     0.50                        uabal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      8     0.50                        uabal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      8     0.50                        uabal2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      3     0.50                        uabd	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     0.50                        uabdl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      3     0.50                        uabdl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      3     0.50                        uabdl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      3     0.50                        uabdl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      3     0.50                        uabdl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      3     0.50                        uabdl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      8     1.00                        uadalp	v0.1d, v0.2s
+# CHECK-NEXT:  1      8     1.00                        uadalp	v0.2d, v0.4s
+# CHECK-NEXT:  1      8     1.00                        uadalp	v0.2s, v0.4h
+# CHECK-NEXT:  1      8     1.00                        uadalp	v0.4h, v0.8b
+# CHECK-NEXT:  1      8     1.00                        uadalp	v0.4s, v0.8h
+# CHECK-NEXT:  1      8     1.00                        uadalp	v0.8h, v0.16b
+# CHECK-NEXT:  1      4     0.50                        uaddl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        uaddl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        uaddl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        uaddl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        uaddl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        uaddl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        uaddlp	v0.1d, v0.2s
+# CHECK-NEXT:  1      4     0.50                        uaddlp	v0.2d, v0.4s
+# CHECK-NEXT:  1      4     0.50                        uaddlp	v0.2s, v0.4h
+# CHECK-NEXT:  1      4     0.50                        uaddlp	v0.4h, v0.8b
+# CHECK-NEXT:  1      4     0.50                        uaddlp	v0.4s, v0.8h
+# CHECK-NEXT:  1      4     0.50                        uaddlp	v0.8h, v0.16b
+# CHECK-NEXT:  1      4     0.50                        uaddw	v0.2d, v0.2d, v0.2s
+# CHECK-NEXT:  1      4     0.50                        uaddw	v0.4s, v0.4s, v0.4h
+# CHECK-NEXT:  1      4     0.50                        uaddw	v0.8h, v0.8h, v0.8b
+# CHECK-NEXT:  1      4     0.50                        uaddw2	v0.2d, v0.2d, v0.4s
+# CHECK-NEXT:  1      4     0.50                        uaddw2	v0.4s, v0.4s, v0.8h
+# CHECK-NEXT:  1      4     0.50                        uaddw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  1      4     0.50                        ucvtf	d21, d14
+# CHECK-NEXT:  1      4     0.50                        ucvtf	d21, d14, #64
+# CHECK-NEXT:  1      4     0.50                        ucvtf	s22, s13
+# CHECK-NEXT:  1      4     0.50                        ucvtf	s22, s13, #32
+# CHECK-NEXT:  1      4     0.50                        ucvtf	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        ucvtf	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      4     0.50                        ucvtf	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        ucvtf	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      4     0.50                        ucvtf	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        ucvtf	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        ucvtf	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      4     0.50                        ucvtf	v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        uhadd	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        uhadd	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        uhsub	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        umax	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        umax	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        umax	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        umaxp	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        umaxp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        umaxp	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        umin	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        umin	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        umin	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        uminp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        uminp	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        uminp	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        umlal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        umlal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        umlal	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        umlal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        umlal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        umlal2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        umlsl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        umlsl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        umlsl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        umlsl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        umlsl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        umlsl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        umull	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        umull	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        umull	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        umull2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        umull2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        umull2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        uqadd	h0, h1, h5
+# CHECK-NEXT:  1      4     0.50                        uqadd	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        uqrshl	b11, b20, b30
+# CHECK-NEXT:  1      4     0.50                        uqrshl	s23, s20, s16
+# CHECK-NEXT:  1      4     0.50                        uqrshl	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        uqrshl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        uqrshl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        uqrshl	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        uqrshrn	b10, h12, #5
+# CHECK-NEXT:  1      4     0.50                        uqrshrn	h12, s10, #14
+# CHECK-NEXT:  1      4     0.50                        uqrshrn	s10, d10, #25
+# CHECK-NEXT:  1      4     0.50                        uqrshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  1      4     0.50                        uqrshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  1      4     0.50                        uqrshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  1      4     0.50                        uqrshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  1      4     0.50                        uqrshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  1      4     0.50                        uqrshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  1      4     0.50                        uqshl	b11, b20, b30
+# CHECK-NEXT:  1      4     0.50                        uqshl	b18, b15, #6
+# CHECK-NEXT:  1      4     0.50                        uqshl	d15, d12, #19
+# CHECK-NEXT:  1      4     0.50                        uqshl	h11, h18, #7
+# CHECK-NEXT:  1      4     0.50                        uqshl	s14, s19, #18
+# CHECK-NEXT:  1      4     0.50                        uqshl	s23, s20, s16
+# CHECK-NEXT:  1      4     0.50                        uqshl	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      4     0.50                        uqshl	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        uqshl	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      4     0.50                        uqshl	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        uqshl	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      4     0.50                        uqshl	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      4     0.50                        uqshl	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      4     0.50                        uqshl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        uqshl	v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      4     0.50                        uqshl	v0.8h, v0.8h, #3
+# CHECK-NEXT:  1      4     0.50                        uqshl	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        uqshrn	b12, h10, #7
+# CHECK-NEXT:  1      4     0.50                        uqshrn	h10, s14, #5
+# CHECK-NEXT:  1      4     0.50                        uqshrn	s10, d12, #13
+# CHECK-NEXT:  1      4     0.50                        uqshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  1      4     0.50                        uqshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  1      4     0.50                        uqshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  1      4     0.50                        uqshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  1      4     0.50                        uqshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  1      4     0.50                        uqshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  1      4     0.50                        uqsub	d16, d16, d16
+# CHECK-NEXT:  1      4     0.50                        uqsub	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        uqxtn	b18, h18
+# CHECK-NEXT:  1      4     0.50                        uqxtn	h20, s17
+# CHECK-NEXT:  1      4     0.50                        uqxtn	s19, d14
+# CHECK-NEXT:  1      4     0.50                        uqxtn	v0.2s, v0.2d
+# CHECK-NEXT:  1      4     0.50                        uqxtn	v0.4h, v0.4s
+# CHECK-NEXT:  1      4     0.50                        uqxtn	v0.8b, v0.8h
+# CHECK-NEXT:  1      4     0.50                        uqxtn2	v0.16b, v0.8h
+# CHECK-NEXT:  1      4     0.50                        uqxtn2	v0.4s, v0.2d
+# CHECK-NEXT:  1      4     0.50                        uqxtn2	v0.8h, v0.4s
+# CHECK-NEXT:  1      4     0.50                        urecpe	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        urecpe	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        urhadd	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        urhadd	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        urhadd	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      3     0.50                        urshl	d8, d7, d4
+# CHECK-NEXT:  1      3     0.50                        urshl	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      3     0.50                        urshl	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      3     0.50                        urshl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      3     0.50                        urshl	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      3     0.50                        urshr	d20, d23, #31
+# CHECK-NEXT:  1      3     0.50                        urshr	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      3     0.50                        urshr	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      3     0.50                        urshr	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      3     0.50                        urshr	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      3     0.50                        urshr	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      3     0.50                        urshr	v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      3     0.50                        urshr	v0.8h, v0.8h, #3
+# CHECK-NEXT:  1      12    9.00                        ursqrte	v0.2s, v0.2s
+# CHECK-NEXT:  1      12    9.00                        ursqrte	v0.4s, v0.4s
+# CHECK-NEXT:  1      8     1.00                        ursra	d18, d10, #13
+# CHECK-NEXT:  1      8     1.00                        ursra	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      8     1.00                        ursra	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      8     1.00                        ursra	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      8     1.00                        ursra	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      8     1.00                        ursra	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      8     1.00                        ursra	v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      8     1.00                        ursra	v0.8h, v0.8h, #3
+# CHECK-NEXT:  1      3     0.50                        ushl	d0, d0, d0
+# CHECK-NEXT:  1      3     0.50                        ushl	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      3     0.50                        ushl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      3     0.50                        ushl	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      3     0.50                        ushll	v0.4s, v0.4h, #3
+# CHECK-NEXT:  1      3     0.50                        ushll2	v0.8h, v0.16b, #3
+# CHECK-NEXT:  1      3     0.50                        ushr	d10, d17, #18
+# CHECK-NEXT:  1      3     0.50                        ushr	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      3     0.50                        ushr	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      3     0.50                        ushr	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      3     0.50                        ushr	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      3     0.50                        ushr	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      3     0.50                        ushr	v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      3     0.50                        ushr	v0.8h, v0.8h, #3
+# CHECK-NEXT:  1      4     0.50                        usqadd	b19, b14
+# CHECK-NEXT:  1      4     0.50                        usqadd	d18, d22
+# CHECK-NEXT:  1      4     0.50                        usqadd	h20, h15
+# CHECK-NEXT:  1      4     0.50                        usqadd	s21, s12
+# CHECK-NEXT:  1      4     0.50                        usqadd	v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        usqadd	v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        usqadd	v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        usqadd	v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        usqadd	v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        usqadd	v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        usqadd	v0.8h, v0.8h
+# CHECK-NEXT:  1      8     1.00                        usra	d20, d13, #61
+# CHECK-NEXT:  1      8     1.00                        usra	v0.16b, v0.16b, #3
+# CHECK-NEXT:  1      8     1.00                        usra	v0.2d, v0.2d, #3
+# CHECK-NEXT:  1      8     1.00                        usra	v0.2s, v0.2s, #3
+# CHECK-NEXT:  1      8     1.00                        usra	v0.4h, v0.4h, #3
+# CHECK-NEXT:  1      8     1.00                        usra	v0.4s, v0.4s, #3
+# CHECK-NEXT:  1      8     1.00                        usra	v0.8b, v0.8b, #3
+# CHECK-NEXT:  1      8     1.00                        usra	v0.8h, v0.8h, #3
+# CHECK-NEXT:  1      4     0.50                        usubl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        usubl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        usubl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        usubl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        usubl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        usubl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        usubw	v0.2d, v0.2d, v0.2s
+# CHECK-NEXT:  1      4     0.50                        usubw	v0.4s, v0.4s, v0.4h
+# CHECK-NEXT:  1      4     0.50                        usubw	v0.8h, v0.8h, v0.8b
+# CHECK-NEXT:  1      4     0.50                        usubw2	v0.2d, v0.2d, v0.4s
+# CHECK-NEXT:  1      4     0.50                        usubw2	v0.4s, v0.4s, v0.8h
+# CHECK-NEXT:  1      4     0.50                        usubw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  1      4     0.50                        uzp1	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        uzp1	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        uzp1	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        uzp1	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        uzp1	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        uzp1	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        uzp1	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        uzp2	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        uzp2	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        uzp2	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        uzp2	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        uzp2	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        uzp2	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        uzp2	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        xtn	v0.2s, v0.2d
+# CHECK-NEXT:  1      4     0.50                        xtn	v0.4h, v0.4s
+# CHECK-NEXT:  1      4     0.50                        xtn	v0.8b, v0.8h
+# CHECK-NEXT:  1      4     0.50                        xtn2	v0.16b, v0.8h
+# CHECK-NEXT:  1      4     0.50                        xtn2	v0.4s, v0.2d
+# CHECK-NEXT:  1      4     0.50                        xtn2	v0.8h, v0.4s
+# CHECK-NEXT:  1      4     0.50                        zip1	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        zip1	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        zip1	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        zip1	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        zip1	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        zip1	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        zip1	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  1      4     0.50                        zip2	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  1      4     0.50                        zip2	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  1      4     0.50                        zip2	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  1      4     0.50                        zip2	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  1      4     0.50                        zip2	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  1      4     0.50                        zip2	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  1      4     0.50                        zip2	v0.8h, v0.8h, v0.8h
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - CortexA510UnitALU0
+# CHECK-NEXT: [1.0] - CortexA510UnitALU12
+# CHECK-NEXT: [1.1] - CortexA510UnitALU12
+# CHECK-NEXT: [2]   - CortexA510UnitB
+# CHECK-NEXT: [3]   - CortexA510UnitDiv
+# CHECK-NEXT: [4]   - CortexA510UnitLd1
+# CHECK-NEXT: [5]   - CortexA510UnitLdSt
+# CHECK-NEXT: [6]   - CortexA510UnitMAC
+# CHECK-NEXT: [7]   - CortexA510UnitPAC
+# CHECK-NEXT: [8]   - CortexA510UnitVALU0
+# CHECK-NEXT: [9]   - CortexA510UnitVALU1
+# CHECK-NEXT: [10.0] - CortexA510UnitVMAC
+# CHECK-NEXT: [10.1] - CortexA510UnitVMAC
+# CHECK-NEXT: [11]  - CortexA510UnitVMC
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1.0]  [1.1]  [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10.0] [10.1] [11]
+# CHECK-NEXT:  -      -      -      -      -     39.00  91.00   -      -     509.00 509.00 3.00   3.00   197.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1.0]  [1.1]  [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10.0] [10.1] [11]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     abs	d29, d24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     abs	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     abs	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     abs	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     abs	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     abs	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     abs	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     abs	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	d17, d31, d29
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     addhn	v0.2s, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     addhn	v0.4h, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     addhn	v0.8b, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     addhn2	v0.16b, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     addhn2	v0.4s, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     addhn2	v0.8h, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     addp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     addp	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     and	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     bic	v0.4h, #15, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     bic	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     bif	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     bit	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     bsl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cls	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cls	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cls	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cls	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cls	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cls	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     clz	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     clz	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     clz	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     clz	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     clz	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     clz	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmeq	d20, d21, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmeq	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmeq	v0.16b, v0.16b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmeq	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmge	d20, d21, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmge	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmge	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmge	v0.8b, v0.8b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmgt	d20, d21, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmgt	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmgt	v0.2s, v0.2s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmgt	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmhi	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmhi	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmhs	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmhs	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmle	d20, d21, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmle	v0.2d, v0.2d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmlt	d20, d21, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmlt	v0.8h, v0.8h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmtst	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmtst	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cnt	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cnt	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     dup	v0.16b, w28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     dup	v0.2d, x28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     dup	v0.2s, w28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     dup	v0.4h, w28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     dup	v0.4s, w28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     dup	v0.8b, w28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     dup	v0.8h, w28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     eor	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ext	v0.16b, v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ext	v0.8b, v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fabd	d29, d24, d20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fabd	s29, s24, s20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fabd	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fabs	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fabs	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fabs	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fabs	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fabs	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facge	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facge	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facge	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facgt	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facgt	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facgt	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fadd	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     faddp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     faddp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmeq	d20, d21, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmeq	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmeq	s10, s11, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmeq	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmeq	v0.2s, v0.2s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmeq	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmge	d20, d21, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmge	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmge	s10, s11, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmge	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmge	v0.2d, v0.2d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmge	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmgt	d20, d21, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmgt	d20, d21, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmgt	s10, s11, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmgt	s10, s11, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmgt	v0.4s, v0.4s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmgt	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmle	d20, d21, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmle	s10, s11, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmle	v0.2d, v0.2d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmlt	d20, d21, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmlt	s10, s11, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmlt	v0.4s, v0.4s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtas	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtas	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtas	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtas	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtas	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtas	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtas	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtau	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtau	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtau	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtau	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtau	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtau	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtau	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtl	v0.2d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtl	v0.4s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtl2	v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtl2	v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtms	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtms	s22, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtms	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtms	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtms	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtms	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtms	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtmu	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtmu	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtmu	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtmu	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtmu	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtmu	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtmu	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtn	v0.2s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtn	v0.4h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtn2	v0.4s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtn2	v0.8h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtns	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtns	s22, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtns	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtns	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtns	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtns	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtns	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtnu	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtnu	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtnu	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtnu	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtnu	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtnu	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtnu	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtps	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtps	s22, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtps	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtps	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtps	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtps	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtps	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtpu	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtpu	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtpu	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtpu	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtpu	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtpu	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtpu	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtxn	s22, d13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtxn	v0.2s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtxn2	v0.4s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	d21, d12, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	s21, s12, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	d21, d12, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	s12, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	s21, s12, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     10.00  fdiv	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmax	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmax	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmax	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmaxnm	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmaxnm	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmaxnm	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmaxnmp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmaxnmp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmaxnmp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmaxp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmaxp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmaxp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmin	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmin	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmin	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fminnm	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fminnm	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fminnm	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fminnmp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fminnmp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fminnmp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fminp	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fminp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fminp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmla	d0, d1, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmla	s0, s1, v0.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmla	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmls	d0, d4, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmls	s3, s5, v0.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmls	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmov	v0.2d, #-1.25000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmov	v0.2s, #13.00000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmov	v0.4s, #1.00000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmul	d0, d1, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmul	s0, s1, v0.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmul	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmulx	d0, d4, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmulx	d23, d11, d1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmulx	s20, s22, s15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmulx	s3, s5, v0.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmulx	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmulx	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmulx	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fneg	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fneg	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fneg	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fneg	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fneg	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frecpe	d13, d13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frecpe	s19, s14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frecpe	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frecpe	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frecpe	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frecpe	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frecpe	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frecps	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frecps	d22, d30, d21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frecps	s21, s16, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frecpx	d16, d19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frecpx	s18, s10
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frinta	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frinta	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frinta	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frinta	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frinta	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frinti	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frinti	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frinti	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frinti	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frinti	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintm	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintm	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintm	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintm	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintm	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintn	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintn	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintn	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintn	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintn	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintp	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintp	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintp	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintp	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintp	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintx	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintx	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintx	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintx	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintx	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintz	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintz	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintz	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintz	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintz	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     19.00  frsqrte	d21, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     9.00   frsqrte	s22, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     19.00  frsqrte	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     9.00   frsqrte	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     5.00   frsqrte	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     9.00   frsqrte	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     5.00   frsqrte	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     19.00  frsqrts	d8, d22, d18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     9.00   frsqrts	s21, s5, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     19.00  frsqrts	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     19.00  fsqrt	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     9.00   fsqrt	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     5.00   fsqrt	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     9.00   fsqrt	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     5.00   fsqrt	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fsub	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1	{ v0.16b }, [x0]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld1	{ v0.2d, v1.2d, v2.2d }, [x0], #48
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld1	{ v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld1	{ v0.4s, v1.4s }, [sp], #32
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld1	{ v0.4s, v1.4s, v2.4s }, [sp]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld1	{ v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld1	{ v0.8h }, [x15], x2
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1	{ v0.8h, v1.8h }, [x15]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1	{ v0.b }[9], [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1	{ v0.b }[9], [x0], #1
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1r	{ v0.16b }, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1r	{ v0.16b }, [x0], #1
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1r	{ v0.8h }, [x15]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1r	{ v0.8h }, [x15], #2
+# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -      -     ld2	{ v0.16b, v1.16b }, [x0], x1
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld2	{ v0.8b, v1.8b }, [x0]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld2	{ v0.h, v1.h }[7], [x15]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld2	{ v0.h, v1.h }[7], [x15], #4
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld2r	{ v0.2d, v1.2d }, [x0]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld2r	{ v0.2d, v1.2d }, [x0], #16
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld2r	{ v0.4s, v1.4s }, [sp]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld2r	{ v0.4s, v1.4s }, [sp], #8
+# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -      -      -      -      -      -     ld3	{ v0.4h, v1.4h, v2.4h }, [x15]
+# CHECK-NEXT:  -      -      -      -      -     1.50   1.50    -      -      -      -      -      -      -     ld3	{ v0.8h, v1.8h, v2.8h }, [x15], x2
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld3	{ v0.s, v1.s, v2.s }[3], [sp]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld3	{ v0.s, v1.s, v2.s }[3], [sp], x3
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld3r	{ v0.4h, v1.4h, v2.4h }, [x15]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld3r	{ v0.4h, v1.4h, v2.4h }, [x15], #6
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld3r	{ v0.8b, v1.8b, v2.8b }, [x0]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld3r	{ v0.8b, v1.8b, v2.8b }, [x0], #3
+# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -      -     ld4	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+# CHECK-NEXT:  -      -      -      -      -     2.00   2.00    -      -      -      -      -      -      -     ld4	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld4	{ v0.d, v1.d, v2.d, v3.d }[1], [x0], #32
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld4	{ v0.h, v1.h, v2.h, v3.h }[7], [x0], x0
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld4r	{ v0.1d, v1.1d, v2.1d, v3.1d }, [sp]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld4r	{ v0.1d, v1.1d, v2.1d, v3.1d }, [sp], x7
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+# CHECK-NEXT:  -      -      -      -      -     1.00   1.00    -      -      -      -      -      -      -     ld4r	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp], x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mla	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mls	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	b0, v0.b[15]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	d6, v0.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	h2, v0.h[5]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	s17, v0.s[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     movi	d15, #0xff00ff00ff00ff
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     movi	v0.16b, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     movi	v0.2d, #0xff0000ff0000ffff
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     movi	v0.2s, #8, msl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     movi	v0.4s, #255, lsl #24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     movi	v0.8b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mul	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mvni	v0.2s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mvni	v0.4s, #16, msl #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     neg	d29, d24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     neg	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     neg	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     neg	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     neg	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     neg	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     neg	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     neg	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mvn	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mvn	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     orn	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     orr	v0.8h, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     pmul	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     pmul	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     pmull	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     pmull2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     raddhn	v0.2s, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     raddhn	v0.4h, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     raddhn	v0.8b, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     raddhn2	v0.16b, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     raddhn2	v0.4s, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     raddhn2	v0.8h, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rbit	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rbit	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rev16	v21.8b, v1.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rev16	v30.16b, v31.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rev32	v0.4h, v9.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rev32	v21.8b, v1.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rev32	v30.16b, v31.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rev32	v4.8h, v7.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rev64	v0.16b, v31.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rev64	v1.8b, v9.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rev64	v13.4h, v21.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rev64	v2.8h, v4.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rev64	v4.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rev64	v6.4s, v8.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rsubhn	v0.2s, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rsubhn	v0.4h, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rsubhn	v0.8b, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rsubhn2	v0.16b, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rsubhn2	v0.4s, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rsubhn2	v0.8h, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saba	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sabal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sabal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sabal	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sabal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sabal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sabal2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sabd	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sabdl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sabdl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sabdl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sabdl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sabdl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sabdl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     sadalp	v0.1d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     sadalp	v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     sadalp	v0.2s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     sadalp	v0.4h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     sadalp	v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     sadalp	v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddlp	v0.1d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddlp	v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddlp	v0.2s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddlp	v0.4h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddlp	v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddlp	v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddw	v0.2d, v0.2d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddw	v0.4s, v0.4s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddw	v0.8h, v0.8h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddw2	v0.2d, v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddw2	v0.4s, v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	d21, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	d21, d12, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	s22, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	s22, s13, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shadd	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shl	d7, d10, #12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shl	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shl	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shl	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shl	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shll	v0.2d, v0.2s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shll	v0.4s, v0.4h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shll	v0.8h, v0.8b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shll	v0.2d, v0.2s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shll	v0.4s, v0.4h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shll	v0.8h, v0.8b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shll2	v0.2d, v0.4s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shll2	v0.4s, v0.8h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shll2	v0.8h, v0.16b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shll2	v0.2d, v0.4s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shll2	v0.4s, v0.8h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shll2	v0.8h, v0.16b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shsub	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shsub	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sli	d10, d14, #12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sli	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sli	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sli	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sli	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sli	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sli	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sli	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smax	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smax	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smax	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smaxp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smaxp	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smaxp	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smin	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smin	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smin	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sminp	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sminp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sminp	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smlal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smlal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smlal	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smlal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smlal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smlal2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smlsl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smlsl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smlsl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smlsl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smlsl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smlsl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smull	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smull	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smull	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smull2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smull2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smull2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqabs	b19, b14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqabs	d18, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqabs	h21, h15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqabs	s20, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqabs	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqabs	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqabs	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqabs	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqabs	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqabs	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqabs	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqadd	b20, b11, b15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqadd	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqadd	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmlal	d19, s24, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmlal	d8, s9, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmlal	s0, h0, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmlal	s17, h27, h12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmlal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmlal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmlal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmlal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmlsl	d12, s23, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmlsl	d8, s9, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmlsl	s0, h0, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmlsl	s14, h12, h25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmlsl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmlsl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmlsl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmlsl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmulh	h10, h11, h12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmulh	h7, h15, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmulh	s15, s14, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmulh	s20, s21, s2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmulh	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmulh	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmull	d1, s1, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmull	d15, s22, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmull	s1, h1, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmull	s12, h22, h12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmull	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmull	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmull2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdmull2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqneg	b19, b14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqneg	d18, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqneg	h21, h15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqneg	s20, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqneg	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqneg	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqneg	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqneg	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqneg	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqneg	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqneg	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrdmulh	h10, h11, h12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrdmulh	h7, h15, v0.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrdmulh	s15, s14, v0.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrdmulh	s20, s21, s2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrdmulh	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrdmulh	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshl	d31, d31, d31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshl	h3, h4, h15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshl	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshl	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrn	b10, h13, #2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrn	h15, s10, #6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrn	s15, d12, #9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrun	b17, h10, #6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrun	h10, s13, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrun	s22, d16, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrun	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrun	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrun	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrun2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrun2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrun2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	b11, b19, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	d15, d16, #51
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	d31, d31, d31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	h13, h18, #11
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	h3, h4, h15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	s14, s17, #22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshlu	b15, b18, #6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshlu	d11, d13, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshlu	h19, h17, #6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshlu	s16, s14, #25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshlu	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshlu	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshlu	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshlu	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshlu	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshlu	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshlu	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrn	b10, h15, #5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrn	h17, s10, #4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrn	s18, d10, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrun	b15, h10, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrun	h20, s14, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrun	s10, d15, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrun	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrun	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrun	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrun2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrun2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrun2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsub	s20, s10, s7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsub	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsub	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsub	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtn	b18, h18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtn	h20, s17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtn	s19, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtn	v0.2s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtn	v0.4h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtn	v0.8b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtn2	v0.16b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtn2	v0.4s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtn2	v0.8h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtun	b19, h14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtun	h21, s15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtun	s20, d12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtun	v0.2s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtun	v0.4h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtun	v0.8b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtun2	v0.16b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtun2	v0.4s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtun2	v0.8h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srhadd	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srhadd	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srhadd	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sri	d10, d12, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sri	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sri	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sri	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sri	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sri	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sri	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sri	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshl	d16, d16, d16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshl	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshl	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshr	d19, d18, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshr	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshr	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshr	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshr	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshr	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshr	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshr	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     srsra	d15, d11, #19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     srsra	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     srsra	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     srsra	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     srsra	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     srsra	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     srsra	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     srsra	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshl	d31, d31, d31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshl	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshl	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshl	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshl	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshll	v0.2d, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshll2	v0.4s, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshr	d15, d16, #12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshr	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshr	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshr	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshr	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshr	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshr	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshr	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     ssra	d18, d12, #21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     ssra	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     ssra	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     ssra	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     ssra	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     ssra	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     ssra	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     ssra	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssubl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssubl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssubl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssubl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssubl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssubl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssubw	v0.2d, v0.2d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssubw	v0.4s, v0.4s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssubw	v0.8h, v0.8h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssubw2	v0.2d, v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssubw2	v0.4s, v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssubw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1	{ v0.16b }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     st1	{ v0.2d, v1.2d, v2.2d }, [x0], #48
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -      -      -      -      -      -      -     st1	{ v0.2d, v1.2d, v2.2d, v3.2d }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1	{ v0.4s, v1.4s }, [sp], #32
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     st1	{ v0.4s, v1.4s, v2.4s }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -      -      -      -      -      -      -     st1	{ v0.8b, v1.8b, v2.8b, v3.8b }, [x0], x3
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1	{ v0.8h }, [x15], x2
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1	{ v0.8h, v1.8h }, [x15]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1	{ v0.d }[1], [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1	{ v0.d }[1], [x0], #8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -      -      -      -      -      -      -     st2	{ v0.16b, v1.16b }, [x0], x1
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     st2	{ v0.8b, v1.8b }, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     st2	{ v0.s, v1.s }[3], [sp]
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     st2	{ v0.s, v1.s }[3], [sp], #8
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -      -      -      -      -      -      -     st3	{ v0.4h, v1.4h, v2.4h }, [x15]
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -      -      -      -      -      -      -     st3	{ v0.8h, v1.8h, v2.8h }, [x15], x2
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     st3	{ v0.h, v1.h, v2.h }[7], [x15]
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     st3	{ v0.h, v1.h, v2.h }[7], [x15], #6
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -      -      -      -      -      -      -     st4	{ v0.2s, v1.2s, v2.2s, v3.2s }, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -     4.00    -      -      -      -      -      -      -     st4	{ v0.4s, v1.4s, v2.4s, v3.4s }, [sp], #64
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     st4	{ v0.b, v1.b, v2.b, v3.b }[9], [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     st4	{ v0.b, v1.b, v2.b, v3.b }[9], [x0], x5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	d15, d5, d16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     suqadd	b19, b14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     suqadd	d18, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     suqadd	h20, h15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     suqadd	s21, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     suqadd	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     suqadd	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     suqadd	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     suqadd	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     suqadd	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     suqadd	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     suqadd	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbl	v0.16b, { v0.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbl	v0.16b, { v0.16b, v1.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbl	v0.16b, { v0.16b, v1.16b, v2.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbl	v0.16b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbl	v0.8b, { v0.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbl	v0.8b, { v0.16b, v1.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbl	v0.8b, { v0.16b, v1.16b, v2.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbl	v0.8b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbx	v0.16b, { v0.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbx	v0.16b, { v0.16b, v1.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbx	v0.16b, { v0.16b, v1.16b, v2.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbx	v0.16b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbx	v0.8b, { v0.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbx	v0.8b, { v0.16b, v1.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbx	v0.8b, { v0.16b, v1.16b, v2.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbx	v0.8b, { v0.16b, v1.16b, v2.16b, v3.16b }, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     trn1	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     trn1	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     trn1	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     trn1	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     trn1	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     trn1	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     trn1	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     trn2	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     trn2	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     trn2	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     trn2	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     trn2	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     trn2	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     trn2	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaba	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uabal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uabal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uabal	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uabal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uabal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uabal2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uabd	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uabdl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uabdl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uabdl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uabdl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uabdl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uabdl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     uadalp	v0.1d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     uadalp	v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     uadalp	v0.2s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     uadalp	v0.4h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     uadalp	v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     uadalp	v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddlp	v0.1d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddlp	v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddlp	v0.2s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddlp	v0.4h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddlp	v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddlp	v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddw	v0.2d, v0.2d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddw	v0.4s, v0.4s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddw	v0.8h, v0.8h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddw2	v0.2d, v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddw2	v0.4s, v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	d21, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	d21, d14, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	s22, s13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	s22, s13, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uhadd	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uhadd	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uhsub	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umax	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umax	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umax	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umaxp	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umaxp	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umaxp	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umin	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umin	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umin	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uminp	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uminp	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uminp	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umlal	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umlal	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umlal	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umlal2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umlal2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umlal2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umlsl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umlsl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umlsl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umlsl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umlsl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umlsl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umull	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umull	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umull	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umull2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umull2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umull2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqadd	h0, h1, h5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqadd	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshl	b11, b20, b30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshl	s23, s20, s16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshl	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshl	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshrn	b10, h12, #5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshrn	h12, s10, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshrn	s10, d10, #25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	b11, b20, b30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	b18, b15, #6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	d15, d12, #19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	h11, h18, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	s14, s19, #18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	s23, s20, s16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshrn	b12, h10, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshrn	h10, s14, #5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshrn	s10, d12, #13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshrn	v0.2s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshrn	v0.4h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshrn	v0.8b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshrn2	v0.16b, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshrn2	v0.4s, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshrn2	v0.8h, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqsub	d16, d16, d16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqsub	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqxtn	b18, h18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqxtn	h20, s17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqxtn	s19, d14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqxtn	v0.2s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqxtn	v0.4h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqxtn	v0.8b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqxtn2	v0.16b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqxtn2	v0.4s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqxtn2	v0.8h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urecpe	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urecpe	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urhadd	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urhadd	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urhadd	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshl	d8, d7, d4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshl	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshl	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshl	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshr	d20, d23, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshr	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshr	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshr	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshr	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshr	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshr	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshr	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     9.00   ursqrte	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     9.00   ursqrte	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     ursra	d18, d10, #13
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     ursra	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     ursra	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     ursra	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     ursra	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     ursra	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     ursra	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     ursra	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ushl	d0, d0, d0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ushl	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ushl	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ushl	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ushll	v0.4s, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ushll2	v0.8h, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ushr	d10, d17, #18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ushr	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ushr	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ushr	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ushr	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ushr	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ushr	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ushr	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usqadd	b19, b14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usqadd	d18, d22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usqadd	h20, h15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usqadd	s21, s12
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usqadd	v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usqadd	v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usqadd	v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usqadd	v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usqadd	v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usqadd	v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usqadd	v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     usra	d20, d13, #61
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     usra	v0.16b, v0.16b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     usra	v0.2d, v0.2d, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     usra	v0.2s, v0.2s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     usra	v0.4h, v0.4h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     usra	v0.4s, v0.4s, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     usra	v0.8b, v0.8b, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     usra	v0.8h, v0.8h, #3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usubl	v0.2d, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usubl	v0.4s, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usubl	v0.8h, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usubl2	v0.2d, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usubl2	v0.4s, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usubl2	v0.8h, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usubw	v0.2d, v0.2d, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usubw	v0.4s, v0.4s, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usubw	v0.8h, v0.8h, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usubw2	v0.2d, v0.2d, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usubw2	v0.4s, v0.4s, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usubw2	v0.8h, v0.8h, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uzp1	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uzp1	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uzp1	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uzp1	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uzp1	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uzp1	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uzp1	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uzp2	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uzp2	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uzp2	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uzp2	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uzp2	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uzp2	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uzp2	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     xtn	v0.2s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     xtn	v0.4h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     xtn	v0.8b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     xtn2	v0.16b, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     xtn2	v0.4s, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     xtn2	v0.8h, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip1	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip1	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip1	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip1	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip1	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip1	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip1	v0.8h, v0.8h, v0.8h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip2	v0.16b, v0.16b, v0.16b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip2	v0.2d, v0.2d, v0.2d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip2	v0.2s, v0.2s, v0.2s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip2	v0.4h, v0.4h, v0.4h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip2	v0.4s, v0.4s, v0.4s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip2	v0.8b, v0.8b, v0.8b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip2	v0.8h, v0.8h, v0.8h
diff --git a/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-sve-instructions.s b/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-sve-instructions.s
new file mode 100644
index 0000000000000..867bb10e7d257
--- /dev/null
+++ b/llvm/test/tools/llvm-mca/AArch64/Cortex/A510-sve-instructions.s
@@ -0,0 +1,10263 @@
+# NOTE: Assertions have been autogenerated by utils/update_mca_test_checks.py
+# RUN: llvm-mca -mtriple=aarch64 -mcpu=cortex-a510 -mattr=+sve2-aes,+sve2-sha3,+sve2-sm4 -instruction-tables < %s | FileCheck %s
+
+abs	z0.b, p0/m, z0.b
+abs	z0.d, p0/m, z0.d
+abs	z0.h, p0/m, z0.h
+abs	z0.s, p0/m, z0.s
+abs	z31.b, p7/m, z31.b
+abs	z31.d, p7/m, z31.d
+abs	z31.h, p7/m, z31.h
+abs	z31.s, p7/m, z31.s
+adclb	z0.d, z1.d, z31.d
+adclb	z0.s, z1.s, z31.s
+adclt	z0.d, z1.d, z31.d
+adclt	z0.s, z1.s, z31.s
+add	z0.b, p0/m, z0.b, z0.b
+add	z0.b, z0.b, #0
+add	z0.b, z0.b, z0.b
+add	z0.d, p0/m, z0.d, z0.d
+add	z0.d, z0.d, #0
+add	z0.d, z0.d, #0, lsl #8
+add	z0.d, z0.d, z0.d
+add	z0.h, p0/m, z0.h, z0.h
+add	z0.h, z0.h, #0
+add	z0.h, z0.h, #0, lsl #8
+add	z0.h, z0.h, z0.h
+add	z0.s, p0/m, z0.s, z0.s
+add	z0.s, z0.s, #0
+add	z0.s, z0.s, #0, lsl #8
+add	z0.s, z0.s, z0.s
+add	z0.s, z1.s, z2.s
+add	z21.b, p5/m, z21.b, z10.b
+add	z21.b, z10.b, z21.b
+add	z21.d, p5/m, z21.d, z10.d
+add	z21.d, z10.d, z21.d
+add	z21.h, p5/m, z21.h, z10.h
+add	z21.h, z10.h, z21.h
+add	z21.s, p5/m, z21.s, z10.s
+add	z21.s, z10.s, z21.s
+add	z23.b, p3/m, z23.b, z13.b
+add	z23.b, z13.b, z8.b
+add	z23.d, p3/m, z23.d, z13.d
+add	z23.d, z13.d, z8.d
+add	z23.h, p3/m, z23.h, z13.h
+add	z23.h, z13.h, z8.h
+add	z23.s, p3/m, z23.s, z13.s
+add	z23.s, z13.s, z8.s
+add	z31.b, p7/m, z31.b, z31.b
+add	z31.b, z31.b, #255
+add	z31.b, z31.b, z31.b
+add	z31.d, p7/m, z31.d, z31.d
+add	z31.d, z31.d, #65280
+add	z31.d, z31.d, z31.d
+add	z31.h, p7/m, z31.h, z31.h
+add	z31.h, z31.h, #65280
+add	z31.h, z31.h, z31.h
+add	z31.s, p7/m, z31.s, z31.s
+add	z31.s, z31.s, #65280
+add	z31.s, z31.s, z31.s
+addhnb	z0.b, z1.h, z31.h
+addhnb	z0.h, z1.s, z31.s
+addhnb	z0.s, z1.d, z31.d
+addhnt	z0.b, z1.h, z31.h
+addhnt	z0.h, z1.s, z31.s
+addhnt	z0.s, z1.d, z31.d
+addp	z0.b, p0/m, z0.b, z1.b
+addp	z0.h, p0/m, z0.h, z1.h
+addp	z29.s, p7/m, z29.s, z30.s
+addp	z31.d, p7/m, z31.d, z30.d
+addpl	sp, sp, #31
+addpl	x0, x0, #-32
+addpl	x21, x21, #0
+addpl	x23, x8, #-1
+addvl	sp, sp, #31
+addvl	x0, x0, #-32
+addvl	x21, x21, #0
+addvl	x23, x8, #-1
+adr	z0.d, [z0.d, z0.d, lsl #1]
+adr	z0.d, [z0.d, z0.d, lsl #2]
+adr	z0.d, [z0.d, z0.d, lsl #3]
+adr	z0.d, [z0.d, z0.d, sxtw #1]
+adr	z0.d, [z0.d, z0.d, sxtw #2]
+adr	z0.d, [z0.d, z0.d, sxtw #3]
+adr	z0.d, [z0.d, z0.d, sxtw]
+adr	z0.d, [z0.d, z0.d, uxtw #1]
+adr	z0.d, [z0.d, z0.d, uxtw #2]
+adr	z0.d, [z0.d, z0.d, uxtw #3]
+adr	z0.d, [z0.d, z0.d, uxtw]
+adr	z0.d, [z0.d, z0.d]
+adr	z0.s, [z0.s, z0.s, lsl #1]
+adr	z0.s, [z0.s, z0.s, lsl #2]
+adr	z0.s, [z0.s, z0.s, lsl #3]
+adr	z0.s, [z0.s, z0.s]
+aesd	z0.b, z0.b, z31.b
+aese	z0.b, z0.b, z31.b
+aesimc	z0.b, z0.b
+aesimc	z31.b, z31.b
+aesmc	z0.b, z0.b
+aesmc	z31.b, z31.b
+and	p0.b, p0/z, p0.b, p1.b
+and	z0.d, z0.d, #0x6
+and	z0.d, z0.d, #0xfffffffffffffff9
+and	z0.d, z0.d, z0.d
+and	z0.s, z0.s, #0x6
+and	z0.s, z0.s, #0xfffffff9
+and	z23.d, z13.d, z8.d
+and	z23.h, z23.h, #0x6
+and	z23.h, z23.h, #0xfff9
+and	z31.b, p7/m, z31.b, z31.b
+and	z31.d, p7/m, z31.d, z31.d
+and	z31.h, p7/m, z31.h, z31.h
+and	z31.s, p7/m, z31.s, z31.s
+and	z5.b, z5.b, #0x6
+and	z5.b, z5.b, #0xf9
+ands	p0.b, p0/z, p0.b, p1.b
+andv	b0, p7, z31.b
+andv	d0, p7, z31.d
+andv	h0, p7, z31.h
+andv	s0, p7, z31.s
+asr	z0.b, p0/m, z0.b, #1
+asr	z0.b, p0/m, z0.b, z0.b
+asr	z0.b, p0/m, z0.b, z1.d
+asr	z0.b, z0.b, #1
+asr	z0.b, z1.b, z2.d
+asr	z0.d, p0/m, z0.d, #1
+asr	z0.d, p0/m, z0.d, z0.d
+asr	z0.d, z0.d, #1
+asr	z0.h, p0/m, z0.h, #1
+asr	z0.h, p0/m, z0.h, z0.h
+asr	z0.h, p0/m, z0.h, z1.d
+asr	z0.h, z0.h, #1
+asr	z0.h, z1.h, z2.d
+asr	z0.s, p0/m, z0.s, #1
+asr	z0.s, p0/m, z0.s, z0.s
+asr	z0.s, p0/m, z0.s, z1.d
+asr	z0.s, z0.s, #1
+asr	z0.s, z1.s, z2.d
+asr	z31.b, p0/m, z31.b, #8
+asr	z31.b, z31.b, #8
+asr	z31.d, p0/m, z31.d, #64
+asr	z31.d, z31.d, #64
+asr	z31.h, p0/m, z31.h, #16
+asr	z31.h, z31.h, #16
+asr	z31.s, p0/m, z31.s, #32
+asr	z31.s, z31.s, #32
+asrd	z0.b, p0/m, z0.b, #1
+asrd	z0.d, p0/m, z0.d, #1
+asrd	z0.h, p0/m, z0.h, #1
+asrd	z0.s, p0/m, z0.s, #1
+asrd	z31.b, p0/m, z31.b, #8
+asrd	z31.d, p0/m, z31.d, #64
+asrd	z31.h, p0/m, z31.h, #16
+asrd	z31.s, p0/m, z31.s, #32
+asrr	z0.b, p0/m, z0.b, z0.b
+asrr	z0.d, p0/m, z0.d, z0.d
+asrr	z0.h, p0/m, z0.h, z0.h
+asrr	z0.s, p0/m, z0.s, z0.s
+bcax	z29.d, z29.d, z30.d, z31.d
+bdep	z0.b, z1.b, z31.b
+bdep	z0.d, z1.d, z31.d
+bdep	z0.h, z1.h, z31.h
+bdep	z0.s, z1.s, z31.s
+bext	z0.b, z1.b, z31.b
+bext	z0.d, z1.d, z31.d
+bext	z0.h, z1.h, z31.h
+bext	z0.s, z1.s, z31.s
+bfcvt	z0.h, p0/m, z1.s
+bfcvtnt	z0.h, p0/m, z1.s
+bfdot	z0.s, z1.h, z2.h
+bfdot	z0.s, z1.h, z2.h[0]
+bfdot	z0.s, z1.h, z2.h[3]
+bfmlalb	z0.s, z1.h, z2.h
+bfmlalb	z0.s, z1.h, z2.h[0]
+bfmlalb	z0.s, z1.h, z2.h[7]
+bfmlalb	z10.s, z21.h, z14.h
+bfmlalb	z21.s, z14.h, z3.h[2]
+bfmlalt	z0.s, z1.h, z2.h
+bfmlalt	z0.s, z1.h, z2.h[0]
+bfmlalt	z0.s, z1.h, z2.h[7]
+bfmlalt	z0.s, z1.h, z7.h[7]
+bfmlalt	z14.s, z10.h, z21.h
+bfmmla	z0.s, z1.h, z2.h
+bgrp	z0.b, z1.b, z31.b
+bgrp	z0.d, z1.d, z31.d
+bgrp	z0.h, z1.h, z31.h
+bgrp	z0.s, z1.s, z31.s
+bic	p0.b, p0/z, p0.b, p0.b
+bic	p15.b, p15/z, p15.b, p15.b
+bic	z0.d, z0.d, z0.d
+bic	z23.d, z13.d, z8.d
+bic	z31.b, p7/m, z31.b, z31.b
+bic	z31.d, p7/m, z31.d, z31.d
+bic	z31.h, p7/m, z31.h, z31.h
+bic	z31.s, p7/m, z31.s, z31.s
+bics	p0.b, p0/z, p0.b, p0.b
+bics	p15.b, p15/z, p15.b, p15.b
+brka	p0.b, p15/m, p15.b
+brka	p0.b, p15/z, p15.b
+brkas	p0.b, p15/z, p15.b
+brkb	p0.b, p15/m, p15.b
+brkb	p0.b, p15/z, p15.b
+brkbs	p0.b, p15/z, p15.b
+brkn	p0.b, p15/z, p1.b, p0.b
+brkn	p15.b, p15/z, p15.b, p15.b
+brkns	p0.b, p15/z, p1.b, p0.b
+brkns	p15.b, p15/z, p15.b, p15.b
+brkpa	p0.b, p15/z, p1.b, p2.b
+brkpa	p15.b, p15/z, p15.b, p15.b
+brkpas	p0.b, p15/z, p1.b, p2.b
+brkpas	p15.b, p15/z, p15.b, p15.b
+brkpb	p0.b, p15/z, p1.b, p2.b
+brkpb	p15.b, p15/z, p15.b, p15.b
+brkpbs	p0.b, p15/z, p1.b, p2.b
+brkpbs	p15.b, p15/z, p15.b, p15.b
+bsl	z0.d, z0.d, z1.d, z2.d
+bsl1n	z0.d, z0.d, z1.d, z2.d
+bsl2n	z0.d, z0.d, z1.d, z2.d
+cadd	z0.b, z0.b, z0.b, #90
+cadd	z0.d, z0.d, z0.d, #90
+cadd	z0.h, z0.h, z0.h, #90
+cadd	z0.s, z0.s, z0.s, #90
+cadd	z31.b, z31.b, z31.b, #270
+cadd	z31.d, z31.d, z31.d, #270
+cadd	z31.h, z31.h, z31.h, #270
+cadd	z31.s, z31.s, z31.s, #270
+cdot	z0.d, z1.h, z15.h[1], #0
+cdot	z0.d, z1.h, z31.h, #0
+cdot	z0.d, z1.h, z31.h, #180
+cdot	z0.d, z1.h, z31.h, #270
+cdot	z0.d, z1.h, z31.h, #90
+cdot	z0.s, z1.b, z31.b, #0
+cdot	z0.s, z1.b, z7.b[3], #0
+cdot	z29.d, z30.h, z0.h[0], #180
+cdot	z31.d, z30.h, z7.h[1], #270
+cdot	z5.d, z6.h, z3.h[0], #90
+clasta	b0, p7, b0, z31.b
+clasta	d0, p7, d0, z31.d
+clasta	h0, p7, h0, z31.h
+clasta	s0, p7, s0, z31.s
+clasta	w0, p7, w0, z31.b
+clasta	w0, p7, w0, z31.h
+clasta	w0, p7, w0, z31.s
+clasta	x0, p7, x0, z31.d
+clasta	z0.b, p7, z0.b, z31.b
+clasta	z0.d, p7, z0.d, z31.d
+clasta	z0.h, p7, z0.h, z31.h
+clasta	z0.s, p7, z0.s, z31.s
+clastb	b0, p7, b0, z31.b
+clastb	d0, p7, d0, z31.d
+clastb	h0, p7, h0, z31.h
+clastb	s0, p7, s0, z31.s
+clastb	w0, p7, w0, z31.b
+clastb	w0, p7, w0, z31.h
+clastb	w0, p7, w0, z31.s
+clastb	x0, p7, x0, z31.d
+clastb	z0.b, p7, z0.b, z31.b
+clastb	z0.d, p7, z0.d, z31.d
+clastb	z0.h, p7, z0.h, z31.h
+clastb	z0.s, p7, z0.s, z31.s
+cls	z31.b, p7/m, z31.b
+cls	z31.d, p7/m, z31.d
+cls	z31.h, p7/m, z31.h
+cls	z31.s, p7/m, z31.s
+clz	z31.b, p7/m, z31.b
+clz	z31.d, p7/m, z31.d
+clz	z31.h, p7/m, z31.h
+clz	z31.s, p7/m, z31.s
+cmla	z0.b, z1.b, z2.b, #0
+cmla	z0.d, z1.d, z2.d, #0
+cmla	z0.h, z1.h, z2.h, #0
+cmla	z0.h, z1.h, z2.h[0], #0
+cmla	z0.s, z1.s, z2.s, #0
+cmla	z0.s, z1.s, z2.s[0], #0
+cmla	z15.b, z16.b, z17.b, #270
+cmla	z15.d, z16.d, z17.d, #270
+cmla	z15.h, z16.h, z17.h, #270
+cmla	z15.s, z16.s, z17.s, #270
+cmla	z29.b, z30.b, z31.b, #90
+cmla	z29.d, z30.d, z31.d, #90
+cmla	z29.h, z30.h, z31.h, #90
+cmla	z29.s, z30.s, z31.s, #90
+cmla	z31.b, z31.b, z31.b, #180
+cmla	z31.d, z31.d, z31.d, #180
+cmla	z31.h, z30.h, z7.h[0], #180
+cmla	z31.h, z31.h, z31.h, #180
+cmla	z31.s, z30.s, z7.s[0], #180
+cmla	z31.s, z31.s, z31.s, #180
+cmpeq	p0.b, p0/z, z0.b, #-16
+cmpeq	p0.b, p0/z, z0.b, #15
+cmpeq	p0.b, p0/z, z0.b, z0.b
+cmpeq	p0.b, p0/z, z0.b, z0.d
+cmpeq	p0.d, p0/z, z0.d, #-16
+cmpeq	p0.d, p0/z, z0.d, #15
+cmpeq	p0.d, p0/z, z0.d, z0.d
+cmpeq	p0.h, p0/z, z0.h, #-16
+cmpeq	p0.h, p0/z, z0.h, #15
+cmpeq	p0.h, p0/z, z0.h, z0.d
+cmpeq	p0.h, p0/z, z0.h, z0.h
+cmpeq	p0.s, p0/z, z0.s, #-16
+cmpeq	p0.s, p0/z, z0.s, #15
+cmpeq	p0.s, p0/z, z0.s, z0.d
+cmpeq	p0.s, p0/z, z0.s, z0.s
+cmpge	p0.b, p0/z, z0.b, #-16
+cmpge	p0.b, p0/z, z0.b, #15
+cmpge	p0.b, p0/z, z0.b, z0.b
+cmpge	p0.b, p0/z, z0.b, z0.d
+cmpge	p0.b, p0/z, z1.b, z0.b
+cmpge	p0.d, p0/z, z0.d, #-16
+cmpge	p0.d, p0/z, z0.d, #15
+cmpge	p0.d, p0/z, z0.d, z0.d
+cmpge	p0.d, p0/z, z1.d, z0.d
+cmpge	p0.h, p0/z, z0.h, #-16
+cmpge	p0.h, p0/z, z0.h, #15
+cmpge	p0.h, p0/z, z0.h, z0.d
+cmpge	p0.h, p0/z, z0.h, z0.h
+cmpge	p0.h, p0/z, z1.h, z0.h
+cmpge	p0.s, p0/z, z0.s, #-16
+cmpge	p0.s, p0/z, z0.s, #15
+cmpge	p0.s, p0/z, z0.s, z0.d
+cmpge	p0.s, p0/z, z0.s, z0.s
+cmpge	p0.s, p0/z, z1.s, z0.s
+cmpgt	p0.b, p0/z, z0.b, #-16
+cmpgt	p0.b, p0/z, z0.b, #15
+cmpgt	p0.b, p0/z, z0.b, z0.b
+cmpgt	p0.b, p0/z, z0.b, z0.d
+cmpgt	p0.b, p0/z, z1.b, z0.b
+cmpgt	p0.d, p0/z, z0.d, #-16
+cmpgt	p0.d, p0/z, z0.d, #15
+cmpgt	p0.d, p0/z, z0.d, z0.d
+cmpgt	p0.d, p0/z, z1.d, z0.d
+cmpgt	p0.h, p0/z, z0.h, #-16
+cmpgt	p0.h, p0/z, z0.h, #15
+cmpgt	p0.h, p0/z, z0.h, z0.d
+cmpgt	p0.h, p0/z, z0.h, z0.h
+cmpgt	p0.h, p0/z, z1.h, z0.h
+cmpgt	p0.s, p0/z, z0.s, #-16
+cmpgt	p0.s, p0/z, z0.s, #15
+cmpgt	p0.s, p0/z, z0.s, z0.d
+cmpgt	p0.s, p0/z, z0.s, z0.s
+cmpgt	p0.s, p0/z, z1.s, z0.s
+cmphi	p0.b, p0/z, z0.b, #0
+cmphi	p0.b, p0/z, z0.b, #127
+cmphi	p0.b, p0/z, z0.b, z0.b
+cmphi	p0.b, p0/z, z0.b, z0.d
+cmphi	p0.b, p0/z, z1.b, z0.b
+cmphi	p0.d, p0/z, z0.d, #0
+cmphi	p0.d, p0/z, z0.d, #127
+cmphi	p0.d, p0/z, z0.d, z0.d
+cmphi	p0.d, p0/z, z1.d, z0.d
+cmphi	p0.h, p0/z, z0.h, #0
+cmphi	p0.h, p0/z, z0.h, #127
+cmphi	p0.h, p0/z, z0.h, z0.d
+cmphi	p0.h, p0/z, z0.h, z0.h
+cmphi	p0.h, p0/z, z1.h, z0.h
+cmphi	p0.s, p0/z, z0.s, #0
+cmphi	p0.s, p0/z, z0.s, #127
+cmphi	p0.s, p0/z, z0.s, z0.d
+cmphi	p0.s, p0/z, z0.s, z0.s
+cmphi	p0.s, p0/z, z1.s, z0.s
+cmphs	p0.b, p0/z, z0.b, #0
+cmphs	p0.b, p0/z, z0.b, #127
+cmphs	p0.b, p0/z, z0.b, z0.b
+cmphs	p0.b, p0/z, z0.b, z0.d
+cmphs	p0.b, p0/z, z1.b, z0.b
+cmphs	p0.d, p0/z, z0.d, #0
+cmphs	p0.d, p0/z, z0.d, #127
+cmphs	p0.d, p0/z, z0.d, z0.d
+cmphs	p0.d, p0/z, z1.d, z0.d
+cmphs	p0.h, p0/z, z0.h, #0
+cmphs	p0.h, p0/z, z0.h, #127
+cmphs	p0.h, p0/z, z0.h, z0.d
+cmphs	p0.h, p0/z, z0.h, z0.h
+cmphs	p0.h, p0/z, z1.h, z0.h
+cmphs	p0.s, p0/z, z0.s, #0
+cmphs	p0.s, p0/z, z0.s, #127
+cmphs	p0.s, p0/z, z0.s, z0.d
+cmphs	p0.s, p0/z, z0.s, z0.s
+cmphs	p0.s, p0/z, z1.s, z0.s
+cmple	p0.b, p0/z, z0.b, #-16
+cmple	p0.b, p0/z, z0.b, #15
+cmple	p0.b, p0/z, z0.b, z0.d
+cmple	p0.d, p0/z, z0.d, #-16
+cmple	p0.d, p0/z, z0.d, #15
+cmple	p0.h, p0/z, z0.h, #-16
+cmple	p0.h, p0/z, z0.h, #15
+cmple	p0.h, p0/z, z0.h, z0.d
+cmple	p0.s, p0/z, z0.s, #-16
+cmple	p0.s, p0/z, z0.s, #15
+cmple	p0.s, p0/z, z0.s, z0.d
+cmplo	p0.b, p0/z, z0.b, #0
+cmplo	p0.b, p0/z, z0.b, #127
+cmplo	p0.b, p0/z, z0.b, z0.d
+cmplo	p0.d, p0/z, z0.d, #0
+cmplo	p0.d, p0/z, z0.d, #127
+cmplo	p0.h, p0/z, z0.h, #0
+cmplo	p0.h, p0/z, z0.h, #127
+cmplo	p0.h, p0/z, z0.h, z0.d
+cmplo	p0.s, p0/z, z0.s, #0
+cmplo	p0.s, p0/z, z0.s, #127
+cmplo	p0.s, p0/z, z0.s, z0.d
+cmpls	p0.b, p0/z, z0.b, #0
+cmpls	p0.b, p0/z, z0.b, #127
+cmpls	p0.b, p0/z, z0.b, z0.d
+cmpls	p0.d, p0/z, z0.d, #0
+cmpls	p0.d, p0/z, z0.d, #127
+cmpls	p0.h, p0/z, z0.h, #0
+cmpls	p0.h, p0/z, z0.h, #127
+cmpls	p0.h, p0/z, z0.h, z0.d
+cmpls	p0.s, p0/z, z0.s, #0
+cmpls	p0.s, p0/z, z0.s, #127
+cmpls	p0.s, p0/z, z0.s, z0.d
+cmplt	p0.b, p0/z, z0.b, #-16
+cmplt	p0.b, p0/z, z0.b, #15
+cmplt	p0.b, p0/z, z0.b, z0.d
+cmplt	p0.d, p0/z, z0.d, #-16
+cmplt	p0.d, p0/z, z0.d, #15
+cmplt	p0.h, p0/z, z0.h, #-16
+cmplt	p0.h, p0/z, z0.h, #15
+cmplt	p0.h, p0/z, z0.h, z0.d
+cmplt	p0.s, p0/z, z0.s, #-16
+cmplt	p0.s, p0/z, z0.s, #15
+cmplt	p0.s, p0/z, z0.s, z0.d
+cmpne	p0.b, p0/z, z0.b, #-16
+cmpne	p0.b, p0/z, z0.b, #15
+cmpne	p0.b, p0/z, z0.b, z0.b
+cmpne	p0.b, p0/z, z0.b, z0.d
+cmpne	p0.d, p0/z, z0.d, #-16
+cmpne	p0.d, p0/z, z0.d, #15
+cmpne	p0.d, p0/z, z0.d, z0.d
+cmpne	p0.h, p0/z, z0.h, #-16
+cmpne	p0.h, p0/z, z0.h, #15
+cmpne	p0.h, p0/z, z0.h, z0.d
+cmpne	p0.h, p0/z, z0.h, z0.h
+cmpne	p0.s, p0/z, z0.s, #-16
+cmpne	p0.s, p0/z, z0.s, #15
+cmpne	p0.s, p0/z, z0.s, z0.d
+cmpne	p0.s, p0/z, z0.s, z0.s
+cnot	z31.b, p7/m, z31.b
+cnot	z31.d, p7/m, z31.d
+cnot	z31.h, p7/m, z31.h
+cnot	z31.s, p7/m, z31.s
+cnt	z31.b, p7/m, z31.b
+cnt	z31.d, p7/m, z31.d
+cnt	z31.h, p7/m, z31.h
+cnt	z31.s, p7/m, z31.s
+cntb	x0
+cntb	x0, #28
+cntb	x0, all, mul #16
+cntb	x0, pow2
+cntd	x0
+cntd	x0, #28
+cntd	x0, all, mul #16
+cntd	x0, pow2
+cnth	x0
+cnth	x0, #28
+cnth	x0, all, mul #16
+cnth	x0, pow2
+cntp	x0, p15, p0.b
+cntp	x0, p15, p0.d
+cntp	x0, p15, p0.h
+cntp	x0, p15, p0.s
+cntw	x0
+cntw	x0, #28
+cntw	x0, all, mul #16
+cntw	x0, pow2
+compact	z31.d, p7, z31.d
+compact	z31.s, p7, z31.s
+ctermeq	w30, wzr
+ctermeq	wzr, w30
+ctermeq	x30, xzr
+ctermeq	xzr, x30
+ctermne	w30, wzr
+ctermne	wzr, w30
+ctermne	x30, xzr
+ctermne	xzr, x30
+decb	x0
+decb	x0, #14
+decb	x0, all, mul #16
+decb	x0, pow2
+decb	x0, vl1
+decd	x0
+decd	x0, #14
+decd	x0, all, mul #16
+decd	x0, pow2
+decd	x0, vl1
+dech	x0
+dech	x0, #14
+dech	x0, all, mul #16
+dech	x0, pow2
+dech	x0, vl1
+decp	x0, p0.b
+decp	x0, p0.d
+decp	x0, p0.h
+decp	x0, p0.s
+decp	xzr, p15.b
+decp	xzr, p15.d
+decp	xzr, p15.h
+decp	xzr, p15.s
+decp	z31.d, p15.d
+decp	z31.h, p15.h
+decp	z31.s, p15.s
+decw	x0
+decw	x0, #14
+decw	x0, all, mul #16
+decw	x0, pow2
+decw	x0, vl1
+dupm	z0.d, #0xfffffffffffffff9
+dupm	z0.s, #0xfffffff9
+dupm	z23.h, #0xfff9
+dupm	z5.b, #0xf9
+eor	p0.b, p0/z, p0.b, p1.b
+eor	z0.d, z0.d, #0x6
+eor	z0.d, z0.d, #0xfffffffffffffff9
+eor	z0.d, z0.d, z0.d
+eor	z0.s, z0.s, #0x6
+eor	z0.s, z0.s, #0xfffffff9
+eor	z23.d, z13.d, z8.d
+eor	z23.h, z23.h, #0x6
+eor	z23.h, z23.h, #0xfff9
+eor	z31.b, p7/m, z31.b, z31.b
+eor	z31.d, p7/m, z31.d, z31.d
+eor	z31.h, p7/m, z31.h, z31.h
+eor	z31.s, p7/m, z31.s, z31.s
+eor	z5.b, z5.b, #0x6
+eor	z5.b, z5.b, #0xf9
+eor3	z29.d, z29.d, z30.d, z31.d
+eorbt	z0.b, z1.b, z31.b
+eorbt	z0.d, z1.d, z31.d
+eorbt	z0.h, z1.h, z31.h
+eorbt	z0.s, z1.s, z31.s
+eors	p0.b, p0/z, p0.b, p1.b
+eortb	z0.b, z1.b, z31.b
+eortb	z0.d, z1.d, z31.d
+eortb	z0.h, z1.h, z31.h
+eortb	z0.s, z1.s, z31.s
+eorv	b0, p7, z31.b
+eorv	d0, p7, z31.d
+eorv	h0, p7, z31.h
+eorv	s0, p7, z31.s
+ext	z0.b, { z1.b, z2.b }, #0
+ext	z31.b, z31.b, z0.b, #0
+ext	z31.b, z31.b, z0.b, #255
+ext	z31.b, { z30.b, z31.b }, #255
+fabd	z0.d, p7/m, z0.d, z31.d
+fabd	z0.h, p7/m, z0.h, z31.h
+fabd	z0.s, p7/m, z0.s, z31.s
+fabs	z31.d, p7/m, z31.d
+fabs	z31.h, p7/m, z31.h
+fabs	z31.s, p7/m, z31.s
+facge	p0.d, p0/z, z0.d, z1.d
+facge	p0.d, p0/z, z1.d, z0.d
+facge	p0.h, p0/z, z0.h, z1.h
+facge	p0.h, p0/z, z1.h, z0.h
+facge	p0.s, p0/z, z0.s, z1.s
+facge	p0.s, p0/z, z1.s, z0.s
+facgt	p0.d, p0/z, z0.d, z1.d
+facgt	p0.d, p0/z, z1.d, z0.d
+facgt	p0.h, p0/z, z0.h, z1.h
+facgt	p0.h, p0/z, z1.h, z0.h
+facgt	p0.s, p0/z, z0.s, z1.s
+facgt	p0.s, p0/z, z1.s, z0.s
+fadd	z0.d, p0/m, z0.d, #0.5
+fadd	z0.d, p7/m, z0.d, z31.d
+fadd	z0.d, z1.d, z31.d
+fadd	z0.h, p0/m, z0.h, #0.5
+fadd	z0.h, p7/m, z0.h, z31.h
+fadd	z0.h, z1.h, z31.h
+fadd	z0.s, p0/m, z0.s, #0.5
+fadd	z0.s, p7/m, z0.s, z31.s
+fadd	z0.s, z1.s, z31.s
+fadd	z31.d, p7/m, z31.d, #1.0
+fadd	z31.h, p7/m, z31.h, #1.0
+fadd	z31.s, p7/m, z31.s, #1.0
+fadda	d0, p7, d0, z31.d
+fadda	h0, p7, h0, z31.h
+fadda	s0, p7, s0, z31.s
+faddp	z0.h, p0/m, z0.h, z1.h
+faddp	z29.s, p3/m, z29.s, z30.s
+faddp	z31.d, p7/m, z31.d, z30.d
+faddv	d0, p7, z31.d
+faddv	h0, p7, z31.h
+faddv	s0, p7, z31.s
+fcadd	z0.d, p0/m, z0.d, z0.d, #90
+fcadd	z0.h, p0/m, z0.h, z0.h, #90
+fcadd	z0.s, p0/m, z0.s, z0.s, #90
+fcadd	z31.d, p7/m, z31.d, z31.d, #270
+fcadd	z31.h, p7/m, z31.h, z31.h, #270
+fcadd	z31.s, p7/m, z31.s, z31.s, #270
+fcmeq	p0.d, p0/z, z0.d, #0.0
+fcmeq	p0.d, p0/z, z0.d, z1.d
+fcmeq	p0.h, p0/z, z0.h, #0.0
+fcmeq	p0.h, p0/z, z0.h, z1.h
+fcmeq	p0.s, p0/z, z0.s, #0.0
+fcmeq	p0.s, p0/z, z0.s, z1.s
+fcmge	p0.d, p0/z, z0.d, #0.0
+fcmge	p0.d, p0/z, z0.d, z1.d
+fcmge	p0.d, p0/z, z1.d, z0.d
+fcmge	p0.h, p0/z, z0.h, #0.0
+fcmge	p0.h, p0/z, z0.h, z1.h
+fcmge	p0.h, p0/z, z1.h, z0.h
+fcmge	p0.s, p0/z, z0.s, #0.0
+fcmge	p0.s, p0/z, z0.s, z1.s
+fcmge	p0.s, p0/z, z1.s, z0.s
+fcmgt	p0.d, p0/z, z0.d, #0.0
+fcmgt	p0.d, p0/z, z0.d, z1.d
+fcmgt	p0.d, p0/z, z1.d, z0.d
+fcmgt	p0.h, p0/z, z0.h, #0.0
+fcmgt	p0.h, p0/z, z0.h, z1.h
+fcmgt	p0.h, p0/z, z1.h, z0.h
+fcmgt	p0.s, p0/z, z0.s, #0.0
+fcmgt	p0.s, p0/z, z0.s, z1.s
+fcmgt	p0.s, p0/z, z1.s, z0.s
+fcmla	z0.d, p0/m, z0.d, z0.d, #0
+fcmla	z0.d, p0/m, z1.d, z2.d, #90
+fcmla	z0.h, p0/m, z0.h, z0.h, #0
+fcmla	z0.h, p0/m, z1.h, z2.h, #90
+fcmla	z0.h, z0.h, z0.h[0], #0
+fcmla	z0.s, p0/m, z0.s, z0.s, #0
+fcmla	z0.s, p0/m, z1.s, z2.s, #90
+fcmla	z21.s, z10.s, z5.s[1], #90
+fcmla	z23.s, z13.s, z8.s[0], #270
+fcmla	z29.d, p7/m, z30.d, z31.d, #180
+fcmla	z29.h, p7/m, z30.h, z31.h, #180
+fcmla	z29.s, p7/m, z30.s, z31.s, #180
+fcmla	z31.d, p7/m, z31.d, z31.d, #270
+fcmla	z31.h, p7/m, z31.h, z31.h, #270
+fcmla	z31.h, z31.h, z7.h[3], #270
+fcmla	z31.s, p7/m, z31.s, z31.s, #270
+fcmle	p0.d, p0/z, z0.d, #0.0
+fcmle	p0.h, p0/z, z0.h, #0.0
+fcmle	p0.s, p0/z, z0.s, #0.0
+fcmlt	p0.d, p0/z, z0.d, #0.0
+fcmlt	p0.h, p0/z, z0.h, #0.0
+fcmlt	p0.s, p0/z, z0.s, #0.0
+fcmne	p0.d, p0/z, z0.d, #0.0
+fcmne	p0.d, p0/z, z0.d, z1.d
+fcmne	p0.h, p0/z, z0.h, #0.0
+fcmne	p0.h, p0/z, z0.h, z1.h
+fcmne	p0.s, p0/z, z0.s, #0.0
+fcmne	p0.s, p0/z, z0.s, z1.s
+fcmuo	p0.d, p0/z, z0.d, z1.d
+fcmuo	p0.h, p0/z, z0.h, z1.h
+fcmuo	p0.s, p0/z, z0.s, z1.s
+fcvt	z0.d, p0/m, z0.h
+fcvt	z0.d, p0/m, z0.s
+fcvt	z0.h, p0/m, z0.d
+fcvt	z0.h, p0/m, z0.s
+fcvt	z0.s, p0/m, z0.d
+fcvt	z0.s, p0/m, z0.h
+fcvtlt	z0.s, p0/m, z1.h
+fcvtlt	z30.d, p7/m, z31.s
+fcvtnt	z0.h, p0/m, z1.s
+fcvtnt	z30.s, p7/m, z31.d
+fcvtx	z0.s, p0/m, z0.d
+fcvtx	z30.s, p7/m, z31.d
+fcvtxnt	z0.s, p0/m, z1.d
+fcvtxnt	z30.s, p7/m, z31.d
+fcvtzs	z0.d, p0/m, z0.d
+fcvtzs	z0.d, p0/m, z0.h
+fcvtzs	z0.d, p0/m, z0.s
+fcvtzs	z0.h, p0/m, z0.h
+fcvtzs	z0.s, p0/m, z0.d
+fcvtzs	z0.s, p0/m, z0.h
+fcvtzs	z0.s, p0/m, z0.s
+fcvtzu	z0.d, p0/m, z0.d
+fcvtzu	z0.d, p0/m, z0.h
+fcvtzu	z0.d, p0/m, z0.s
+fcvtzu	z0.h, p0/m, z0.h
+fcvtzu	z0.s, p0/m, z0.d
+fcvtzu	z0.s, p0/m, z0.h
+fcvtzu	z0.s, p0/m, z0.s
+fdiv	z0.d, p7/m, z0.d, z31.d
+fdiv	z0.h, p7/m, z0.h, z31.h
+fdiv	z0.s, p7/m, z0.s, z31.s
+fdivr	z0.d, p7/m, z0.d, z31.d
+fdivr	z0.h, p7/m, z0.h, z31.h
+fdivr	z0.s, p7/m, z0.s, z31.s
+fexpa	z0.d, z31.d
+fexpa	z0.h, z31.h
+fexpa	z0.s, z31.s
+flogb	z31.d, p7/m, z31.d
+flogb	z31.h, p7/m, z31.h
+flogb	z31.s, p7/m, z31.s
+fmad	z0.d, p7/m, z1.d, z31.d
+fmad	z0.h, p7/m, z1.h, z31.h
+fmad	z0.s, p7/m, z1.s, z31.s
+fmax	z0.d, p0/m, z0.d, #0.0
+fmax	z0.d, p7/m, z0.d, z31.d
+fmax	z0.h, p0/m, z0.h, #0.0
+fmax	z0.h, p7/m, z0.h, z31.h
+fmax	z0.s, p0/m, z0.s, #0.0
+fmax	z0.s, p7/m, z0.s, z31.s
+fmax	z31.d, p7/m, z31.d, #1.0
+fmax	z31.h, p7/m, z31.h, #1.0
+fmax	z31.s, p7/m, z31.s, #1.0
+fmaxnm	z0.d, p0/m, z0.d, #0.0
+fmaxnm	z0.d, p7/m, z0.d, z31.d
+fmaxnm	z0.h, p0/m, z0.h, #0.0
+fmaxnm	z0.h, p7/m, z0.h, z31.h
+fmaxnm	z0.s, p0/m, z0.s, #0.0
+fmaxnm	z0.s, p7/m, z0.s, z31.s
+fmaxnm	z31.d, p7/m, z31.d, #1.0
+fmaxnm	z31.h, p7/m, z31.h, #1.0
+fmaxnm	z31.s, p7/m, z31.s, #1.0
+fmaxnmp	z0.h, p0/m, z0.h, z1.h
+fmaxnmp	z29.s, p3/m, z29.s, z30.s
+fmaxnmp	z31.d, p7/m, z31.d, z30.d
+fmaxnmv	d0, p7, z31.d
+fmaxnmv	h0, p7, z31.h
+fmaxnmv	s0, p7, z31.s
+fmaxp	z0.h, p0/m, z0.h, z1.h
+fmaxp	z29.s, p3/m, z29.s, z30.s
+fmaxp	z31.d, p7/m, z31.d, z30.d
+fmaxv	d0, p7, z31.d
+fmaxv	h0, p7, z31.h
+fmaxv	s0, p7, z31.s
+fmin	z0.d, p0/m, z0.d, #0.0
+fmin	z0.d, p7/m, z0.d, z31.d
+fmin	z0.h, p0/m, z0.h, #0.0
+fmin	z0.h, p7/m, z0.h, z31.h
+fmin	z0.s, p0/m, z0.s, #0.0
+fmin	z0.s, p7/m, z0.s, z31.s
+fmin	z31.d, p7/m, z31.d, #1.0
+fmin	z31.h, p7/m, z31.h, #1.0
+fmin	z31.s, p7/m, z31.s, #1.0
+fminnm	z0.d, p0/m, z0.d, #0.0
+fminnm	z0.d, p7/m, z0.d, z31.d
+fminnm	z0.h, p0/m, z0.h, #0.0
+fminnm	z0.h, p7/m, z0.h, z31.h
+fminnm	z0.s, p0/m, z0.s, #0.0
+fminnm	z0.s, p7/m, z0.s, z31.s
+fminnm	z31.d, p7/m, z31.d, #1.0
+fminnm	z31.h, p7/m, z31.h, #1.0
+fminnm	z31.s, p7/m, z31.s, #1.0
+fminnmp	z0.h, p0/m, z0.h, z1.h
+fminnmp	z29.s, p3/m, z29.s, z30.s
+fminnmp	z31.d, p7/m, z31.d, z30.d
+fminnmv	d0, p7, z31.d
+fminnmv	h0, p7, z31.h
+fminnmv	s0, p7, z31.s
+fminp	z0.h, p0/m, z0.h, z1.h
+fminp	z29.s, p3/m, z29.s, z30.s
+fminp	z31.d, p7/m, z31.d, z30.d
+fminv	d0, p7, z31.d
+fminv	h0, p7, z31.h
+fminv	s0, p7, z31.s
+fmla	z0.d, p7/m, z1.d, z31.d
+fmla	z0.d, z1.d, z7.d[1]
+fmla	z0.h, p7/m, z1.h, z31.h
+fmla	z0.h, z1.h, z7.h[7]
+fmla	z0.s, p7/m, z1.s, z31.s
+fmla	z0.s, z1.s, z7.s[3]
+fmlalb	z0.s, z1.h, z7.h[0]
+fmlalb	z29.s, z30.h, z31.h
+fmlalb	z30.s, z31.h, z7.h[7]
+fmlalt	z0.s, z1.h, z7.h[0]
+fmlalt	z29.s, z30.h, z31.h
+fmlalt	z30.s, z31.h, z7.h[7]
+fmls	z0.d, p7/m, z1.d, z31.d
+fmls	z0.d, z1.d, z7.d[1]
+fmls	z0.h, p7/m, z1.h, z31.h
+fmls	z0.h, z1.h, z7.h[7]
+fmls	z0.s, p7/m, z1.s, z31.s
+fmls	z0.s, z1.s, z7.s[3]
+fmlslb	z0.s, z1.h, z7.h[0]
+fmlslb	z29.s, z30.h, z31.h
+fmlslb	z30.s, z31.h, z7.h[7]
+fmlslt	z0.s, z1.h, z7.h[0]
+fmlslt	z29.s, z30.h, z31.h
+fmlslt	z30.s, z31.h, z7.h[7]
+fmov	z0.d, #-10.00000000
+fmov	z0.d, #0.12500000
+fmov	z0.d, p0/m, #-10.00000000
+fmov	z0.d, p0/m, #0.12500000
+fmov	z0.h, #-0.12500000
+fmov	z0.h, p0/m, #-0.12500000
+fmov	z0.s, #-0.12500000
+fmov	z0.s, p0/m, #-0.12500000
+fmsb	z0.d, p7/m, z1.d, z31.d
+fmsb	z0.h, p7/m, z1.h, z31.h
+fmsb	z0.s, p7/m, z1.s, z31.s
+fmul	z0.d, p0/m, z0.d, #0.5
+fmul	z0.d, p7/m, z0.d, z31.d
+fmul	z0.d, z0.d, z0.d[0]
+fmul	z0.d, z1.d, z31.d
+fmul	z0.h, p0/m, z0.h, #0.5
+fmul	z0.h, p7/m, z0.h, z31.h
+fmul	z0.h, z0.h, z0.h[0]
+fmul	z0.h, z1.h, z31.h
+fmul	z0.s, p0/m, z0.s, #0.5
+fmul	z0.s, p7/m, z0.s, z31.s
+fmul	z0.s, z0.s, z0.s[0]
+fmul	z0.s, z1.s, z31.s
+fmul	z31.d, p7/m, z31.d, #2.0
+fmul	z31.d, z31.d, z15.d[1]
+fmul	z31.h, p7/m, z31.h, #2.0
+fmul	z31.h, z31.h, z7.h[7]
+fmul	z31.s, p7/m, z31.s, #2.0
+fmul	z31.s, z31.s, z7.s[3]
+fmulx	z0.d, p7/m, z0.d, z31.d
+fmulx	z0.h, p7/m, z0.h, z31.h
+fmulx	z0.s, p7/m, z0.s, z31.s
+fneg	z31.d, p7/m, z31.d
+fneg	z31.h, p7/m, z31.h
+fneg	z31.s, p7/m, z31.s
+fnmad	z0.d, p7/m, z1.d, z31.d
+fnmad	z0.h, p7/m, z1.h, z31.h
+fnmad	z0.s, p7/m, z1.s, z31.s
+fnmla	z0.d, p7/m, z1.d, z31.d
+fnmla	z0.h, p7/m, z1.h, z31.h
+fnmla	z0.s, p7/m, z1.s, z31.s
+fnmls	z0.d, p7/m, z1.d, z31.d
+fnmls	z0.h, p7/m, z1.h, z31.h
+fnmls	z0.s, p7/m, z1.s, z31.s
+fnmsb	z0.d, p7/m, z1.d, z31.d
+fnmsb	z0.h, p7/m, z1.h, z31.h
+fnmsb	z0.s, p7/m, z1.s, z31.s
+frecpe	z0.d, z31.d
+frecpe	z0.h, z31.h
+frecpe	z0.s, z31.s
+frecps	z0.d, z1.d, z31.d
+frecps	z0.h, z1.h, z31.h
+frecps	z0.s, z1.s, z31.s
+frecpx	z31.d, p7/m, z31.d
+frecpx	z31.h, p7/m, z31.h
+frecpx	z31.s, p7/m, z31.s
+frinta	z31.d, p7/m, z31.d
+frinta	z31.h, p7/m, z31.h
+frinta	z31.s, p7/m, z31.s
+frinti	z31.d, p7/m, z31.d
+frinti	z31.h, p7/m, z31.h
+frinti	z31.s, p7/m, z31.s
+frintm	z31.d, p7/m, z31.d
+frintm	z31.h, p7/m, z31.h
+frintm	z31.s, p7/m, z31.s
+frintn	z31.d, p7/m, z31.d
+frintn	z31.h, p7/m, z31.h
+frintn	z31.s, p7/m, z31.s
+frintp	z31.d, p7/m, z31.d
+frintp	z31.h, p7/m, z31.h
+frintp	z31.s, p7/m, z31.s
+frintx	z31.d, p7/m, z31.d
+frintx	z31.h, p7/m, z31.h
+frintx	z31.s, p7/m, z31.s
+frintz	z31.d, p7/m, z31.d
+frintz	z31.h, p7/m, z31.h
+frintz	z31.s, p7/m, z31.s
+frsqrte	z0.d, z31.d
+frsqrte	z0.h, z31.h
+frsqrte	z0.s, z31.s
+frsqrts	z0.d, z1.d, z31.d
+frsqrts	z0.h, z1.h, z31.h
+frsqrts	z0.s, z1.s, z31.s
+fscale	z0.d, p7/m, z0.d, z31.d
+fscale	z0.h, p7/m, z0.h, z31.h
+fscale	z0.s, p7/m, z0.s, z31.s
+fsqrt	z31.d, p7/m, z31.d
+fsqrt	z31.h, p7/m, z31.h
+fsqrt	z31.s, p7/m, z31.s
+fsub	z0.d, p0/m, z0.d, #0.5
+fsub	z0.d, p7/m, z0.d, z31.d
+fsub	z0.d, z1.d, z31.d
+fsub	z0.h, p0/m, z0.h, #0.5
+fsub	z0.h, p7/m, z0.h, z31.h
+fsub	z0.h, z1.h, z31.h
+fsub	z0.s, p0/m, z0.s, #0.5
+fsub	z0.s, p7/m, z0.s, z31.s
+fsub	z0.s, z1.s, z31.s
+fsub	z31.d, p7/m, z31.d, #1.0
+fsub	z31.h, p7/m, z31.h, #1.0
+fsub	z31.s, p7/m, z31.s, #1.0
+fsubr	z0.d, p0/m, z0.d, #0.5
+fsubr	z0.d, p7/m, z0.d, z31.d
+fsubr	z0.h, p0/m, z0.h, #0.5
+fsubr	z0.h, p7/m, z0.h, z31.h
+fsubr	z0.s, p0/m, z0.s, #0.5
+fsubr	z0.s, p7/m, z0.s, z31.s
+fsubr	z31.d, p7/m, z31.d, #1.0
+fsubr	z31.h, p7/m, z31.h, #1.0
+fsubr	z31.s, p7/m, z31.s, #1.0
+ftmad	z0.d, z0.d, z31.d, #7
+ftmad	z0.h, z0.h, z31.h, #7
+ftmad	z0.s, z0.s, z31.s, #7
+ftsmul	z0.d, z1.d, z31.d
+ftsmul	z0.h, z1.h, z31.h
+ftsmul	z0.s, z1.s, z31.s
+ftssel	z0.d, z1.d, z31.d
+ftssel	z0.h, z1.h, z31.h
+ftssel	z0.s, z1.s, z31.s
+histcnt	z0.s, p0/z, z1.s, z2.s
+histcnt	z29.d, p7/z, z30.d, z31.d
+histseg	z0.b, z1.b, z31.b
+incb	x0
+incb	x0, #14
+incb	x0, all, mul #16
+incb	x0, pow2
+incb	x0, vl1
+incd	x0
+incd	x0, #14
+incd	x0, all, mul #16
+incd	x0, pow2
+incd	x0, vl1
+incd	z0.d
+incd	z0.d, all, mul #16
+inch	x0
+inch	x0, #14
+inch	x0, all, mul #16
+inch	x0, pow2
+inch	x0, vl1
+inch	z0.h
+inch	z0.h, all, mul #16
+incp	x0, p0.b
+incp	x0, p0.d
+incp	x0, p0.h
+incp	x0, p0.s
+incp	xzr, p15.b
+incp	xzr, p15.d
+incp	xzr, p15.h
+incp	xzr, p15.s
+incp	z31.d, p15.d
+incp	z31.h, p15.h
+incp	z31.s, p15.s
+incw	x0
+incw	x0, #14
+incw	x0, all, mul #16
+incw	x0, pow2
+incw	x0, vl1
+incw	z0.s
+incw	z0.s, all, mul #16
+index	z0.b, #0, #0
+index	z0.d, #0, #0
+index	z0.h, #0, #0
+index	z0.h, w0, w0
+index	z0.s, #0, #0
+index	z21.b, w10, w21
+index	z21.d, x10, x21
+index	z21.s, w10, w21
+index	z23.b, #13, w8
+index	z23.b, w13, #8
+index	z23.d, #13, x8
+index	z23.d, x13, #8
+index	z23.h, #13, w8
+index	z23.h, w13, #8
+index	z23.s, #13, w8
+index	z23.s, w13, #8
+index	z31.b, #-1, #-1
+index	z31.b, #-1, wzr
+index	z31.b, wzr, #-1
+index	z31.b, wzr, wzr
+index	z31.d, #-1, #-1
+index	z31.d, #-1, xzr
+index	z31.d, xzr, #-1
+index	z31.d, xzr, xzr
+index	z31.h, #-1, #-1
+index	z31.h, #-1, wzr
+index	z31.h, wzr, #-1
+index	z31.h, wzr, wzr
+index	z31.s, #-1, #-1
+index	z31.s, #-1, wzr
+index	z31.s, wzr, #-1
+index	z31.s, wzr, wzr
+insr	z0.b, w0
+insr	z0.d, x0
+insr	z0.h, w0
+insr	z0.s, w0
+insr	z31.b, b31
+insr	z31.b, wzr
+insr	z31.d, d31
+insr	z31.d, xzr
+insr	z31.h, h31
+insr	z31.h, wzr
+insr	z31.s, s31
+insr	z31.s, wzr
+lasta	b0, p7, z31.b
+lasta	d0, p7, z31.d
+lasta	h0, p7, z31.h
+lasta	s0, p7, z31.s
+lasta	w0, p7, z31.b
+lasta	w0, p7, z31.h
+lasta	w0, p7, z31.s
+lasta	x0, p7, z31.d
+lastb	b0, p7, z31.b
+lastb	d0, p7, z31.d
+lastb	h0, p7, z31.h
+lastb	s0, p7, z31.s
+lastb	w0, p7, z31.b
+lastb	w0, p7, z31.h
+lastb	w0, p7, z31.s
+lastb	x0, p7, z31.d
+ld1b	{ z0.b }, p0/z, [sp, x0]
+ld1b	{ z0.b }, p0/z, [x0, x0]
+ld1b	{ z0.b }, p0/z, [x0]
+ld1b	{ z0.d }, p0/z, [x0]
+ld1b	{ z0.d }, p0/z, [z0.d]
+ld1b	{ z0.h }, p0/z, [x0]
+ld1b	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+ld1b	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+ld1b	{ z0.s }, p0/z, [x0]
+ld1b	{ z0.s }, p0/z, [z0.s]
+ld1b	{ z21.b }, p5/z, [x10, #5, mul vl]
+ld1b	{ z21.d }, p5/z, [x10, #5, mul vl]
+ld1b	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ld1b	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ld1b	{ z21.h }, p5/z, [x10, #5, mul vl]
+ld1b	{ z21.s }, p5/z, [x10, #5, mul vl]
+ld1b	{ z21.s }, p5/z, [x10, x21]
+ld1b	{ z23.d }, p3/z, [x13, x8]
+ld1b	{ z31.b }, p7/z, [sp, #-1, mul vl]
+ld1b	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ld1b	{ z31.d }, p7/z, [sp, z31.d]
+ld1b	{ z31.d }, p7/z, [z31.d, #31]
+ld1b	{ z31.h }, p7/z, [sp, #-1, mul vl]
+ld1b	{ z31.s }, p7/z, [sp, #-1, mul vl]
+ld1b	{ z31.s }, p7/z, [z31.s, #31]
+ld1b	{ z5.h }, p3/z, [x17, x16]
+ld1d	{ z0.d }, p0/z, [x0, z0.d, sxtw #3]
+ld1d	{ z0.d }, p0/z, [x0, z0.d, uxtw #3]
+ld1d	{ z0.d }, p0/z, [x0]
+ld1d	{ z0.d }, p0/z, [z0.d]
+ld1d	{ z21.d }, p5/z, [x10, #5, mul vl]
+ld1d	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ld1d	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ld1d	{ z23.d }, p3/z, [sp, x8, lsl #3]
+ld1d	{ z23.d }, p3/z, [x13, x8, lsl #3]
+ld1d	{ z23.d }, p3/z, [x13, z8.d, lsl #3]
+ld1d	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ld1d	{ z31.d }, p7/z, [sp, z31.d]
+ld1d	{ z31.d }, p7/z, [z31.d, #248]
+ld1h	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+ld1h	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+ld1h	{ z0.d }, p0/z, [x0]
+ld1h	{ z0.d }, p0/z, [z0.d]
+ld1h	{ z0.h }, p0/z, [x0]
+ld1h	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+ld1h	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+ld1h	{ z0.s }, p0/z, [x0]
+ld1h	{ z0.s }, p0/z, [z0.s]
+ld1h	{ z21.d }, p5/z, [x10, #5, mul vl]
+ld1h	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ld1h	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ld1h	{ z21.h }, p5/z, [x10, #5, mul vl]
+ld1h	{ z21.s }, p5/z, [x10, #5, mul vl]
+ld1h	{ z21.s }, p5/z, [x10, x21, lsl #1]
+ld1h	{ z23.d }, p3/z, [x13, x8, lsl #1]
+ld1h	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+ld1h	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ld1h	{ z31.d }, p7/z, [sp, z31.d]
+ld1h	{ z31.d }, p7/z, [z31.d, #62]
+ld1h	{ z31.h }, p7/z, [sp, #-1, mul vl]
+ld1h	{ z31.s }, p7/z, [sp, #-1, mul vl]
+ld1h	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+ld1h	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+ld1h	{ z31.s }, p7/z, [z31.s, #62]
+ld1h	{ z5.h }, p3/z, [sp, x16, lsl #1]
+ld1h	{ z5.h }, p3/z, [x17, x16, lsl #1]
+ld1rb	{ z0.b }, p0/z, [x0]
+ld1rb	{ z0.d }, p0/z, [x0]
+ld1rb	{ z0.h }, p0/z, [x0]
+ld1rb	{ z0.s }, p0/z, [x0]
+ld1rb	{ z31.b }, p7/z, [sp, #63]
+ld1rb	{ z31.d }, p7/z, [sp, #63]
+ld1rb	{ z31.h }, p7/z, [sp, #63]
+ld1rb	{ z31.s }, p7/z, [sp, #63]
+ld1rd	{ z0.d }, p0/z, [x0]
+ld1rd	{ z31.d }, p7/z, [sp, #504]
+ld1rh	{ z0.d }, p0/z, [x0]
+ld1rh	{ z0.h }, p0/z, [x0]
+ld1rh	{ z0.s }, p0/z, [x0]
+ld1rh	{ z31.d }, p7/z, [sp, #126]
+ld1rh	{ z31.h }, p7/z, [sp, #126]
+ld1rh	{ z31.s }, p7/z, [sp, #126]
+ld1rqb	{ z0.b }, p0/z, [x0, x0]
+ld1rqb	{ z0.b }, p0/z, [x0]
+ld1rqb	{ z21.b }, p5/z, [x10, #112]
+ld1rqb	{ z23.b }, p3/z, [x13, #-128]
+ld1rqb	{ z31.b }, p7/z, [sp, #-16]
+ld1rqd	{ z0.d }, p0/z, [x0, x0, lsl #3]
+ld1rqd	{ z0.d }, p0/z, [x0]
+ld1rqd	{ z23.d }, p3/z, [x13, #-128]
+ld1rqd	{ z23.d }, p3/z, [x13, #112]
+ld1rqd	{ z31.d }, p7/z, [sp, #-16]
+ld1rqh	{ z0.h }, p0/z, [x0, x0, lsl #1]
+ld1rqh	{ z0.h }, p0/z, [x0]
+ld1rqh	{ z23.h }, p3/z, [x13, #-128]
+ld1rqh	{ z23.h }, p3/z, [x13, #112]
+ld1rqh	{ z31.h }, p7/z, [sp, #-16]
+ld1rqw	{ z0.s }, p0/z, [x0, x0, lsl #2]
+ld1rqw	{ z0.s }, p0/z, [x0]
+ld1rqw	{ z23.s }, p3/z, [x13, #-128]
+ld1rqw	{ z23.s }, p3/z, [x13, #112]
+ld1rqw	{ z31.s }, p7/z, [sp, #-16]
+ld1rsb	{ z0.d }, p0/z, [x0]
+ld1rsb	{ z0.h }, p0/z, [x0]
+ld1rsb	{ z0.s }, p0/z, [x0]
+ld1rsb	{ z31.d }, p7/z, [sp, #63]
+ld1rsb	{ z31.h }, p7/z, [sp, #63]
+ld1rsb	{ z31.s }, p7/z, [sp, #63]
+ld1rsh	{ z0.d }, p0/z, [x0]
+ld1rsh	{ z0.s }, p0/z, [x0]
+ld1rsh	{ z31.d }, p7/z, [sp, #126]
+ld1rsh	{ z31.s }, p7/z, [sp, #126]
+ld1rsw	{ z0.d }, p0/z, [x0]
+ld1rsw	{ z31.d }, p7/z, [sp, #252]
+ld1rw	{ z0.d }, p0/z, [x0]
+ld1rw	{ z0.s }, p0/z, [x0]
+ld1rw	{ z31.d }, p7/z, [sp, #252]
+ld1rw	{ z31.s }, p7/z, [sp, #252]
+ld1sb	{ z0.d }, p0/z, [x0]
+ld1sb	{ z0.d }, p0/z, [z0.d]
+ld1sb	{ z0.h }, p0/z, [sp, x0]
+ld1sb	{ z0.h }, p0/z, [x0, x0]
+ld1sb	{ z0.h }, p0/z, [x0]
+ld1sb	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+ld1sb	{ z0.s }, p0/z, [x0]
+ld1sb	{ z0.s }, p0/z, [z0.s]
+ld1sb	{ z21.d }, p5/z, [x10, #5, mul vl]
+ld1sb	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ld1sb	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ld1sb	{ z21.h }, p5/z, [x10, #5, mul vl]
+ld1sb	{ z21.s }, p5/z, [x10, #5, mul vl]
+ld1sb	{ z21.s }, p5/z, [x10, x21]
+ld1sb	{ z23.d }, p3/z, [x13, x8]
+ld1sb	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ld1sb	{ z31.d }, p7/z, [sp, z31.d]
+ld1sb	{ z31.d }, p7/z, [z31.d, #31]
+ld1sb	{ z31.h }, p7/z, [sp, #-1, mul vl]
+ld1sb	{ z31.s }, p7/z, [sp, #-1, mul vl]
+ld1sb	{ z31.s }, p7/z, [z31.s, #31]
+ld1sh	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+ld1sh	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+ld1sh	{ z0.d }, p0/z, [x0]
+ld1sh	{ z0.d }, p0/z, [z0.d]
+ld1sh	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+ld1sh	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+ld1sh	{ z0.s }, p0/z, [x0]
+ld1sh	{ z0.s }, p0/z, [z0.s]
+ld1sh	{ z21.d }, p5/z, [x10, #5, mul vl]
+ld1sh	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ld1sh	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ld1sh	{ z21.s }, p5/z, [sp, x21, lsl #1]
+ld1sh	{ z21.s }, p5/z, [x10, #5, mul vl]
+ld1sh	{ z21.s }, p5/z, [x10, x21, lsl #1]
+ld1sh	{ z23.d }, p3/z, [x13, x8, lsl #1]
+ld1sh	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+ld1sh	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ld1sh	{ z31.d }, p7/z, [sp, z31.d]
+ld1sh	{ z31.d }, p7/z, [z31.d, #62]
+ld1sh	{ z31.s }, p7/z, [sp, #-1, mul vl]
+ld1sh	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+ld1sh	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+ld1sh	{ z31.s }, p7/z, [z31.s, #62]
+ld1sw	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+ld1sw	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+ld1sw	{ z0.d }, p0/z, [x0]
+ld1sw	{ z0.d }, p0/z, [z0.d]
+ld1sw	{ z21.d }, p5/z, [x10, #5, mul vl]
+ld1sw	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ld1sw	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ld1sw	{ z23.d }, p3/z, [sp, x8, lsl #2]
+ld1sw	{ z23.d }, p3/z, [x13, x8, lsl #2]
+ld1sw	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+ld1sw	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ld1sw	{ z31.d }, p7/z, [sp, z31.d]
+ld1sw	{ z31.d }, p7/z, [z31.d, #124]
+ld1w	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+ld1w	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+ld1w	{ z0.d }, p0/z, [x0]
+ld1w	{ z0.d }, p0/z, [z0.d]
+ld1w	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+ld1w	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+ld1w	{ z0.s }, p0/z, [x0]
+ld1w	{ z0.s }, p0/z, [z0.s]
+ld1w	{ z21.d }, p5/z, [x10, #5, mul vl]
+ld1w	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ld1w	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ld1w	{ z21.s }, p5/z, [sp, x21, lsl #2]
+ld1w	{ z21.s }, p5/z, [x10, #5, mul vl]
+ld1w	{ z21.s }, p5/z, [x10, x21, lsl #2]
+ld1w	{ z23.d }, p3/z, [x13, x8, lsl #2]
+ld1w	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+ld1w	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ld1w	{ z31.d }, p7/z, [sp, z31.d]
+ld1w	{ z31.d }, p7/z, [z31.d, #124]
+ld1w	{ z31.s }, p7/z, [sp, #-1, mul vl]
+ld1w	{ z31.s }, p7/z, [sp, z31.s, sxtw #2]
+ld1w	{ z31.s }, p7/z, [sp, z31.s, uxtw #2]
+ld1w	{ z31.s }, p7/z, [z31.s, #124]
+ld2b	{ z0.b, z1.b }, p0/z, [x0, x0]
+ld2b	{ z0.b, z1.b }, p0/z, [x0]
+ld2b	{ z21.b, z22.b }, p5/z, [x10, #10, mul vl]
+ld2b	{ z23.b, z24.b }, p3/z, [x13, #-16, mul vl]
+ld2b	{ z5.b, z6.b }, p3/z, [x17, x16]
+ld2d	{ z0.d, z1.d }, p0/z, [x0, x0, lsl #3]
+ld2d	{ z0.d, z1.d }, p0/z, [x0]
+ld2d	{ z21.d, z22.d }, p5/z, [x10, #10, mul vl]
+ld2d	{ z23.d, z24.d }, p3/z, [x13, #-16, mul vl]
+ld2d	{ z5.d, z6.d }, p3/z, [x17, x16, lsl #3]
+ld2h	{ z0.h, z1.h }, p0/z, [x0, x0, lsl #1]
+ld2h	{ z0.h, z1.h }, p0/z, [x0]
+ld2h	{ z21.h, z22.h }, p5/z, [x10, #10, mul vl]
+ld2h	{ z23.h, z24.h }, p3/z, [x13, #-16, mul vl]
+ld2h	{ z5.h, z6.h }, p3/z, [x17, x16, lsl #1]
+ld2w	{ z0.s, z1.s }, p0/z, [x0, x0, lsl #2]
+ld2w	{ z0.s, z1.s }, p0/z, [x0]
+ld2w	{ z21.s, z22.s }, p5/z, [x10, #10, mul vl]
+ld2w	{ z23.s, z24.s }, p3/z, [x13, #-16, mul vl]
+ld2w	{ z5.s, z6.s }, p3/z, [x17, x16, lsl #2]
+ld3b	{ z0.b, z1.b, z2.b }, p0/z, [x0, x0]
+ld3b	{ z0.b, z1.b, z2.b }, p0/z, [x0]
+ld3b	{ z21.b, z22.b, z23.b }, p5/z, [x10, #15, mul vl]
+ld3b	{ z23.b, z24.b, z25.b }, p3/z, [x13, #-24, mul vl]
+ld3b	{ z5.b, z6.b, z7.b }, p3/z, [x17, x16]
+ld3d	{ z0.d, z1.d, z2.d }, p0/z, [x0, x0, lsl #3]
+ld3d	{ z0.d, z1.d, z2.d }, p0/z, [x0]
+ld3d	{ z21.d, z22.d, z23.d }, p5/z, [x10, #15, mul vl]
+ld3d	{ z23.d, z24.d, z25.d }, p3/z, [x13, #-24, mul vl]
+ld3d	{ z5.d, z6.d, z7.d }, p3/z, [x17, x16, lsl #3]
+ld3h	{ z0.h, z1.h, z2.h }, p0/z, [x0, x0, lsl #1]
+ld3h	{ z0.h, z1.h, z2.h }, p0/z, [x0]
+ld3h	{ z21.h, z22.h, z23.h }, p5/z, [x10, #15, mul vl]
+ld3h	{ z23.h, z24.h, z25.h }, p3/z, [x13, #-24, mul vl]
+ld3h	{ z5.h, z6.h, z7.h }, p3/z, [x17, x16, lsl #1]
+ld3w	{ z0.s, z1.s, z2.s }, p0/z, [x0, x0, lsl #2]
+ld3w	{ z0.s, z1.s, z2.s }, p0/z, [x0]
+ld3w	{ z21.s, z22.s, z23.s }, p5/z, [x10, #15, mul vl]
+ld3w	{ z23.s, z24.s, z25.s }, p3/z, [x13, #-24, mul vl]
+ld3w	{ z5.s, z6.s, z7.s }, p3/z, [x17, x16, lsl #2]
+ld4b	{ z0.b, z1.b, z2.b, z3.b }, p0/z, [x0, x0]
+ld4b	{ z0.b, z1.b, z2.b, z3.b }, p0/z, [x0]
+ld4b	{ z21.b, z22.b, z23.b, z24.b }, p5/z, [x10, #20, mul vl]
+ld4b	{ z23.b, z24.b, z25.b, z26.b }, p3/z, [x13, #-32, mul vl]
+ld4b	{ z5.b, z6.b, z7.b, z8.b }, p3/z, [x17, x16]
+ld4d	{ z0.d, z1.d, z2.d, z3.d }, p0/z, [x0, x0, lsl #3]
+ld4d	{ z0.d, z1.d, z2.d, z3.d }, p0/z, [x0]
+ld4d	{ z21.d, z22.d, z23.d, z24.d }, p5/z, [x10, #20, mul vl]
+ld4d	{ z23.d, z24.d, z25.d, z26.d }, p3/z, [x13, #-32, mul vl]
+ld4d	{ z5.d, z6.d, z7.d, z8.d }, p3/z, [x17, x16, lsl #3]
+ld4h	{ z0.h, z1.h, z2.h, z3.h }, p0/z, [x0, x0, lsl #1]
+ld4h	{ z0.h, z1.h, z2.h, z3.h }, p0/z, [x0]
+ld4h	{ z21.h, z22.h, z23.h, z24.h }, p5/z, [x10, #20, mul vl]
+ld4h	{ z23.h, z24.h, z25.h, z26.h }, p3/z, [x13, #-32, mul vl]
+ld4h	{ z5.h, z6.h, z7.h, z8.h }, p3/z, [x17, x16, lsl #1]
+ld4w	{ z0.s, z1.s, z2.s, z3.s }, p0/z, [x0, x0, lsl #2]
+ld4w	{ z0.s, z1.s, z2.s, z3.s }, p0/z, [x0]
+ld4w	{ z21.s, z22.s, z23.s, z24.s }, p5/z, [x10, #20, mul vl]
+ld4w	{ z23.s, z24.s, z25.s, z26.s }, p3/z, [x13, #-32, mul vl]
+ld4w	{ z5.s, z6.s, z7.s, z8.s }, p3/z, [x17, x16, lsl #2]
+ldff1b	{ z0.d }, p0/z, [x0, x0]
+ldff1b	{ z0.d }, p0/z, [z0.d]
+ldff1b	{ z0.h }, p0/z, [x0, x0]
+ldff1b	{ z0.s }, p0/z, [x0, x0]
+ldff1b	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+ldff1b	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+ldff1b	{ z0.s }, p0/z, [z0.s]
+ldff1b	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ldff1b	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ldff1b	{ z31.b }, p7/z, [sp]
+ldff1b	{ z31.d }, p7/z, [sp, z31.d]
+ldff1b	{ z31.d }, p7/z, [sp]
+ldff1b	{ z31.d }, p7/z, [z31.d, #31]
+ldff1b	{ z31.h }, p7/z, [sp]
+ldff1b	{ z31.s }, p7/z, [sp]
+ldff1b	{ z31.s }, p7/z, [z31.s, #31]
+ldff1d	{ z0.d }, p0/z, [x0, x0, lsl #3]
+ldff1d	{ z0.d }, p0/z, [x0, z0.d, sxtw #3]
+ldff1d	{ z0.d }, p0/z, [x0, z0.d, uxtw #3]
+ldff1d	{ z0.d }, p0/z, [z0.d]
+ldff1d	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ldff1d	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ldff1d	{ z23.d }, p3/z, [x13, z8.d, lsl #3]
+ldff1d	{ z31.d }, p7/z, [sp, z31.d]
+ldff1d	{ z31.d }, p7/z, [sp]
+ldff1d	{ z31.d }, p7/z, [z31.d, #248]
+ldff1h	{ z0.d }, p0/z, [x0, x0, lsl #1]
+ldff1h	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+ldff1h	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+ldff1h	{ z0.d }, p0/z, [z0.d]
+ldff1h	{ z0.h }, p0/z, [x0, x0, lsl #1]
+ldff1h	{ z0.s }, p0/z, [x0, x0, lsl #1]
+ldff1h	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+ldff1h	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+ldff1h	{ z0.s }, p0/z, [z0.s]
+ldff1h	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ldff1h	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ldff1h	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+ldff1h	{ z31.d }, p7/z, [sp, z31.d]
+ldff1h	{ z31.d }, p7/z, [sp]
+ldff1h	{ z31.d }, p7/z, [z31.d, #62]
+ldff1h	{ z31.h }, p7/z, [sp]
+ldff1h	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+ldff1h	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+ldff1h	{ z31.s }, p7/z, [sp]
+ldff1h	{ z31.s }, p7/z, [z31.s, #62]
+ldff1sb	{ z0.d }, p0/z, [x0, x0]
+ldff1sb	{ z0.d }, p0/z, [z0.d]
+ldff1sb	{ z0.h }, p0/z, [x0, x0]
+ldff1sb	{ z0.s }, p0/z, [x0, x0]
+ldff1sb	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+ldff1sb	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+ldff1sb	{ z0.s }, p0/z, [z0.s]
+ldff1sb	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ldff1sb	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ldff1sb	{ z31.d }, p7/z, [sp, z31.d]
+ldff1sb	{ z31.d }, p7/z, [sp]
+ldff1sb	{ z31.d }, p7/z, [z31.d, #31]
+ldff1sb	{ z31.h }, p7/z, [sp]
+ldff1sb	{ z31.s }, p7/z, [sp]
+ldff1sb	{ z31.s }, p7/z, [z31.s, #31]
+ldff1sh	{ z0.d }, p0/z, [x0, x0, lsl #1]
+ldff1sh	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+ldff1sh	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+ldff1sh	{ z0.d }, p0/z, [z0.d]
+ldff1sh	{ z0.s }, p0/z, [x0, x0, lsl #1]
+ldff1sh	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+ldff1sh	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+ldff1sh	{ z0.s }, p0/z, [z0.s]
+ldff1sh	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ldff1sh	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ldff1sh	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+ldff1sh	{ z31.d }, p7/z, [sp, z31.d]
+ldff1sh	{ z31.d }, p7/z, [sp]
+ldff1sh	{ z31.d }, p7/z, [z31.d, #62]
+ldff1sh	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+ldff1sh	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+ldff1sh	{ z31.s }, p7/z, [sp]
+ldff1sh	{ z31.s }, p7/z, [z31.s, #62]
+ldff1sw	{ z0.d }, p0/z, [x0, x0, lsl #2]
+ldff1sw	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+ldff1sw	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+ldff1sw	{ z0.d }, p0/z, [z0.d]
+ldff1sw	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ldff1sw	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ldff1sw	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+ldff1sw	{ z31.d }, p7/z, [sp, z31.d]
+ldff1sw	{ z31.d }, p7/z, [sp]
+ldff1sw	{ z31.d }, p7/z, [z31.d, #124]
+ldff1w	{ z0.d }, p0/z, [x0, x0, lsl #2]
+ldff1w	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+ldff1w	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+ldff1w	{ z0.d }, p0/z, [z0.d]
+ldff1w	{ z0.s }, p0/z, [x0, x0, lsl #2]
+ldff1w	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+ldff1w	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+ldff1w	{ z0.s }, p0/z, [z0.s]
+ldff1w	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+ldff1w	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+ldff1w	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+ldff1w	{ z31.d }, p7/z, [sp, z31.d]
+ldff1w	{ z31.d }, p7/z, [sp]
+ldff1w	{ z31.d }, p7/z, [z31.d, #124]
+ldff1w	{ z31.s }, p7/z, [sp, z31.s, sxtw #2]
+ldff1w	{ z31.s }, p7/z, [sp, z31.s, uxtw #2]
+ldff1w	{ z31.s }, p7/z, [sp]
+ldff1w	{ z31.s }, p7/z, [z31.s, #124]
+ldnf1b	{ z0.b }, p0/z, [x0]
+ldnf1b	{ z0.d }, p0/z, [x0]
+ldnf1b	{ z0.h }, p0/z, [x0]
+ldnf1b	{ z0.s }, p0/z, [x0]
+ldnf1b	{ z21.b }, p5/z, [x10, #5, mul vl]
+ldnf1b	{ z21.d }, p5/z, [x10, #5, mul vl]
+ldnf1b	{ z21.h }, p5/z, [x10, #5, mul vl]
+ldnf1b	{ z21.s }, p5/z, [x10, #5, mul vl]
+ldnf1b	{ z31.b }, p7/z, [sp, #-1, mul vl]
+ldnf1b	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ldnf1b	{ z31.h }, p7/z, [sp, #-1, mul vl]
+ldnf1b	{ z31.s }, p7/z, [sp, #-1, mul vl]
+ldnf1d	{ z0.d }, p0/z, [x0]
+ldnf1d	{ z21.d }, p5/z, [x10, #5, mul vl]
+ldnf1d	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ldnf1h	{ z0.d }, p0/z, [x0]
+ldnf1h	{ z0.h }, p0/z, [x0]
+ldnf1h	{ z0.s }, p0/z, [x0]
+ldnf1h	{ z21.d }, p5/z, [x10, #5, mul vl]
+ldnf1h	{ z21.h }, p5/z, [x10, #5, mul vl]
+ldnf1h	{ z21.s }, p5/z, [x10, #5, mul vl]
+ldnf1h	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ldnf1h	{ z31.h }, p7/z, [sp, #-1, mul vl]
+ldnf1h	{ z31.s }, p7/z, [sp, #-1, mul vl]
+ldnf1sb	{ z0.d }, p0/z, [x0]
+ldnf1sb	{ z0.h }, p0/z, [x0]
+ldnf1sb	{ z0.s }, p0/z, [x0]
+ldnf1sb	{ z21.d }, p5/z, [x10, #5, mul vl]
+ldnf1sb	{ z21.h }, p5/z, [x10, #5, mul vl]
+ldnf1sb	{ z21.s }, p5/z, [x10, #5, mul vl]
+ldnf1sb	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ldnf1sb	{ z31.h }, p7/z, [sp, #-1, mul vl]
+ldnf1sb	{ z31.s }, p7/z, [sp, #-1, mul vl]
+ldnf1sh	{ z0.d }, p0/z, [x0]
+ldnf1sh	{ z0.s }, p0/z, [x0]
+ldnf1sh	{ z21.d }, p5/z, [x10, #5, mul vl]
+ldnf1sh	{ z21.s }, p5/z, [x10, #5, mul vl]
+ldnf1sh	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ldnf1sh	{ z31.s }, p7/z, [sp, #-1, mul vl]
+ldnf1sw	{ z0.d }, p0/z, [x0]
+ldnf1sw	{ z21.d }, p5/z, [x10, #5, mul vl]
+ldnf1sw	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ldnf1w	{ z0.d }, p0/z, [x0]
+ldnf1w	{ z0.s }, p0/z, [x0]
+ldnf1w	{ z21.d }, p5/z, [x10, #5, mul vl]
+ldnf1w	{ z21.s }, p5/z, [x10, #5, mul vl]
+ldnf1w	{ z31.d }, p7/z, [sp, #-1, mul vl]
+ldnf1w	{ z31.s }, p7/z, [sp, #-1, mul vl]
+ldnt1b	{ z0.b }, p0/z, [x0, x0]
+ldnt1b	{ z0.b }, p0/z, [x0]
+ldnt1b	{ z0.d }, p0/z, [z1.d]
+ldnt1b	{ z0.s }, p0/z, [z1.s]
+ldnt1b	{ z21.b }, p5/z, [x10, #7, mul vl]
+ldnt1b	{ z23.b }, p3/z, [x13, #-8, mul vl]
+ldnt1b	{ z31.d }, p7/z, [z31.d, x0]
+ldnt1b	{ z31.d }, p7/z, [z31.d]
+ldnt1b	{ z31.s }, p7/z, [z31.s, x0]
+ldnt1b	{ z31.s }, p7/z, [z31.s]
+ldnt1d	{ z0.d }, p0/z, [x0, x0, lsl #3]
+ldnt1d	{ z0.d }, p0/z, [x0]
+ldnt1d	{ z0.d }, p0/z, [z1.d]
+ldnt1d	{ z21.d }, p5/z, [x10, #7, mul vl]
+ldnt1d	{ z23.d }, p3/z, [x13, #-8, mul vl]
+ldnt1d	{ z31.d }, p7/z, [z31.d, x0]
+ldnt1d	{ z31.d }, p7/z, [z31.d]
+ldnt1h	{ z0.d }, p0/z, [z1.d]
+ldnt1h	{ z0.h }, p0/z, [x0, x0, lsl #1]
+ldnt1h	{ z0.h }, p0/z, [x0]
+ldnt1h	{ z0.s }, p0/z, [z1.s]
+ldnt1h	{ z21.h }, p5/z, [x10, #7, mul vl]
+ldnt1h	{ z23.h }, p3/z, [x13, #-8, mul vl]
+ldnt1h	{ z31.d }, p7/z, [z31.d, x0]
+ldnt1h	{ z31.d }, p7/z, [z31.d]
+ldnt1h	{ z31.s }, p7/z, [z31.s, x0]
+ldnt1h	{ z31.s }, p7/z, [z31.s]
+ldnt1sb	{ z0.d }, p0/z, [z1.d]
+ldnt1sb	{ z0.s }, p0/z, [z1.s]
+ldnt1sb	{ z31.d }, p7/z, [z31.d, x0]
+ldnt1sb	{ z31.d }, p7/z, [z31.d]
+ldnt1sb	{ z31.s }, p7/z, [z31.s, x0]
+ldnt1sb	{ z31.s }, p7/z, [z31.s]
+ldnt1sh	{ z0.d }, p0/z, [z1.d]
+ldnt1sh	{ z0.s }, p0/z, [z1.s]
+ldnt1sh	{ z31.d }, p7/z, [z31.d, x0]
+ldnt1sh	{ z31.d }, p7/z, [z31.d]
+ldnt1sh	{ z31.s }, p7/z, [z31.s, x0]
+ldnt1sh	{ z31.s }, p7/z, [z31.s]
+ldnt1sw	{ z0.d }, p0/z, [z1.d]
+ldnt1sw	{ z31.d }, p7/z, [z31.d, x0]
+ldnt1sw	{ z31.d }, p7/z, [z31.d]
+ldnt1w	{ z0.d }, p0/z, [z1.d]
+ldnt1w	{ z0.s }, p0/z, [x0, x0, lsl #2]
+ldnt1w	{ z0.s }, p0/z, [x0]
+ldnt1w	{ z0.s }, p0/z, [z1.s]
+ldnt1w	{ z21.s }, p5/z, [x10, #7, mul vl]
+ldnt1w	{ z23.s }, p3/z, [x13, #-8, mul vl]
+ldnt1w	{ z31.d }, p7/z, [z31.d, x0]
+ldnt1w	{ z31.d }, p7/z, [z31.d]
+ldnt1w	{ z31.s }, p7/z, [z31.s, x0]
+ldnt1w	{ z31.s }, p7/z, [z31.s]
+ldr	p0, [x0]
+ldr	p5, [x10, #255, mul vl]
+ldr	p7, [x13, #-256, mul vl]
+ldr	z0, [x0]
+ldr	z23, [x13, #255, mul vl]
+ldr	z31, [sp, #-256, mul vl]
+lsl	z0.b, p0/m, z0.b, #0
+lsl	z0.b, p0/m, z0.b, z0.b
+lsl	z0.b, p0/m, z0.b, z1.d
+lsl	z0.b, z0.b, #0
+lsl	z0.b, z1.b, z2.d
+lsl	z0.d, p0/m, z0.d, #0
+lsl	z0.d, p0/m, z0.d, z0.d
+lsl	z0.d, z0.d, #0
+lsl	z0.h, p0/m, z0.h, #0
+lsl	z0.h, p0/m, z0.h, z0.h
+lsl	z0.h, p0/m, z0.h, z1.d
+lsl	z0.h, z0.h, #0
+lsl	z0.h, z1.h, z2.d
+lsl	z0.s, p0/m, z0.s, #0
+lsl	z0.s, p0/m, z0.s, z0.s
+lsl	z0.s, p0/m, z0.s, z1.d
+lsl	z0.s, z0.s, #0
+lsl	z0.s, z1.s, z2.d
+lsl	z31.b, p0/m, z31.b, #7
+lsl	z31.b, z31.b, #7
+lsl	z31.d, p0/m, z31.d, #63
+lsl	z31.d, z31.d, #63
+lsl	z31.h, p0/m, z31.h, #15
+lsl	z31.h, z31.h, #15
+lsl	z31.s, p0/m, z31.s, #31
+lsl	z31.s, z31.s, #31
+lslr	z0.b, p0/m, z0.b, z0.b
+lslr	z0.d, p0/m, z0.d, z0.d
+lslr	z0.h, p0/m, z0.h, z0.h
+lslr	z0.s, p0/m, z0.s, z0.s
+lsr	z0.b, p0/m, z0.b, #1
+lsr	z0.b, p0/m, z0.b, z0.b
+lsr	z0.b, p0/m, z0.b, z1.d
+lsr	z0.b, z0.b, #1
+lsr	z0.b, z1.b, z2.d
+lsr	z0.d, p0/m, z0.d, #1
+lsr	z0.d, p0/m, z0.d, z0.d
+lsr	z0.d, z0.d, #1
+lsr	z0.h, p0/m, z0.h, #1
+lsr	z0.h, p0/m, z0.h, z0.h
+lsr	z0.h, p0/m, z0.h, z1.d
+lsr	z0.h, z0.h, #1
+lsr	z0.h, z1.h, z2.d
+lsr	z0.s, p0/m, z0.s, #1
+lsr	z0.s, p0/m, z0.s, z0.s
+lsr	z0.s, p0/m, z0.s, z1.d
+lsr	z0.s, z0.s, #1
+lsr	z0.s, z1.s, z2.d
+lsr	z31.b, p0/m, z31.b, #8
+lsr	z31.b, z31.b, #8
+lsr	z31.d, p0/m, z31.d, #64
+lsr	z31.d, z31.d, #64
+lsr	z31.h, p0/m, z31.h, #16
+lsr	z31.h, z31.h, #16
+lsr	z31.s, p0/m, z31.s, #32
+lsr	z31.s, z31.s, #32
+lsrr	z0.b, p0/m, z0.b, z0.b
+lsrr	z0.d, p0/m, z0.d, z0.d
+lsrr	z0.h, p0/m, z0.h, z0.h
+lsrr	z0.s, p0/m, z0.s, z0.s
+mad	z0.b, p7/m, z1.b, z31.b
+mad	z0.d, p7/m, z1.d, z31.d
+mad	z0.h, p7/m, z1.h, z31.h
+mad	z0.s, p7/m, z1.s, z31.s
+match	p0.b, p0/z, z0.b, z0.b
+match	p0.h, p0/z, z0.h, z0.h
+match	p15.b, p7/z, z30.b, z31.b
+match	p15.h, p7/z, z30.h, z31.h
+mla	z0.b, p7/m, z1.b, z31.b
+mla	z0.d, p7/m, z1.d, z31.d
+mla	z0.d, z1.d, z7.d[1]
+mla	z0.h, p7/m, z1.h, z31.h
+mla	z0.h, z1.h, z7.h[7]
+mla	z0.s, p7/m, z1.s, z31.s
+mla	z0.s, z1.s, z7.s[3]
+mls	z0.b, p7/m, z1.b, z31.b
+mls	z0.d, p7/m, z1.d, z31.d
+mls	z0.d, z1.d, z7.d[1]
+mls	z0.h, p7/m, z1.h, z31.h
+mls	z0.h, z1.h, z7.h[7]
+mls	z0.s, p7/m, z1.s, z31.s
+mls	z0.s, z1.s, z7.s[3]
+mov	p0.b, p0.b
+mov	p0.b, p0/m, p0.b
+mov	p0.b, p0/z, p0.b
+mov	p15.b, p15.b
+mov	p15.b, p15/m, p15.b
+mov	p15.b, p15/z, p15.b
+mov	z0.b, #127
+mov	z0.b, b0
+mov	z0.b, p0/m, b0
+mov	z0.b, p0/m, w0
+mov	z0.b, p0/z, #127
+mov	z0.b, w0
+mov	z0.d, #0
+mov	z0.d, #0xe0000000000003ff
+mov	z0.d, #0xffffffffffff7fff
+mov	z0.d, #32768
+mov	z0.d, d0
+mov	z0.d, p0/m, d0
+mov	z0.d, p0/m, x0
+mov	z0.d, x0
+mov	z0.d, z0.d
+mov	z0.h, #-256
+mov	z0.h, #-32768
+mov	z0.h, #0
+mov	z0.h, #32512
+mov	z0.h, #32767
+mov	z0.h, h0
+mov	z0.h, p0/m, h0
+mov	z0.h, p0/m, w0
+mov	z0.h, p0/z, #32512
+mov	z0.h, w0
+mov	z0.q, q0
+mov	z0.s, #0
+mov	z0.s, #0xffff7fff
+mov	z0.s, #32768
+mov	z0.s, p0/m, s0
+mov	z0.s, p0/m, w0
+mov	z0.s, s0
+mov	z0.s, w0
+mov	z21.d, #-128
+mov	z21.d, #-32768
+mov	z21.d, #127
+mov	z21.d, #32512
+mov	z21.d, p0/z, #-128
+mov	z21.d, p0/z, #-32768
+mov	z21.d, p0/z, #127
+mov	z21.d, p0/z, #32512
+mov	z21.d, p15/m, #-128
+mov	z21.d, p15/m, #-32768
+mov	z21.h, #-128
+mov	z21.h, #-32768
+mov	z21.h, #127
+mov	z21.h, #32512
+mov	z21.h, p0/z, #-128
+mov	z21.h, p0/z, #-32768
+mov	z21.h, p0/z, #127
+mov	z21.h, p0/z, #32512
+mov	z21.h, p15/m, #-128
+mov	z21.h, p15/m, #-32768
+mov	z21.s, #-128
+mov	z21.s, #-32768
+mov	z21.s, #127
+mov	z21.s, #32512
+mov	z21.s, p0/z, #-128
+mov	z21.s, p0/z, #-32768
+mov	z21.s, p0/z, #127
+mov	z21.s, p0/z, #32512
+mov	z21.s, p15/m, #-128
+mov	z21.s, p15/m, #-32768
+mov	z31.b, p15/m, z31.b
+mov	z31.b, p7/m, b31
+movprfx z31, z6
+mov	z31.b, p7/m, wsp
+mov	z31.b, wsp
+mov	z31.b, z31.b[63]
+mov	z31.d, p15/m, z31.d
+mov	z31.d, p7/m, d31
+movprfx z31.d, p7/z, z6.d
+mov	z31.d, p7/m, sp
+mov	z31.d, sp
+mov	z31.d, z0.d
+mov	z31.d, z31.d[7]
+mov	z31.h, p15/m, z31.h
+mov	z31.h, p7/m, h31
+mov	z31.h, p7/m, wsp
+mov	z31.h, wsp
+mov	z31.h, z31.h[31]
+mov	z31.s, p15/m, z31.s
+mov	z31.s, p7/m, s31
+mov	z31.s, p7/m, wsp
+mov	z31.s, wsp
+mov	z31.s, z31.s[15]
+mov	z5.b, #-1
+mov	z5.b, #-128
+mov	z5.b, #127
+mov	z5.b, p0/z, #-1
+mov	z5.b, p0/z, #-128
+mov	z5.b, p0/z, #127
+mov	z5.b, p15/m, #-128
+mov	z5.d, #-6
+mov	z5.h, #-6
+mov	z5.q, z17.q[3]
+mov	z5.s, #-6
+movs	p0.b, p0.b
+movs	p0.b, p0/z, p0.b
+movs	p15.b, p15.b
+movs	p15.b, p15/z, p15.b
+mrs	x3, ID_AA64ZFR0_EL1
+mrs	x3, ZCR_EL1
+mrs	x3, ZCR_EL12
+mrs	x3, ZCR_EL2
+mrs	x3, ZCR_EL3
+msb	z0.b, p7/m, z1.b, z31.b
+msb	z0.d, p7/m, z1.d, z31.d
+msb	z0.h, p7/m, z1.h, z31.h
+msb	z0.s, p7/m, z1.s, z31.s
+msr	ZCR_EL1, x3
+msr	ZCR_EL12, x3
+msr	ZCR_EL2, x3
+msr	ZCR_EL3, x3
+mul	z0.b, p7/m, z0.b, z31.b
+mul	z0.b, z1.b, z2.b
+mul	z0.d, p7/m, z0.d, z31.d
+mul	z0.d, z1.d, z15.d[1]
+mul	z0.h, p7/m, z0.h, z31.h
+mul	z0.h, z1.h, z2.h
+mul	z0.h, z1.h, z7.h[7]
+mul	z0.s, p7/m, z0.s, z31.s
+mul	z0.s, z1.s, z7.s[3]
+mul	z29.s, z30.s, z31.s
+mul	z31.b, z31.b, #-128
+mul	z31.b, z31.b, #127
+mul	z31.d, z31.d, #-128
+mul	z31.d, z31.d, #127
+mul	z31.d, z31.d, z31.d
+mul	z31.h, z31.h, #-128
+mul	z31.h, z31.h, #127
+mul	z31.s, z31.s, #-128
+mul	z31.s, z31.s, #127
+nand	p0.b, p0/z, p0.b, p0.b
+nand	p15.b, p15/z, p15.b, p15.b
+nands	p0.b, p0/z, p0.b, p0.b
+nands	p15.b, p15/z, p15.b, p15.b
+nbsl	z0.d, z0.d, z1.d, z2.d
+neg	z0.b, p0/m, z0.b
+neg	z0.d, p0/m, z0.d
+neg	z0.h, p0/m, z0.h
+neg	z0.s, p0/m, z0.s
+neg	z31.b, p7/m, z31.b
+neg	z31.d, p7/m, z31.d
+neg	z31.h, p7/m, z31.h
+neg	z31.s, p7/m, z31.s
+nmatch	p0.b, p0/z, z0.b, z0.b
+nmatch	p0.h, p0/z, z0.h, z0.h
+nmatch	p15.b, p7/z, z30.b, z31.b
+nmatch	p15.h, p7/z, z30.h, z31.h
+nor	p0.b, p0/z, p0.b, p0.b
+nor	p15.b, p15/z, p15.b, p15.b
+nors	p0.b, p0/z, p0.b, p0.b
+nors	p15.b, p15/z, p15.b, p15.b
+not	p0.b, p0/z, p0.b
+not	p15.b, p15/z, p15.b
+not	z31.b, p7/m, z31.b
+not	z31.d, p7/m, z31.d
+not	z31.h, p7/m, z31.h
+not	z31.s, p7/m, z31.s
+nots	p0.b, p0/z, p0.b
+nots	p15.b, p15/z, p15.b
+orn	p0.b, p0/z, p0.b, p0.b
+orn	p15.b, p15/z, p15.b, p15.b
+orns	p0.b, p0/z, p0.b, p0.b
+orns	p15.b, p15/z, p15.b, p15.b
+orr	p0.b, p0/z, p0.b, p1.b
+orr	z0.d, z0.d, #0x6
+orr	z0.d, z0.d, #0xfffffffffffffff9
+orr	z0.s, z0.s, #0x6
+orr	z0.s, z0.s, #0xfffffff9
+orr	z23.d, z13.d, z8.d
+orr	z23.h, z23.h, #0x6
+orr	z23.h, z23.h, #0xfff9
+orr	z31.b, p7/m, z31.b, z31.b
+orr	z31.d, p7/m, z31.d, z31.d
+orr	z31.h, p7/m, z31.h, z31.h
+orr	z31.s, p7/m, z31.s, z31.s
+orr	z5.b, z5.b, #0x6
+orr	z5.b, z5.b, #0xf9
+orrs	p0.b, p0/z, p0.b, p1.b
+orv	b0, p7, z31.b
+orv	d0, p7, z31.d
+orv	h0, p7, z31.h
+orv	s0, p7, z31.s
+pfalse	p15.b
+pfirst	p0.b, p15, p0.b
+pfirst	p15.b, p15, p15.b
+pmul	z0.b, z1.b, z2.b
+pmul	z29.b, z30.b, z31.b
+pmullb	z0.h, z1.b, z2.b
+pmullb	z29.q, z30.d, z31.d
+pmullb	z31.d, z31.s, z31.s
+pmullt	z0.h, z1.b, z2.b
+pmullt	z29.q, z30.d, z31.d
+pmullt	z31.d, z31.s, z31.s
+pnext	p0.b, p15, p0.b
+pnext	p0.d, p15, p0.d
+pnext	p0.h, p15, p0.h
+pnext	p0.s, p15, p0.s
+pnext	p15.b, p15, p15.b
+prfb	#14, p0, [x0]
+prfb	#15, p0, [x0]
+prfb	#6, p0, [x0]
+prfb	#7, p0, [x0]
+prfb	#7, p3, [z13.s, #31]
+prfb	#7, p3, [z13.s]
+prfb	pldl1keep, p0, [x0, z0.d, uxtw]
+prfb	pldl1keep, p0, [x0, z0.d]
+prfb	pldl1keep, p0, [x0, z0.s, uxtw]
+prfb	pldl1keep, p0, [x0]
+prfb	pldl1strm, p0, [x0, #-32, mul vl]
+prfb	pldl1strm, p0, [x0, #31, mul vl]
+prfb	pldl1strm, p0, [x0]
+prfb	pldl2keep, p0, [x0]
+prfb	pldl2strm, p0, [x0]
+prfb	pldl3keep, p0, [x0]
+prfb	pldl3strm, p0, [x0]
+prfb	pldl3strm, p5, [x10, z21.d, sxtw]
+prfb	pldl3strm, p5, [x10, z21.s, uxtw]
+prfb	pldl3strm, p5, [z10.d, #31]
+prfb	pldl3strm, p5, [z10.d]
+prfb	pstl1keep, p0, [x0]
+prfb	pstl1strm, p0, [x0]
+prfb	pstl2keep, p0, [x0]
+prfb	pstl2strm, p0, [x0]
+prfb	pstl3keep, p0, [x0]
+prfb	pstl3strm, p0, [x0]
+prfd	#14, p0, [x0]
+prfd	#15, p0, [x0]
+prfd	#15, p7, [z31.d, #248]
+prfd	#15, p7, [z31.d]
+prfd	#15, p7, [z31.s, #248]
+prfd	#15, p7, [z31.s]
+prfd	#6, p0, [x0]
+prfd	#7, p0, [x0]
+prfd	pldl1keep, p0, [x0, z0.d, lsl #3]
+prfd	pldl1keep, p0, [x0, z0.d, sxtw #3]
+prfd	pldl1keep, p0, [x0, z0.d, uxtw #3]
+prfd	pldl1keep, p0, [x0, z0.s, sxtw #3]
+prfd	pldl1keep, p0, [x0, z0.s, uxtw #3]
+prfd	pldl1keep, p0, [x0]
+prfd	pldl1strm, p0, [x0, #-32, mul vl]
+prfd	pldl1strm, p0, [x0, #31, mul vl]
+prfd	pldl1strm, p0, [x0]
+prfd	pldl2keep, p0, [x0]
+prfd	pldl2strm, p0, [x0]
+prfd	pldl3keep, p0, [x0]
+prfd	pldl3strm, p0, [x0]
+prfd	pstl1keep, p0, [x0]
+prfd	pstl1strm, p0, [x0]
+prfd	pstl2keep, p0, [x0]
+prfd	pstl2strm, p0, [x0]
+prfd	pstl3keep, p0, [x0]
+prfd	pstl3strm, p0, [x0]
+prfh	#14, p0, [x0]
+prfh	#15, p0, [x0]
+prfh	#15, p7, [z31.d, #62]
+prfh	#15, p7, [z31.d]
+prfh	#15, p7, [z31.s, #62]
+prfh	#15, p7, [z31.s]
+prfh	#6, p0, [x0]
+prfh	#7, p0, [x0]
+prfh	pldl1keep, p0, [x0, z0.d, lsl #1]
+prfh	pldl1keep, p0, [x0]
+prfh	pldl1strm, p0, [x0, #-32, mul vl]
+prfh	pldl1strm, p0, [x0, #31, mul vl]
+prfh	pldl1strm, p0, [x0]
+prfh	pldl2keep, p0, [x0]
+prfh	pldl2strm, p0, [x0]
+prfh	pldl3keep, p0, [x0]
+prfh	pldl3strm, p0, [x0]
+prfh	pldl3strm, p5, [x10, z21.d, sxtw #1]
+prfh	pldl3strm, p5, [x10, z21.d, uxtw #1]
+prfh	pldl3strm, p5, [x10, z21.s, sxtw #1]
+prfh	pldl3strm, p5, [x10, z21.s, uxtw #1]
+prfh	pstl1keep, p0, [x0]
+prfh	pstl1strm, p0, [x0]
+prfh	pstl2keep, p0, [x0]
+prfh	pstl2strm, p0, [x0]
+prfh	pstl3keep, p0, [x0]
+prfh	pstl3strm, p0, [x0]
+prfw	#14, p0, [x0]
+prfw	#15, p0, [x0]
+prfw	#15, p7, [z31.d, #124]
+prfw	#15, p7, [z31.d]
+prfw	#15, p7, [z31.s, #124]
+prfw	#15, p7, [z31.s]
+prfw	#6, p0, [x0]
+prfw	#7, p0, [x0]
+prfw	#7, p3, [x13, z8.d, uxtw #2]
+prfw	pldl1keep, p0, [x0, z0.d, sxtw #2]
+prfw	pldl1keep, p0, [x0, z0.s, uxtw #2]
+prfw	pldl1keep, p0, [x0]
+prfw	pldl1strm, p0, [x0, #-32, mul vl]
+prfw	pldl1strm, p0, [x0, #31, mul vl]
+prfw	pldl1strm, p0, [x0]
+prfw	pldl2keep, p0, [x0]
+prfw	pldl2strm, p0, [x0]
+prfw	pldl3keep, p0, [x0]
+prfw	pldl3strm, p0, [x0]
+prfw	pldl3strm, p5, [x10, z21.d, lsl #2]
+prfw	pldl3strm, p5, [x10, z21.s, sxtw #2]
+prfw	pstl1keep, p0, [x0]
+prfw	pstl1strm, p0, [x0]
+prfw	pstl2keep, p0, [x0]
+prfw	pstl2strm, p0, [x0]
+prfw	pstl3keep, p0, [x0]
+prfw	pstl3strm, p0, [x0]
+ptest	p15, p0.b
+ptest	p15, p15.b
+ptrue	p0.b, pow2
+ptrue	p0.d, pow2
+ptrue	p0.h, pow2
+ptrue	p0.s, pow2
+ptrue	p15.b
+ptrue	p15.d
+ptrue	p15.h
+ptrue	p15.s
+ptrue	p7.s
+ptrue	p7.s, #14
+ptrue	p7.s, #15
+ptrue	p7.s, #16
+ptrue	p7.s, #17
+ptrue	p7.s, #18
+ptrue	p7.s, #19
+ptrue	p7.s, #20
+ptrue	p7.s, #21
+ptrue	p7.s, #22
+ptrue	p7.s, #23
+ptrue	p7.s, #24
+ptrue	p7.s, #25
+ptrue	p7.s, #26
+ptrue	p7.s, #27
+ptrue	p7.s, #28
+ptrue	p7.s, mul3
+ptrue	p7.s, mul4
+ptrue	p7.s, vl1
+ptrue	p7.s, vl128
+ptrue	p7.s, vl16
+ptrue	p7.s, vl2
+ptrue	p7.s, vl256
+ptrue	p7.s, vl3
+ptrue	p7.s, vl32
+ptrue	p7.s, vl4
+ptrue	p7.s, vl5
+ptrue	p7.s, vl6
+ptrue	p7.s, vl64
+ptrue	p7.s, vl7
+ptrue	p7.s, vl8
+ptrues	p0.b, pow2
+ptrues	p0.d, pow2
+ptrues	p0.h, pow2
+ptrues	p0.s, pow2
+ptrues	p15.b
+ptrues	p15.d
+ptrues	p15.h
+ptrues	p15.s
+ptrues	p7.s
+ptrues	p7.s, #14
+ptrues	p7.s, #15
+ptrues	p7.s, #16
+ptrues	p7.s, #17
+ptrues	p7.s, #18
+ptrues	p7.s, #19
+ptrues	p7.s, #20
+ptrues	p7.s, #21
+ptrues	p7.s, #22
+ptrues	p7.s, #23
+ptrues	p7.s, #24
+ptrues	p7.s, #25
+ptrues	p7.s, #26
+ptrues	p7.s, #27
+ptrues	p7.s, #28
+ptrues	p7.s, mul3
+ptrues	p7.s, mul4
+ptrues	p7.s, vl1
+ptrues	p7.s, vl128
+ptrues	p7.s, vl16
+ptrues	p7.s, vl2
+ptrues	p7.s, vl256
+ptrues	p7.s, vl3
+ptrues	p7.s, vl32
+ptrues	p7.s, vl4
+ptrues	p7.s, vl5
+ptrues	p7.s, vl6
+ptrues	p7.s, vl64
+ptrues	p7.s, vl7
+ptrues	p7.s, vl8
+punpkhi	p0.h, p0.b
+punpkhi	p15.h, p15.b
+punpklo	p0.h, p0.b
+punpklo	p15.h, p15.b
+raddhnb	z0.b, z1.h, z31.h
+raddhnb	z0.h, z1.s, z31.s
+raddhnb	z0.s, z1.d, z31.d
+raddhnt	z0.b, z1.h, z31.h
+raddhnt	z0.h, z1.s, z31.s
+raddhnt	z0.s, z1.d, z31.d
+rax1	z0.d, z1.d, z31.d
+rbit	z0.b, p7/m, z31.b
+rbit	z0.d, p7/m, z31.d
+rbit	z0.h, p7/m, z31.h
+rbit	z0.s, p7/m, z31.s
+rdffr	p0.b
+rdffr	p0.b, p0/z
+rdffr	p15.b
+rdffr	p15.b, p15/z
+rdffrs	p0.b, p0/z
+rdffrs	p15.b, p15/z
+rdvl	x0, #0
+rdvl	x21, #-32
+rdvl	x23, #31
+rdvl	xzr, #-1
+rev	z0.b, z31.b
+rev	z0.d, z31.d
+rev	z0.h, z31.h
+rev	z0.s, z31.s
+revb	z0.d, p7/m, z31.d
+revb	z0.h, p7/m, z31.h
+revb	z0.s, p7/m, z31.s
+revh	z0.d, p7/m, z31.d
+revh	z0.s, p7/m, z31.s
+revw	z0.d, p7/m, z31.d
+rshrnb	z0.b, z0.h, #1
+rshrnb	z0.h, z0.s, #1
+rshrnb	z0.s, z0.d, #1
+rshrnb	z31.b, z31.h, #8
+rshrnb	z31.h, z31.s, #16
+rshrnb	z31.s, z31.d, #32
+rshrnt	z0.b, z0.h, #1
+rshrnt	z0.h, z0.s, #1
+rshrnt	z0.s, z0.d, #1
+rshrnt	z31.b, z31.h, #8
+rshrnt	z31.h, z31.s, #16
+rshrnt	z31.s, z31.d, #32
+rsubhnb	z0.b, z1.h, z31.h
+rsubhnb	z0.h, z1.s, z31.s
+rsubhnb	z0.s, z1.d, z31.d
+rsubhnt	z0.b, z1.h, z31.h
+rsubhnt	z0.h, z1.s, z31.s
+rsubhnt	z0.s, z1.d, z31.d
+saba	z0.b, z1.b, z31.b
+saba	z0.d, z1.d, z31.d
+saba	z0.h, z1.h, z31.h
+saba	z0.s, z1.s, z31.s
+sabalb	z0.d, z1.s, z31.s
+sabalb	z0.h, z1.b, z31.b
+sabalb	z0.s, z1.h, z31.h
+sabalt	z0.d, z1.s, z31.s
+sabalt	z0.h, z1.b, z31.b
+sabalt	z0.s, z1.h, z31.h
+sabd	z31.b, p7/m, z31.b, z31.b
+sabd	z31.d, p7/m, z31.d, z31.d
+sabd	z31.h, p7/m, z31.h, z31.h
+sabd	z31.s, p7/m, z31.s, z31.s
+sabdlb	z0.h, z1.b, z2.b
+sabdlb	z29.s, z30.h, z31.h
+sabdlb	z31.d, z31.s, z31.s
+sabdlt	z0.h, z1.b, z2.b
+sabdlt	z29.s, z30.h, z31.h
+sabdlt	z31.d, z31.s, z31.s
+sadalp	z0.h, p0/m, z1.b
+sadalp	z29.s, p0/m, z30.h
+sadalp	z30.d, p7/m, z31.s
+saddlb	z0.h, z1.b, z2.b
+saddlb	z29.s, z30.h, z31.h
+saddlb	z31.d, z31.s, z31.s
+saddlbt	z0.d, z1.s, z31.s
+saddlbt	z0.h, z1.b, z31.b
+saddlbt	z0.s, z1.h, z31.h
+saddlt	z0.h, z1.b, z2.b
+saddlt	z29.s, z30.h, z31.h
+saddlt	z31.d, z31.s, z31.s
+saddv	d0, p7, z31.b
+saddv	d0, p7, z31.h
+saddv	d0, p7, z31.s
+saddwb	z0.h, z1.h, z2.b
+saddwb	z29.s, z30.s, z31.h
+saddwb	z31.d, z31.d, z31.s
+saddwt	z0.h, z1.h, z2.b
+saddwt	z29.s, z30.s, z31.h
+saddwt	z31.d, z31.d, z31.s
+sbclb	z0.d, z1.d, z31.d
+sbclb	z0.s, z1.s, z31.s
+sbclt	z0.d, z1.d, z31.d
+sbclt	z0.s, z1.s, z31.s
+scvtf	z0.d, p0/m, z0.d
+scvtf	z0.d, p0/m, z0.s
+scvtf	z0.h, p0/m, z0.d
+scvtf	z0.h, p0/m, z0.h
+scvtf	z0.h, p0/m, z0.s
+scvtf	z0.s, p0/m, z0.d
+scvtf	z0.s, p0/m, z0.s
+sdiv	z0.d, p7/m, z0.d, z31.d
+sdiv	z0.s, p7/m, z0.s, z31.s
+sdivr	z0.d, p7/m, z0.d, z31.d
+sdivr	z0.s, p7/m, z0.s, z31.s
+sdot	z0.d, z1.h, z15.h[1]
+sdot	z0.d, z1.h, z31.h
+sdot	z0.s, z1.b, z31.b
+sdot	z0.s, z1.b, z7.b[3]
+sel	z23.b, p11, z13.b, z8.b
+sel	z23.d, p11, z13.d, z8.d
+sel	z23.h, p11, z13.h, z8.h
+sel	z23.s, p11, z13.s, z8.s
+setffr
+shadd	z0.b, p0/m, z0.b, z1.b
+shadd	z0.h, p0/m, z0.h, z1.h
+shadd	z29.s, p7/m, z29.s, z30.s
+shadd	z31.d, p7/m, z31.d, z30.d
+shrnb	z0.b, z0.h, #1
+shrnb	z0.h, z0.s, #1
+shrnb	z0.s, z0.d, #1
+shrnb	z31.b, z31.h, #8
+shrnb	z31.h, z31.s, #16
+shrnb	z31.s, z31.d, #32
+shrnt	z0.b, z0.h, #1
+shrnt	z0.h, z0.s, #1
+shrnt	z0.s, z0.d, #1
+shrnt	z31.b, z31.h, #8
+shrnt	z31.h, z31.s, #16
+shrnt	z31.s, z31.d, #32
+shsub	z0.b, p0/m, z0.b, z1.b
+shsub	z0.h, p0/m, z0.h, z1.h
+shsub	z29.s, p7/m, z29.s, z30.s
+shsub	z31.d, p7/m, z31.d, z30.d
+shsubr	z0.b, p0/m, z0.b, z1.b
+shsubr	z0.h, p0/m, z0.h, z1.h
+shsubr	z29.s, p7/m, z29.s, z30.s
+shsubr	z31.d, p7/m, z31.d, z30.d
+sli	z0.b, z0.b, #0
+sli	z0.d, z0.d, #0
+sli	z0.h, z0.h, #0
+sli	z0.s, z0.s, #0
+sli	z31.b, z31.b, #7
+sli	z31.d, z31.d, #63
+sli	z31.h, z31.h, #15
+sli	z31.s, z31.s, #31
+sm4e	z0.s, z0.s, z31.s
+sm4ekey	z0.s, z1.s, z31.s
+smax	z0.b, z0.b, #-128
+smax	z0.d, z0.d, #-128
+smax	z0.h, z0.h, #-128
+smax	z0.s, z0.s, #-128
+smax	z31.b, p7/m, z31.b, z31.b
+smax	z31.b, z31.b, #127
+smax	z31.d, p7/m, z31.d, z31.d
+smax	z31.d, z31.d, #127
+smax	z31.h, p7/m, z31.h, z31.h
+smax	z31.h, z31.h, #127
+smax	z31.s, p7/m, z31.s, z31.s
+smax	z31.s, z31.s, #127
+smaxp	z0.b, p0/m, z0.b, z1.b
+smaxp	z0.h, p0/m, z0.h, z1.h
+smaxp	z29.s, p7/m, z29.s, z30.s
+smaxp	z31.d, p7/m, z31.d, z30.d
+smaxv	b0, p7, z31.b
+smaxv	d0, p7, z31.d
+smaxv	h0, p7, z31.h
+smaxv	s0, p7, z31.s
+smin	z0.b, z0.b, #-128
+smin	z0.d, z0.d, #-128
+smin	z0.h, z0.h, #-128
+smin	z0.s, z0.s, #-128
+smin	z31.b, p7/m, z31.b, z31.b
+smin	z31.b, z31.b, #127
+smin	z31.d, p7/m, z31.d, z31.d
+smin	z31.d, z31.d, #127
+smin	z31.h, p7/m, z31.h, z31.h
+smin	z31.h, z31.h, #127
+smin	z31.s, p7/m, z31.s, z31.s
+smin	z31.s, z31.s, #127
+sminp	z0.b, p0/m, z0.b, z1.b
+sminp	z0.h, p0/m, z0.h, z1.h
+sminp	z29.s, p7/m, z29.s, z30.s
+sminp	z31.d, p7/m, z31.d, z30.d
+sminv	b0, p7, z31.b
+sminv	d0, p7, z31.d
+sminv	h0, p7, z31.h
+sminv	s0, p7, z31.s
+smlalb	z0.d, z1.s, z15.s[1]
+smlalb	z0.d, z1.s, z31.s
+smlalb	z0.h, z1.b, z31.b
+smlalb	z0.s, z1.h, z31.h
+smlalb	z0.s, z1.h, z7.h[7]
+smlalt	z0.d, z1.s, z15.s[1]
+smlalt	z0.d, z1.s, z31.s
+smlalt	z0.h, z1.b, z31.b
+smlalt	z0.s, z1.h, z31.h
+smlalt	z0.s, z1.h, z7.h[7]
+smlslb	z0.d, z1.s, z15.s[1]
+smlslb	z0.d, z1.s, z31.s
+smlslb	z0.h, z1.b, z31.b
+smlslb	z0.s, z1.h, z31.h
+smlslb	z0.s, z1.h, z7.h[7]
+smlslt	z0.d, z1.s, z15.s[1]
+smlslt	z0.d, z1.s, z31.s
+smlslt	z0.h, z1.b, z31.b
+smlslt	z0.s, z1.h, z31.h
+smlslt	z0.s, z1.h, z7.h[7]
+smmla	z0.s, z1.b, z2.b
+smulh	z0.b, p7/m, z0.b, z31.b
+smulh	z0.b, z1.b, z2.b
+smulh	z0.d, p7/m, z0.d, z31.d
+smulh	z0.h, p7/m, z0.h, z31.h
+smulh	z0.h, z1.h, z2.h
+smulh	z0.s, p7/m, z0.s, z31.s
+smulh	z29.s, z30.s, z31.s
+smulh	z31.d, z31.d, z31.d
+smullb	z0.d, z1.s, z15.s[1]
+smullb	z0.h, z1.b, z2.b
+smullb	z0.s, z1.h, z7.h[7]
+smullb	z29.s, z30.h, z31.h
+smullb	z31.d, z31.s, z31.s
+smullt	z0.d, z1.s, z15.s[1]
+smullt	z0.h, z1.b, z2.b
+smullt	z0.s, z1.h, z7.h[7]
+smullt	z29.s, z30.h, z31.h
+smullt	z31.d, z31.s, z31.s
+splice	z29.b, p7, { z30.b, z31.b }
+splice	z29.d, p7, { z30.d, z31.d }
+splice	z29.h, p7, { z30.h, z31.h }
+splice	z29.s, p7, { z30.s, z31.s }
+splice	z31.b, p7, z31.b, z31.b
+splice	z31.d, p7, z31.d, z31.d
+splice	z31.h, p7, z31.h, z31.h
+splice	z31.s, p7, z31.s, z31.s
+sqabs	z31.b, p7/m, z31.b
+sqabs	z31.d, p7/m, z31.d
+sqabs	z31.h, p7/m, z31.h
+sqabs	z31.s, p7/m, z31.s
+sqadd	z0.b, p0/m, z0.b, z1.b
+sqadd	z0.b, z0.b, #0
+sqadd	z0.b, z0.b, z0.b
+sqadd	z0.d, z0.d, #0
+sqadd	z0.d, z0.d, #0, lsl #8
+sqadd	z0.d, z0.d, z0.d
+sqadd	z0.h, p0/m, z0.h, z1.h
+sqadd	z0.h, z0.h, #0
+sqadd	z0.h, z0.h, #0, lsl #8
+sqadd	z0.h, z0.h, z0.h
+sqadd	z0.s, z0.s, #0
+sqadd	z0.s, z0.s, #0, lsl #8
+sqadd	z0.s, z0.s, z0.s
+sqadd	z29.s, p7/m, z29.s, z30.s
+sqadd	z31.b, z31.b, #255
+sqadd	z31.d, p7/m, z31.d, z30.d
+sqadd	z31.d, z31.d, #65280
+sqadd	z31.h, z31.h, #65280
+sqadd	z31.s, z31.s, #65280
+sqcadd	z0.b, z0.b, z0.b, #90
+sqcadd	z0.d, z0.d, z0.d, #90
+sqcadd	z0.h, z0.h, z0.h, #90
+sqcadd	z0.s, z0.s, z0.s, #90
+sqcadd	z31.b, z31.b, z31.b, #270
+sqcadd	z31.d, z31.d, z31.d, #270
+sqcadd	z31.h, z31.h, z31.h, #270
+sqcadd	z31.s, z31.s, z31.s, #270
+sqdecb	x0
+sqdecb	x0, #14
+sqdecb	x0, all, mul #16
+sqdecb	x0, pow2
+sqdecb	x0, vl1
+sqdecb	x0, w0
+sqdecb	x0, w0, all, mul #16
+sqdecb	x0, w0, pow2
+sqdecb	x0, w0, pow2, mul #16
+sqdecd	x0
+sqdecd	x0, #14
+sqdecd	x0, all, mul #16
+sqdecd	x0, pow2
+sqdecd	x0, vl1
+sqdecd	x0, w0
+sqdecd	x0, w0, all, mul #16
+sqdecd	x0, w0, pow2
+sqdecd	x0, w0, pow2, mul #16
+sqdecd	z0.d
+sqdecd	z0.d, all, mul #16
+sqdecd	z0.d, pow2
+sqdecd	z0.d, pow2, mul #16
+sqdech	x0
+sqdech	x0, #14
+sqdech	x0, all, mul #16
+sqdech	x0, pow2
+sqdech	x0, vl1
+sqdech	x0, w0
+sqdech	x0, w0, all, mul #16
+sqdech	x0, w0, pow2
+sqdech	x0, w0, pow2, mul #16
+sqdech	z0.h
+sqdech	z0.h, all, mul #16
+sqdech	z0.h, pow2
+sqdech	z0.h, pow2, mul #16
+sqdecp	x0, p0.b
+sqdecp	x0, p0.d
+sqdecp	x0, p0.h
+sqdecp	x0, p0.s
+sqdecp	xzr, p15.b, wzr
+sqdecp	xzr, p15.d, wzr
+sqdecp	xzr, p15.h, wzr
+sqdecp	xzr, p15.s, wzr
+sqdecp	z0.d, p0.d
+sqdecp	z0.h, p0.h
+sqdecp	z0.s, p0.s
+sqdecw	x0
+sqdecw	x0, #14
+sqdecw	x0, all, mul #16
+sqdecw	x0, pow2
+sqdecw	x0, vl1
+sqdecw	x0, w0
+sqdecw	x0, w0, all, mul #16
+sqdecw	x0, w0, pow2
+sqdecw	x0, w0, pow2, mul #16
+sqdecw	z0.s
+sqdecw	z0.s, all, mul #16
+sqdecw	z0.s, pow2
+sqdecw	z0.s, pow2, mul #16
+sqdmlalb	z0.d, z1.s, z15.s[3]
+sqdmlalb	z0.d, z1.s, z31.s
+sqdmlalb	z0.h, z1.b, z31.b
+sqdmlalb	z0.s, z1.h, z31.h
+sqdmlalb	z0.s, z1.h, z7.h[7]
+sqdmlalbt	z0.d, z1.s, z31.s
+sqdmlalbt	z0.h, z1.b, z31.b
+sqdmlalbt	z0.s, z1.h, z31.h
+sqdmlalt	z0.d, z1.s, z15.s[3]
+sqdmlalt	z0.d, z1.s, z31.s
+sqdmlalt	z0.h, z1.b, z31.b
+sqdmlalt	z0.s, z1.h, z31.h
+sqdmlalt	z0.s, z1.h, z7.h[7]
+sqdmlslb	z0.d, z1.s, z15.s[3]
+sqdmlslb	z0.d, z1.s, z31.s
+sqdmlslb	z0.h, z1.b, z31.b
+sqdmlslb	z0.s, z1.h, z31.h
+sqdmlslb	z0.s, z1.h, z7.h[7]
+sqdmlslbt	z0.d, z1.s, z31.s
+sqdmlslbt	z0.h, z1.b, z31.b
+sqdmlslbt	z0.s, z1.h, z31.h
+sqdmlslt	z0.d, z1.s, z15.s[3]
+sqdmlslt	z0.d, z1.s, z31.s
+sqdmlslt	z0.h, z1.b, z31.b
+sqdmlslt	z0.s, z1.h, z31.h
+sqdmlslt	z0.s, z1.h, z7.h[7]
+sqdmulh	z0.b, z1.b, z2.b
+sqdmulh	z0.d, z1.d, z15.d[1]
+sqdmulh	z0.h, z1.h, z2.h
+sqdmulh	z0.h, z1.h, z7.h[7]
+sqdmulh	z0.s, z1.s, z7.s[3]
+sqdmulh	z29.s, z30.s, z31.s
+sqdmulh	z31.d, z31.d, z31.d
+sqdmullb	z0.d, z1.s, z15.s[1]
+sqdmullb	z0.h, z1.b, z2.b
+sqdmullb	z0.s, z1.h, z7.h[7]
+sqdmullb	z29.s, z30.h, z31.h
+sqdmullb	z31.d, z31.s, z31.s
+sqdmullt	z0.d, z1.s, z15.s[1]
+sqdmullt	z0.h, z1.b, z2.b
+sqdmullt	z0.s, z1.h, z7.h[7]
+sqdmullt	z29.s, z30.h, z31.h
+sqdmullt	z31.d, z31.s, z31.s
+sqincb	x0
+sqincb	x0, #14
+sqincb	x0, all, mul #16
+sqincb	x0, pow2
+sqincb	x0, vl1
+sqincb	x0, w0
+sqincb	x0, w0, all, mul #16
+sqincb	x0, w0, pow2
+sqincb	x0, w0, pow2, mul #16
+sqincd	x0
+sqincd	x0, #14
+sqincd	x0, all, mul #16
+sqincd	x0, pow2
+sqincd	x0, vl1
+sqincd	x0, w0
+sqincd	x0, w0, all, mul #16
+sqincd	x0, w0, pow2
+sqincd	x0, w0, pow2, mul #16
+sqincd	z0.d
+sqincd	z0.d, all, mul #16
+sqincd	z0.d, pow2
+sqincd	z0.d, pow2, mul #16
+sqinch	x0
+sqinch	x0, #14
+sqinch	x0, all, mul #16
+sqinch	x0, pow2
+sqinch	x0, vl1
+sqinch	x0, w0
+sqinch	x0, w0, all, mul #16
+sqinch	x0, w0, pow2
+sqinch	x0, w0, pow2, mul #16
+sqinch	z0.h
+sqinch	z0.h, all, mul #16
+sqinch	z0.h, pow2
+sqinch	z0.h, pow2, mul #16
+sqincp	x0, p0.b
+sqincp	x0, p0.d
+sqincp	x0, p0.h
+sqincp	x0, p0.s
+sqincp	xzr, p15.b, wzr
+sqincp	xzr, p15.d, wzr
+sqincp	xzr, p15.h, wzr
+sqincp	xzr, p15.s, wzr
+sqincp	z0.d, p0.d
+sqincp	z0.h, p0.h
+sqincp	z0.s, p0.s
+sqincw	x0
+sqincw	x0, #14
+sqincw	x0, all, mul #16
+sqincw	x0, pow2
+sqincw	x0, vl1
+sqincw	x0, w0
+sqincw	x0, w0, all, mul #16
+sqincw	x0, w0, pow2
+sqincw	x0, w0, pow2, mul #16
+sqincw	z0.s
+sqincw	z0.s, all, mul #16
+sqincw	z0.s, pow2
+sqincw	z0.s, pow2, mul #16
+sqneg	z31.b, p7/m, z31.b
+sqneg	z31.d, p7/m, z31.d
+sqneg	z31.h, p7/m, z31.h
+sqneg	z31.s, p7/m, z31.s
+sqrdcmlah	z0.b, z1.b, z2.b, #0
+sqrdcmlah	z0.d, z1.d, z2.d, #0
+sqrdcmlah	z0.h, z1.h, z2.h, #0
+sqrdcmlah	z0.h, z1.h, z2.h[0], #0
+sqrdcmlah	z0.s, z1.s, z2.s, #0
+sqrdcmlah	z0.s, z1.s, z2.s[0], #0
+sqrdcmlah	z15.b, z16.b, z17.b, #270
+sqrdcmlah	z15.d, z16.d, z17.d, #270
+sqrdcmlah	z15.h, z16.h, z17.h, #270
+sqrdcmlah	z15.s, z16.s, z17.s, #270
+sqrdcmlah	z29.b, z30.b, z31.b, #90
+sqrdcmlah	z29.d, z30.d, z31.d, #90
+sqrdcmlah	z29.h, z30.h, z31.h, #90
+sqrdcmlah	z29.s, z30.s, z31.s, #90
+sqrdcmlah	z31.b, z31.b, z31.b, #180
+sqrdcmlah	z31.d, z31.d, z31.d, #180
+sqrdcmlah	z31.h, z30.h, z7.h[0], #180
+sqrdcmlah	z31.h, z31.h, z31.h, #180
+sqrdcmlah	z31.s, z30.s, z7.s[0], #180
+sqrdcmlah	z31.s, z31.s, z31.s, #180
+sqrdmlah	z0.b, z1.b, z31.b
+sqrdmlah	z0.d, z1.d, z15.d[1]
+sqrdmlah	z0.d, z1.d, z31.d
+sqrdmlah	z0.h, z1.h, z31.h
+sqrdmlah	z0.h, z1.h, z7.h[7]
+sqrdmlah	z0.s, z1.s, z31.s
+sqrdmlah	z0.s, z1.s, z7.s[3]
+sqrdmlsh	z0.b, z1.b, z31.b
+sqrdmlsh	z0.d, z1.d, z15.d[1]
+sqrdmlsh	z0.d, z1.d, z31.d
+sqrdmlsh	z0.h, z1.h, z31.h
+sqrdmlsh	z0.h, z1.h, z7.h[7]
+sqrdmlsh	z0.s, z1.s, z31.s
+sqrdmlsh	z0.s, z1.s, z7.s[3]
+sqrdmulh	z0.b, z1.b, z2.b
+sqrdmulh	z0.d, z1.d, z15.d[1]
+sqrdmulh	z0.h, z1.h, z2.h
+sqrdmulh	z0.h, z1.h, z7.h[7]
+sqrdmulh	z0.s, z1.s, z7.s[3]
+sqrdmulh	z29.s, z30.s, z31.s
+sqrdmulh	z31.d, z31.d, z31.d
+sqrshl	z0.b, p0/m, z0.b, z1.b
+sqrshl	z0.h, p0/m, z0.h, z1.h
+sqrshl	z29.s, p7/m, z29.s, z30.s
+sqrshl	z31.d, p7/m, z31.d, z30.d
+sqrshlr	z0.b, p0/m, z0.b, z1.b
+sqrshlr	z0.h, p0/m, z0.h, z1.h
+sqrshlr	z29.s, p7/m, z29.s, z30.s
+sqrshlr	z31.d, p7/m, z31.d, z30.d
+sqrshrnb	z0.b, z0.h, #1
+sqrshrnb	z0.h, z0.s, #1
+sqrshrnb	z0.s, z0.d, #1
+sqrshrnb	z31.b, z31.h, #8
+sqrshrnb	z31.h, z31.s, #16
+sqrshrnb	z31.s, z31.d, #32
+sqrshrnt	z0.b, z0.h, #1
+sqrshrnt	z0.h, z0.s, #1
+sqrshrnt	z0.s, z0.d, #1
+sqrshrnt	z31.b, z31.h, #8
+sqrshrnt	z31.h, z31.s, #16
+sqrshrnt	z31.s, z31.d, #32
+sqrshrunb	z0.b, z0.h, #1
+sqrshrunb	z0.h, z0.s, #1
+sqrshrunb	z0.s, z0.d, #1
+sqrshrunb	z31.b, z31.h, #8
+sqrshrunb	z31.h, z31.s, #16
+sqrshrunb	z31.s, z31.d, #32
+sqrshrunt	z0.b, z0.h, #1
+sqrshrunt	z0.h, z0.s, #1
+sqrshrunt	z0.s, z0.d, #1
+sqrshrunt	z31.b, z31.h, #8
+sqrshrunt	z31.h, z31.s, #16
+sqrshrunt	z31.s, z31.d, #32
+sqshl	z0.b, p0/m, z0.b, #0
+sqshl	z0.b, p0/m, z0.b, z1.b
+sqshl	z0.d, p0/m, z0.d, #0
+sqshl	z0.h, p0/m, z0.h, #0
+sqshl	z0.h, p0/m, z0.h, z1.h
+sqshl	z0.s, p0/m, z0.s, #0
+sqshl	z29.s, p7/m, z29.s, z30.s
+sqshl	z31.b, p0/m, z31.b, #7
+sqshl	z31.d, p0/m, z31.d, #63
+sqshl	z31.d, p7/m, z31.d, z30.d
+sqshl	z31.h, p0/m, z31.h, #15
+sqshl	z31.s, p0/m, z31.s, #31
+sqshlr	z0.b, p0/m, z0.b, z1.b
+sqshlr	z0.h, p0/m, z0.h, z1.h
+sqshlr	z29.s, p7/m, z29.s, z30.s
+sqshlr	z31.d, p7/m, z31.d, z30.d
+sqshlu	z0.b, p0/m, z0.b, #0
+sqshlu	z0.d, p0/m, z0.d, #0
+sqshlu	z0.h, p0/m, z0.h, #0
+sqshlu	z0.s, p0/m, z0.s, #0
+sqshlu	z31.b, p0/m, z31.b, #7
+sqshlu	z31.d, p0/m, z31.d, #63
+sqshlu	z31.h, p0/m, z31.h, #15
+sqshlu	z31.s, p0/m, z31.s, #31
+sqshrnb	z0.b, z0.h, #1
+sqshrnb	z0.h, z0.s, #1
+sqshrnb	z0.s, z0.d, #1
+sqshrnb	z31.b, z31.h, #8
+sqshrnb	z31.h, z31.s, #16
+sqshrnb	z31.s, z31.d, #32
+sqshrnt	z0.b, z0.h, #1
+sqshrnt	z0.h, z0.s, #1
+sqshrnt	z0.s, z0.d, #1
+sqshrnt	z31.b, z31.h, #8
+sqshrnt	z31.h, z31.s, #16
+sqshrnt	z31.s, z31.d, #32
+sqshrunb	z0.b, z0.h, #1
+sqshrunb	z0.h, z0.s, #1
+sqshrunb	z0.s, z0.d, #1
+sqshrunb	z31.b, z31.h, #8
+sqshrunb	z31.h, z31.s, #16
+sqshrunb	z31.s, z31.d, #32
+sqshrunt	z0.b, z0.h, #1
+sqshrunt	z0.h, z0.s, #1
+sqshrunt	z0.s, z0.d, #1
+sqshrunt	z31.b, z31.h, #8
+sqshrunt	z31.h, z31.s, #16
+sqshrunt	z31.s, z31.d, #32
+sqsub	z0.b, p0/m, z0.b, z1.b
+sqsub	z0.b, z0.b, #0
+sqsub	z0.b, z0.b, z0.b
+sqsub	z0.d, z0.d, #0
+sqsub	z0.d, z0.d, #0, lsl #8
+sqsub	z0.d, z0.d, z0.d
+sqsub	z0.h, p0/m, z0.h, z1.h
+sqsub	z0.h, z0.h, #0
+sqsub	z0.h, z0.h, #0, lsl #8
+sqsub	z0.h, z0.h, z0.h
+sqsub	z0.s, z0.s, #0
+sqsub	z0.s, z0.s, #0, lsl #8
+sqsub	z0.s, z0.s, z0.s
+sqsub	z29.s, p7/m, z29.s, z30.s
+sqsub	z31.b, z31.b, #255
+sqsub	z31.d, p7/m, z31.d, z30.d
+sqsub	z31.d, z31.d, #65280
+sqsub	z31.h, z31.h, #65280
+sqsub	z31.s, z31.s, #65280
+sqsubr	z0.b, p0/m, z0.b, z1.b
+sqsubr	z0.h, p0/m, z0.h, z1.h
+sqsubr	z29.s, p7/m, z29.s, z30.s
+sqsubr	z31.d, p7/m, z31.d, z30.d
+sqxtnb	z0.b, z31.h
+sqxtnb	z0.h, z31.s
+sqxtnb	z0.s, z31.d
+sqxtnt	z0.b, z31.h
+sqxtnt	z0.h, z31.s
+sqxtnt	z0.s, z31.d
+sqxtunb	z0.b, z31.h
+sqxtunb	z0.h, z31.s
+sqxtunb	z0.s, z31.d
+sqxtunt	z0.b, z31.h
+sqxtunt	z0.h, z31.s
+sqxtunt	z0.s, z31.d
+srhadd	z0.b, p0/m, z0.b, z1.b
+srhadd	z0.h, p0/m, z0.h, z1.h
+srhadd	z29.s, p7/m, z29.s, z30.s
+srhadd	z31.d, p7/m, z31.d, z30.d
+sri	z0.b, z0.b, #1
+sri	z0.d, z0.d, #1
+sri	z0.h, z0.h, #1
+sri	z0.s, z0.s, #1
+sri	z31.b, z31.b, #8
+sri	z31.d, z31.d, #64
+sri	z31.h, z31.h, #16
+sri	z31.s, z31.s, #32
+srshl	z0.b, p0/m, z0.b, z1.b
+srshl	z0.h, p0/m, z0.h, z1.h
+srshl	z29.s, p7/m, z29.s, z30.s
+srshl	z31.d, p7/m, z31.d, z30.d
+srshlr	z0.b, p0/m, z0.b, z1.b
+srshlr	z0.h, p0/m, z0.h, z1.h
+srshlr	z29.s, p7/m, z29.s, z30.s
+srshlr	z31.d, p7/m, z31.d, z30.d
+srshr	z0.b, p0/m, z0.b, #1
+srshr	z0.d, p0/m, z0.d, #1
+srshr	z0.h, p0/m, z0.h, #1
+srshr	z0.s, p0/m, z0.s, #1
+srshr	z31.b, p0/m, z31.b, #8
+srshr	z31.d, p0/m, z31.d, #64
+srshr	z31.h, p0/m, z31.h, #16
+srshr	z31.s, p0/m, z31.s, #32
+srsra	z0.b, z0.b, #1
+srsra	z0.d, z0.d, #1
+srsra	z0.h, z0.h, #1
+srsra	z0.s, z0.s, #1
+srsra	z31.b, z31.b, #8
+srsra	z31.d, z31.d, #64
+srsra	z31.h, z31.h, #16
+srsra	z31.s, z31.s, #32
+sshllb	z0.d, z0.s, #0
+sshllb	z0.h, z0.b, #0
+sshllb	z0.s, z0.h, #0
+sshllb	z31.d, z31.s, #31
+sshllb	z31.h, z31.b, #7
+sshllb	z31.s, z31.h, #15
+sshllt	z0.d, z0.s, #0
+sshllt	z0.h, z0.b, #0
+sshllt	z0.s, z0.h, #0
+sshllt	z31.d, z31.s, #31
+sshllt	z31.h, z31.b, #7
+sshllt	z31.s, z31.h, #15
+ssra	z0.b, z0.b, #1
+ssra	z0.d, z0.d, #1
+ssra	z0.h, z0.h, #1
+ssra	z0.s, z0.s, #1
+ssra	z31.b, z31.b, #8
+ssra	z31.d, z31.d, #64
+ssra	z31.h, z31.h, #16
+ssra	z31.s, z31.s, #32
+ssublb	z0.h, z1.b, z2.b
+ssublb	z29.s, z30.h, z31.h
+ssublb	z31.d, z31.s, z31.s
+ssublbt	z0.d, z1.s, z31.s
+ssublbt	z0.h, z1.b, z31.b
+ssublbt	z0.s, z1.h, z31.h
+ssublt	z0.h, z1.b, z2.b
+ssublt	z29.s, z30.h, z31.h
+ssublt	z31.d, z31.s, z31.s
+ssubltb	z0.d, z1.s, z31.s
+ssubltb	z0.h, z1.b, z31.b
+ssubltb	z0.s, z1.h, z31.h
+ssubwb	z0.h, z1.h, z2.b
+ssubwb	z29.s, z30.s, z31.h
+ssubwb	z31.d, z31.d, z31.s
+ssubwt	z0.h, z1.h, z2.b
+ssubwt	z29.s, z30.s, z31.h
+ssubwt	z31.d, z31.d, z31.s
+st1b	{ z0.b }, p0, [x0, x0]
+st1b	{ z0.b }, p0, [x0]
+st1b	{ z0.d }, p0, [x0, x0]
+st1b	{ z0.d }, p0, [x0, z0.d, sxtw]
+st1b	{ z0.d }, p0, [x0, z0.d, uxtw]
+st1b	{ z0.d }, p0, [x0, z0.d]
+st1b	{ z0.d }, p0, [x0]
+st1b	{ z0.d }, p7, [z0.d]
+st1b	{ z0.h }, p0, [x0, x0]
+st1b	{ z0.h }, p0, [x0]
+st1b	{ z0.s }, p0, [x0, x0]
+st1b	{ z0.s }, p0, [x0, z0.s, sxtw]
+st1b	{ z0.s }, p0, [x0, z0.s, uxtw]
+st1b	{ z0.s }, p0, [x0]
+st1b	{ z0.s }, p7, [z0.s]
+st1b	{ z21.b }, p5, [x10, #5, mul vl]
+st1b	{ z21.d }, p5, [x10, #5, mul vl]
+st1b	{ z21.h }, p5, [x10, #5, mul vl]
+st1b	{ z21.s }, p5, [x10, #5, mul vl]
+st1b	{ z31.b }, p7, [sp, #-1, mul vl]
+st1b	{ z31.d }, p7, [sp, #-1, mul vl]
+st1b	{ z31.d }, p7, [z31.d, #31]
+st1b	{ z31.h }, p7, [sp, #-1, mul vl]
+st1b	{ z31.s }, p7, [sp, #-1, mul vl]
+st1b	{ z31.s }, p7, [z31.s, #31]
+st1d	{ z0.d }, p0, [x0, x0, lsl #3]
+st1d	{ z0.d }, p0, [x0, z0.d, lsl #3]
+st1d	{ z0.d }, p0, [x0, z0.d, sxtw #3]
+st1d	{ z0.d }, p0, [x0, z0.d, sxtw]
+st1d	{ z0.d }, p0, [x0, z0.d, uxtw #3]
+st1d	{ z0.d }, p0, [x0, z0.d, uxtw]
+st1d	{ z0.d }, p0, [x0, z0.d]
+st1d	{ z0.d }, p0, [x0]
+st1d	{ z0.d }, p7, [z0.d]
+st1d	{ z21.d }, p5, [x10, #5, mul vl]
+st1d	{ z31.d }, p7, [sp, #-1, mul vl]
+st1d	{ z31.d }, p7, [z31.d, #248]
+st1h	{ z0.d }, p0, [x0, x0, lsl #1]
+st1h	{ z0.d }, p0, [x0, z0.d, lsl #1]
+st1h	{ z0.d }, p0, [x0, z0.d, sxtw #1]
+st1h	{ z0.d }, p0, [x0, z0.d, sxtw]
+st1h	{ z0.d }, p0, [x0, z0.d, uxtw #1]
+st1h	{ z0.d }, p0, [x0, z0.d, uxtw]
+st1h	{ z0.d }, p0, [x0, z0.d]
+st1h	{ z0.d }, p0, [x0]
+st1h	{ z0.d }, p7, [z0.d]
+st1h	{ z0.h }, p0, [x0, x0, lsl #1]
+st1h	{ z0.h }, p0, [x0]
+st1h	{ z0.s }, p0, [x0, x0, lsl #1]
+st1h	{ z0.s }, p0, [x0, z0.s, sxtw #1]
+st1h	{ z0.s }, p0, [x0, z0.s, sxtw]
+st1h	{ z0.s }, p0, [x0, z0.s, uxtw #1]
+st1h	{ z0.s }, p0, [x0, z0.s, uxtw]
+st1h	{ z0.s }, p0, [x0]
+st1h	{ z0.s }, p7, [z0.s]
+st1h	{ z21.d }, p5, [x10, #5, mul vl]
+st1h	{ z21.h }, p5, [x10, #5, mul vl]
+st1h	{ z21.s }, p5, [x10, #5, mul vl]
+st1h	{ z31.d }, p7, [sp, #-1, mul vl]
+st1h	{ z31.d }, p7, [z31.d, #62]
+st1h	{ z31.h }, p7, [sp, #-1, mul vl]
+st1h	{ z31.s }, p7, [sp, #-1, mul vl]
+st1h	{ z31.s }, p7, [z31.s, #62]
+st1w	{ z0.d }, p0, [x0, x0, lsl #2]
+st1w	{ z0.d }, p0, [x0, z0.d, lsl #2]
+st1w	{ z0.d }, p0, [x0, z0.d, sxtw #2]
+st1w	{ z0.d }, p0, [x0, z0.d, sxtw]
+st1w	{ z0.d }, p0, [x0, z0.d, uxtw #2]
+st1w	{ z0.d }, p0, [x0, z0.d, uxtw]
+st1w	{ z0.d }, p0, [x0, z0.d]
+st1w	{ z0.d }, p0, [x0]
+st1w	{ z0.d }, p7, [z0.d]
+st1w	{ z0.s }, p0, [x0, x0, lsl #2]
+st1w	{ z0.s }, p0, [x0, z0.s, sxtw #2]
+st1w	{ z0.s }, p0, [x0, z0.s, sxtw]
+st1w	{ z0.s }, p0, [x0, z0.s, uxtw #2]
+st1w	{ z0.s }, p0, [x0, z0.s, uxtw]
+st1w	{ z0.s }, p0, [x0]
+st1w	{ z0.s }, p7, [z0.s]
+st1w	{ z21.d }, p5, [x10, #5, mul vl]
+st1w	{ z21.s }, p5, [x10, #5, mul vl]
+st1w	{ z31.d }, p7, [sp, #-1, mul vl]
+st1w	{ z31.d }, p7, [z31.d, #124]
+st1w	{ z31.s }, p7, [sp, #-1, mul vl]
+st1w	{ z31.s }, p7, [z31.s, #124]
+st2b	{ z0.b, z1.b }, p0, [x0, x0]
+st2b	{ z0.b, z1.b }, p0, [x0]
+st2b	{ z21.b, z22.b }, p5, [x10, #10, mul vl]
+st2b	{ z23.b, z24.b }, p3, [x13, #-16, mul vl]
+st2b	{ z5.b, z6.b }, p3, [x17, x16]
+st2d	{ z0.d, z1.d }, p0, [x0, x0, lsl #3]
+st2d	{ z0.d, z1.d }, p0, [x0]
+st2d	{ z21.d, z22.d }, p5, [x10, #10, mul vl]
+st2d	{ z23.d, z24.d }, p3, [x13, #-16, mul vl]
+st2d	{ z5.d, z6.d }, p3, [x17, x16, lsl #3]
+st2h	{ z0.h, z1.h }, p0, [x0, x0, lsl #1]
+st2h	{ z0.h, z1.h }, p0, [x0]
+st2h	{ z21.h, z22.h }, p5, [x10, #10, mul vl]
+st2h	{ z23.h, z24.h }, p3, [x13, #-16, mul vl]
+st2h	{ z5.h, z6.h }, p3, [x17, x16, lsl #1]
+st2w	{ z0.s, z1.s }, p0, [x0, x0, lsl #2]
+st2w	{ z0.s, z1.s }, p0, [x0]
+st2w	{ z21.s, z22.s }, p5, [x10, #10, mul vl]
+st2w	{ z23.s, z24.s }, p3, [x13, #-16, mul vl]
+st2w	{ z5.s, z6.s }, p3, [x17, x16, lsl #2]
+st3b	{ z0.b, z1.b, z2.b }, p0, [x0, x0]
+st3b	{ z0.b, z1.b, z2.b }, p0, [x0]
+st3b	{ z21.b, z22.b, z23.b }, p5, [x10, #15, mul vl]
+st3b	{ z23.b, z24.b, z25.b }, p3, [x13, #-24, mul vl]
+st3b	{ z5.b, z6.b, z7.b }, p3, [x17, x16]
+st3d	{ z0.d, z1.d, z2.d }, p0, [x0, x0, lsl #3]
+st3d	{ z0.d, z1.d, z2.d }, p0, [x0]
+st3d	{ z21.d, z22.d, z23.d }, p5, [x10, #15, mul vl]
+st3d	{ z23.d, z24.d, z25.d }, p3, [x13, #-24, mul vl]
+st3d	{ z5.d, z6.d, z7.d }, p3, [x17, x16, lsl #3]
+st3h	{ z0.h, z1.h, z2.h }, p0, [x0, x0, lsl #1]
+st3h	{ z0.h, z1.h, z2.h }, p0, [x0]
+st3h	{ z21.h, z22.h, z23.h }, p5, [x10, #15, mul vl]
+st3h	{ z23.h, z24.h, z25.h }, p3, [x13, #-24, mul vl]
+st3h	{ z5.h, z6.h, z7.h }, p3, [x17, x16, lsl #1]
+st3w	{ z0.s, z1.s, z2.s }, p0, [x0, x0, lsl #2]
+st3w	{ z0.s, z1.s, z2.s }, p0, [x0]
+st3w	{ z21.s, z22.s, z23.s }, p5, [x10, #15, mul vl]
+st3w	{ z23.s, z24.s, z25.s }, p3, [x13, #-24, mul vl]
+st3w	{ z5.s, z6.s, z7.s }, p3, [x17, x16, lsl #2]
+st4b	{ z0.b, z1.b, z2.b, z3.b }, p0, [x0, x0]
+st4b	{ z0.b, z1.b, z2.b, z3.b }, p0, [x0]
+st4b	{ z21.b, z22.b, z23.b, z24.b }, p5, [x10, #20, mul vl]
+st4b	{ z23.b, z24.b, z25.b, z26.b }, p3, [x13, #-32, mul vl]
+st4b	{ z5.b, z6.b, z7.b, z8.b }, p3, [x17, x16]
+st4d	{ z0.d, z1.d, z2.d, z3.d }, p0, [x0, x0, lsl #3]
+st4d	{ z0.d, z1.d, z2.d, z3.d }, p0, [x0]
+st4d	{ z21.d, z22.d, z23.d, z24.d }, p5, [x10, #20, mul vl]
+st4d	{ z23.d, z24.d, z25.d, z26.d }, p3, [x13, #-32, mul vl]
+st4d	{ z5.d, z6.d, z7.d, z8.d }, p3, [x17, x16, lsl #3]
+st4h	{ z0.h, z1.h, z2.h, z3.h }, p0, [x0, x0, lsl #1]
+st4h	{ z0.h, z1.h, z2.h, z3.h }, p0, [x0]
+st4h	{ z21.h, z22.h, z23.h, z24.h }, p5, [x10, #20, mul vl]
+st4h	{ z23.h, z24.h, z25.h, z26.h }, p3, [x13, #-32, mul vl]
+st4h	{ z5.h, z6.h, z7.h, z8.h }, p3, [x17, x16, lsl #1]
+st4w	{ z0.s, z1.s, z2.s, z3.s }, p0, [x0, x0, lsl #2]
+st4w	{ z0.s, z1.s, z2.s, z3.s }, p0, [x0]
+st4w	{ z21.s, z22.s, z23.s, z24.s }, p5, [x10, #20, mul vl]
+st4w	{ z23.s, z24.s, z25.s, z26.s }, p3, [x13, #-32, mul vl]
+st4w	{ z5.s, z6.s, z7.s, z8.s }, p3, [x17, x16, lsl #2]
+stnt1b	{ z0.b }, p0, [x0, x0]
+stnt1b	{ z0.b }, p0, [x0]
+stnt1b	{ z0.d }, p0, [z1.d]
+stnt1b	{ z0.s }, p0, [z1.s]
+stnt1b	{ z21.b }, p5, [x10, #7, mul vl]
+stnt1b	{ z23.b }, p3, [x13, #-8, mul vl]
+stnt1b	{ z31.d }, p7, [z31.d, x0]
+stnt1b	{ z31.d }, p7, [z31.d]
+stnt1b	{ z31.s }, p7, [z31.s, x0]
+stnt1b	{ z31.s }, p7, [z31.s]
+stnt1d	{ z0.d }, p0, [x0, x0, lsl #3]
+stnt1d	{ z0.d }, p0, [x0]
+stnt1d	{ z0.d }, p0, [z1.d]
+stnt1d	{ z21.d }, p5, [x10, #7, mul vl]
+stnt1d	{ z23.d }, p3, [x13, #-8, mul vl]
+stnt1d	{ z31.d }, p7, [z31.d, x0]
+stnt1d	{ z31.d }, p7, [z31.d]
+stnt1h	{ z0.d }, p0, [z1.d]
+stnt1h	{ z0.h }, p0, [x0, x0, lsl #1]
+stnt1h	{ z0.h }, p0, [x0]
+stnt1h	{ z0.s }, p0, [z1.s]
+stnt1h	{ z21.h }, p5, [x10, #7, mul vl]
+stnt1h	{ z23.h }, p3, [x13, #-8, mul vl]
+stnt1h	{ z31.d }, p7, [z31.d, x0]
+stnt1h	{ z31.d }, p7, [z31.d]
+stnt1h	{ z31.s }, p7, [z31.s, x0]
+stnt1h	{ z31.s }, p7, [z31.s]
+stnt1w	{ z0.d }, p0, [z1.d]
+stnt1w	{ z0.s }, p0, [x0, x0, lsl #2]
+stnt1w	{ z0.s }, p0, [x0]
+stnt1w	{ z0.s }, p0, [z1.s]
+stnt1w	{ z21.s }, p5, [x10, #7, mul vl]
+stnt1w	{ z23.s }, p3, [x13, #-8, mul vl]
+stnt1w	{ z31.d }, p7, [z31.d, x0]
+stnt1w	{ z31.d }, p7, [z31.d]
+stnt1w	{ z31.s }, p7, [z31.s, x0]
+stnt1w	{ z31.s }, p7, [z31.s]
+str	p0, [x0]
+str	p15, [sp, #-256, mul vl]
+str	p5, [x10, #255, mul vl]
+str	z0, [x0]
+str	z21, [x10, #-256, mul vl]
+str	z31, [sp, #255, mul vl]
+sub	z0.b, p0/m, z0.b, z0.b
+sub	z0.b, z0.b, #0
+sub	z0.b, z0.b, z0.b
+sub	z0.d, p0/m, z0.d, z0.d
+sub	z0.d, z0.d, #0
+sub	z0.d, z0.d, #0, lsl #8
+sub	z0.d, z0.d, z0.d
+sub	z0.h, p0/m, z0.h, z0.h
+sub	z0.h, z0.h, #0
+sub	z0.h, z0.h, #0, lsl #8
+sub	z0.h, z0.h, z0.h
+sub	z0.s, p0/m, z0.s, z0.s
+sub	z0.s, z0.s, #0
+sub	z0.s, z0.s, #0, lsl #8
+sub	z0.s, z0.s, z0.s
+sub	z21.b, p5/m, z21.b, z10.b
+sub	z21.b, z10.b, z21.b
+sub	z21.d, p5/m, z21.d, z10.d
+sub	z21.d, z10.d, z21.d
+sub	z21.h, p5/m, z21.h, z10.h
+sub	z21.h, z10.h, z21.h
+sub	z21.s, p5/m, z21.s, z10.s
+sub	z21.s, z10.s, z21.s
+sub	z23.b, p3/m, z23.b, z13.b
+sub	z23.b, z13.b, z8.b
+sub	z23.d, p3/m, z23.d, z13.d
+sub	z23.d, z13.d, z8.d
+sub	z23.h, p3/m, z23.h, z13.h
+sub	z23.h, z13.h, z8.h
+sub	z23.s, p3/m, z23.s, z13.s
+sub	z23.s, z13.s, z8.s
+sub	z31.b, p7/m, z31.b, z31.b
+sub	z31.b, z31.b, #255
+sub	z31.b, z31.b, z31.b
+sub	z31.d, p7/m, z31.d, z31.d
+sub	z31.d, z31.d, #65280
+sub	z31.d, z31.d, z31.d
+sub	z31.h, p7/m, z31.h, z31.h
+sub	z31.h, z31.h, #65280
+sub	z31.h, z31.h, z31.h
+sub	z31.s, p7/m, z31.s, z31.s
+sub	z31.s, z31.s, #65280
+sub	z31.s, z31.s, z31.s
+subhnb	z0.b, z1.h, z31.h
+subhnb	z0.h, z1.s, z31.s
+subhnb	z0.s, z1.d, z31.d
+subhnt	z0.b, z1.h, z31.h
+subhnt	z0.h, z1.s, z31.s
+subhnt	z0.s, z1.d, z31.d
+subr	z0.b, p0/m, z0.b, z0.b
+subr	z0.b, z0.b, #0
+subr	z0.d, p0/m, z0.d, z0.d
+subr	z0.d, z0.d, #0
+subr	z0.d, z0.d, #0, lsl #8
+subr	z0.h, p0/m, z0.h, z0.h
+subr	z0.h, z0.h, #0
+subr	z0.h, z0.h, #0, lsl #8
+subr	z0.s, p0/m, z0.s, z0.s
+subr	z0.s, z0.s, #0
+subr	z0.s, z0.s, #0, lsl #8
+subr	z31.b, z31.b, #255
+subr	z31.d, z31.d, #65280
+subr	z31.h, z31.h, #65280
+subr	z31.s, z31.s, #65280
+sunpkhi	z31.d, z31.s
+sunpkhi	z31.h, z31.b
+sunpkhi	z31.s, z31.h
+sunpklo	z31.d, z31.s
+sunpklo	z31.h, z31.b
+sunpklo	z31.s, z31.h
+suqadd	z0.b, p0/m, z0.b, z1.b
+suqadd	z0.h, p0/m, z0.h, z1.h
+suqadd	z29.s, p7/m, z29.s, z30.s
+suqadd	z31.d, p7/m, z31.d, z30.d
+sxtb	z0.d, p0/m, z0.d
+sxtb	z0.h, p0/m, z0.h
+sxtb	z0.s, p0/m, z0.s
+sxtb	z31.d, p7/m, z31.d
+sxtb	z31.h, p7/m, z31.h
+sxtb	z31.s, p7/m, z31.s
+sxth	z0.d, p0/m, z0.d
+sxth	z0.s, p0/m, z0.s
+sxth	z31.d, p7/m, z31.d
+sxth	z31.s, p7/m, z31.s
+sxtw	z0.d, p0/m, z0.d
+sxtw	z31.d, p7/m, z31.d
+tbl	z28.b, { z29.b, z30.b }, z31.b
+tbl	z28.d, { z29.d, z30.d }, z31.d
+tbl	z28.h, { z29.h, z30.h }, z31.h
+tbl	z28.s, { z29.s, z30.s }, z31.s
+tbl	z31.b, { z31.b }, z31.b
+tbl	z31.d, { z31.d }, z31.d
+tbl	z31.h, { z31.h }, z31.h
+tbl	z31.s, { z31.s }, z31.s
+tbx	z31.b, z31.b, z31.b
+tbx	z31.d, z31.d, z31.d
+tbx	z31.h, z31.h, z31.h
+tbx	z31.s, z31.s, z31.s
+trn1	p15.b, p15.b, p15.b
+trn1	p15.d, p15.d, p15.d
+trn1	p15.h, p15.h, p15.h
+trn1	p15.s, p15.s, p15.s
+trn1	z31.b, z31.b, z31.b
+trn1	z31.d, z31.d, z31.d
+trn1	z31.h, z31.h, z31.h
+trn1	z31.s, z31.s, z31.s
+trn2	p15.b, p15.b, p15.b
+trn2	p15.d, p15.d, p15.d
+trn2	p15.h, p15.h, p15.h
+trn2	p15.s, p15.s, p15.s
+trn2	z31.b, z31.b, z31.b
+trn2	z31.d, z31.d, z31.d
+trn2	z31.h, z31.h, z31.h
+trn2	z31.s, z31.s, z31.s
+uaba	z0.b, z1.b, z31.b
+uaba	z0.d, z1.d, z31.d
+uaba	z0.h, z1.h, z31.h
+uaba	z0.s, z1.s, z31.s
+uabalb	z0.d, z1.s, z31.s
+uabalb	z0.h, z1.b, z31.b
+uabalb	z0.s, z1.h, z31.h
+uabalt	z0.d, z1.s, z31.s
+uabalt	z0.h, z1.b, z31.b
+uabalt	z0.s, z1.h, z31.h
+uabd	z31.b, p7/m, z31.b, z31.b
+uabd	z31.d, p7/m, z31.d, z31.d
+uabd	z31.h, p7/m, z31.h, z31.h
+uabd	z31.s, p7/m, z31.s, z31.s
+uabdlb	z0.h, z1.b, z2.b
+uabdlb	z29.s, z30.h, z31.h
+uabdlb	z31.d, z31.s, z31.s
+uabdlt	z0.h, z1.b, z2.b
+uabdlt	z29.s, z30.h, z31.h
+uabdlt	z31.d, z31.s, z31.s
+uadalp	z0.h, p0/m, z1.b
+uadalp	z29.s, p0/m, z30.h
+uadalp	z30.d, p7/m, z31.s
+uaddlb	z0.h, z1.b, z2.b
+uaddlb	z29.s, z30.h, z31.h
+uaddlb	z31.d, z31.s, z31.s
+uaddlt	z0.h, z1.b, z2.b
+uaddlt	z29.s, z30.h, z31.h
+uaddlt	z31.d, z31.s, z31.s
+uaddv	d0, p7, z31.b
+uaddv	d0, p7, z31.d
+uaddv	d0, p7, z31.h
+uaddv	d0, p7, z31.s
+uaddwb	z0.h, z1.h, z2.b
+uaddwb	z29.s, z30.s, z31.h
+uaddwb	z31.d, z31.d, z31.s
+uaddwt	z0.h, z1.h, z2.b
+uaddwt	z29.s, z30.s, z31.h
+uaddwt	z31.d, z31.d, z31.s
+ucvtf	z0.d, p0/m, z0.d
+ucvtf	z0.d, p0/m, z0.s
+ucvtf	z0.h, p0/m, z0.d
+ucvtf	z0.h, p0/m, z0.h
+ucvtf	z0.h, p0/m, z0.s
+ucvtf	z0.s, p0/m, z0.d
+ucvtf	z0.s, p0/m, z0.s
+udiv	z0.d, p7/m, z0.d, z31.d
+udiv	z0.s, p7/m, z0.s, z31.s
+udivr	z0.d, p7/m, z0.d, z31.d
+udivr	z0.s, p7/m, z0.s, z31.s
+udot	z0.d, z1.h, z15.h[1]
+udot	z0.d, z1.h, z31.h
+udot	z0.s, z1.b, z31.b
+udot	z0.s, z1.b, z7.b[3]
+uhadd	z0.b, p0/m, z0.b, z1.b
+uhadd	z0.h, p0/m, z0.h, z1.h
+uhadd	z29.s, p7/m, z29.s, z30.s
+uhadd	z31.d, p7/m, z31.d, z30.d
+uhsub	z0.b, p0/m, z0.b, z1.b
+uhsub	z0.h, p0/m, z0.h, z1.h
+uhsub	z29.s, p7/m, z29.s, z30.s
+uhsub	z31.d, p7/m, z31.d, z30.d
+uhsubr	z0.b, p0/m, z0.b, z1.b
+uhsubr	z0.h, p0/m, z0.h, z1.h
+uhsubr	z29.s, p7/m, z29.s, z30.s
+uhsubr	z31.d, p7/m, z31.d, z30.d
+umax	z0.b, z0.b, #0
+umax	z31.b, p7/m, z31.b, z31.b
+umax	z31.b, z31.b, #255
+umax	z31.d, p7/m, z31.d, z31.d
+umax	z31.h, p7/m, z31.h, z31.h
+umax	z31.s, p7/m, z31.s, z31.s
+umaxp	z0.b, p0/m, z0.b, z1.b
+umaxp	z0.h, p0/m, z0.h, z1.h
+umaxp	z29.s, p7/m, z29.s, z30.s
+umaxp	z31.d, p7/m, z31.d, z30.d
+umaxv	b0, p7, z31.b
+umaxv	d0, p7, z31.d
+umaxv	h0, p7, z31.h
+umaxv	s0, p7, z31.s
+umin	z0.b, z0.b, #0
+umin	z31.b, p7/m, z31.b, z31.b
+umin	z31.b, z31.b, #255
+umin	z31.d, p7/m, z31.d, z31.d
+umin	z31.h, p7/m, z31.h, z31.h
+umin	z31.s, p7/m, z31.s, z31.s
+uminp	z0.b, p0/m, z0.b, z1.b
+uminp	z0.h, p0/m, z0.h, z1.h
+uminp	z29.s, p7/m, z29.s, z30.s
+uminp	z31.d, p7/m, z31.d, z30.d
+uminv	b0, p7, z31.b
+uminv	d0, p7, z31.d
+uminv	h0, p7, z31.h
+uminv	s0, p7, z31.s
+umlalb	z0.d, z1.s, z15.s[1]
+umlalb	z0.d, z1.s, z31.s
+umlalb	z0.h, z1.b, z31.b
+umlalb	z0.s, z1.h, z31.h
+umlalb	z0.s, z1.h, z7.h[7]
+umlalt	z0.d, z1.s, z15.s[1]
+umlalt	z0.d, z1.s, z31.s
+umlalt	z0.h, z1.b, z31.b
+umlalt	z0.s, z1.h, z31.h
+umlalt	z0.s, z1.h, z7.h[7]
+umlslb	z0.d, z1.s, z15.s[1]
+umlslb	z0.d, z1.s, z31.s
+umlslb	z0.h, z1.b, z31.b
+umlslb	z0.s, z1.h, z31.h
+umlslb	z0.s, z1.h, z7.h[7]
+umlslt	z0.d, z1.s, z15.s[1]
+umlslt	z0.d, z1.s, z31.s
+umlslt	z0.h, z1.b, z31.b
+umlslt	z0.s, z1.h, z31.h
+umlslt	z0.s, z1.h, z7.h[7]
+ummla	z0.s, z1.b, z2.b
+umulh	z0.b, p7/m, z0.b, z31.b
+umulh	z0.b, z1.b, z2.b
+umulh	z0.d, p7/m, z0.d, z31.d
+umulh	z0.h, p7/m, z0.h, z31.h
+umulh	z0.h, z1.h, z2.h
+umulh	z0.s, p7/m, z0.s, z31.s
+umulh	z29.s, z30.s, z31.s
+umulh	z31.d, z31.d, z31.d
+umullb	z0.d, z1.s, z15.s[1]
+umullb	z0.h, z1.b, z2.b
+umullb	z0.s, z1.h, z7.h[7]
+umullb	z29.s, z30.h, z31.h
+umullb	z31.d, z31.s, z31.s
+umullt	z0.d, z1.s, z15.s[1]
+umullt	z0.h, z1.b, z2.b
+umullt	z0.s, z1.h, z7.h[7]
+umullt	z29.s, z30.h, z31.h
+umullt	z31.d, z31.s, z31.s
+uqadd	z0.b, p0/m, z0.b, z1.b
+uqadd	z0.b, z0.b, #0
+uqadd	z0.b, z0.b, z0.b
+uqadd	z0.d, z0.d, #0
+uqadd	z0.d, z0.d, #0, lsl #8
+uqadd	z0.d, z0.d, z0.d
+uqadd	z0.h, p0/m, z0.h, z1.h
+uqadd	z0.h, z0.h, #0
+uqadd	z0.h, z0.h, #0, lsl #8
+uqadd	z0.h, z0.h, z0.h
+uqadd	z0.s, z0.s, #0
+uqadd	z0.s, z0.s, #0, lsl #8
+uqadd	z0.s, z0.s, z0.s
+uqadd	z29.s, p7/m, z29.s, z30.s
+uqadd	z31.b, z31.b, #255
+uqadd	z31.d, p7/m, z31.d, z30.d
+uqadd	z31.d, z31.d, #65280
+uqadd	z31.h, z31.h, #65280
+uqadd	z31.s, z31.s, #65280
+uqdecb	w0
+uqdecb	w0, all, mul #16
+uqdecb	w0, pow2
+uqdecb	w0, pow2, mul #16
+uqdecb	x0
+uqdecb	x0, #14
+uqdecb	x0, all, mul #16
+uqdecb	x0, pow2
+uqdecb	x0, vl1
+uqdecd	w0
+uqdecd	w0, all, mul #16
+uqdecd	w0, pow2
+uqdecd	w0, pow2, mul #16
+uqdecd	x0
+uqdecd	x0, #14
+uqdecd	x0, all, mul #16
+uqdecd	x0, pow2
+uqdecd	x0, vl1
+uqdecd	z0.d
+uqdecd	z0.d, all, mul #16
+uqdecd	z0.d, pow2
+uqdecd	z0.d, pow2, mul #16
+uqdech	w0
+uqdech	w0, all, mul #16
+uqdech	w0, pow2
+uqdech	w0, pow2, mul #16
+uqdech	x0
+uqdech	x0, #14
+uqdech	x0, all, mul #16
+uqdech	x0, pow2
+uqdech	x0, vl1
+uqdech	z0.h
+uqdech	z0.h, all, mul #16
+uqdech	z0.h, pow2
+uqdech	z0.h, pow2, mul #16
+uqdecp	wzr, p15.b
+uqdecp	wzr, p15.d
+uqdecp	wzr, p15.h
+uqdecp	wzr, p15.s
+uqdecp	x0, p0.b
+uqdecp	x0, p0.d
+uqdecp	x0, p0.h
+uqdecp	x0, p0.s
+uqdecp	z0.d, p0.d
+uqdecp	z0.h, p0.h
+uqdecp	z0.s, p0.s
+uqdecw	w0
+uqdecw	w0, all, mul #16
+uqdecw	w0, pow2
+uqdecw	w0, pow2, mul #16
+uqdecw	x0
+uqdecw	x0, #14
+uqdecw	x0, all, mul #16
+uqdecw	x0, pow2
+uqdecw	x0, vl1
+uqdecw	z0.s
+uqdecw	z0.s, all, mul #16
+uqdecw	z0.s, pow2
+uqdecw	z0.s, pow2, mul #16
+uqincb	w0
+uqincb	w0, all, mul #16
+uqincb	w0, pow2
+uqincb	w0, pow2, mul #16
+uqincb	x0
+uqincb	x0, #14
+uqincb	x0, all, mul #16
+uqincb	x0, pow2
+uqincb	x0, vl1
+uqincd	w0
+uqincd	w0, all, mul #16
+uqincd	w0, pow2
+uqincd	w0, pow2, mul #16
+uqincd	x0
+uqincd	x0, #14
+uqincd	x0, all, mul #16
+uqincd	x0, pow2
+uqincd	x0, vl1
+uqincd	z0.d
+uqincd	z0.d, all, mul #16
+uqincd	z0.d, pow2
+uqincd	z0.d, pow2, mul #16
+uqinch	w0
+uqinch	w0, all, mul #16
+uqinch	w0, pow2
+uqinch	w0, pow2, mul #16
+uqinch	x0
+uqinch	x0, #14
+uqinch	x0, all, mul #16
+uqinch	x0, pow2
+uqinch	x0, vl1
+uqinch	z0.h
+uqinch	z0.h, all, mul #16
+uqinch	z0.h, pow2
+uqinch	z0.h, pow2, mul #16
+uqincp	wzr, p15.b
+uqincp	wzr, p15.d
+uqincp	wzr, p15.h
+uqincp	wzr, p15.s
+uqincp	x0, p0.b
+uqincp	x0, p0.d
+uqincp	x0, p0.h
+uqincp	x0, p0.s
+uqincp	z0.d, p0.d
+uqincp	z0.h, p0.h
+uqincp	z0.s, p0.s
+uqincw	w0
+uqincw	w0, all, mul #16
+uqincw	w0, pow2
+uqincw	w0, pow2, mul #16
+uqincw	x0
+uqincw	x0, #14
+uqincw	x0, all, mul #16
+uqincw	x0, pow2
+uqincw	x0, vl1
+uqincw	z0.s
+uqincw	z0.s, all, mul #16
+uqincw	z0.s, pow2
+uqincw	z0.s, pow2, mul #16
+uqrshl	z0.b, p0/m, z0.b, z1.b
+uqrshl	z0.h, p0/m, z0.h, z1.h
+uqrshl	z29.s, p7/m, z29.s, z30.s
+uqrshl	z31.d, p7/m, z31.d, z30.d
+uqrshlr	z0.b, p0/m, z0.b, z1.b
+uqrshlr	z0.h, p0/m, z0.h, z1.h
+uqrshlr	z29.s, p7/m, z29.s, z30.s
+uqrshlr	z31.d, p7/m, z31.d, z30.d
+uqrshrnb	z0.b, z0.h, #1
+uqrshrnb	z0.h, z0.s, #1
+uqrshrnb	z0.s, z0.d, #1
+uqrshrnb	z31.b, z31.h, #8
+uqrshrnb	z31.h, z31.s, #16
+uqrshrnb	z31.s, z31.d, #32
+uqrshrnt	z0.b, z0.h, #1
+uqrshrnt	z0.h, z0.s, #1
+uqrshrnt	z0.s, z0.d, #1
+uqrshrnt	z31.b, z31.h, #8
+uqrshrnt	z31.h, z31.s, #16
+uqrshrnt	z31.s, z31.d, #32
+uqshl	z0.b, p0/m, z0.b, #0
+uqshl	z0.b, p0/m, z0.b, z1.b
+uqshl	z0.d, p0/m, z0.d, #0
+uqshl	z0.h, p0/m, z0.h, #0
+uqshl	z0.h, p0/m, z0.h, z1.h
+uqshl	z0.s, p0/m, z0.s, #0
+uqshl	z29.s, p7/m, z29.s, z30.s
+uqshl	z31.b, p0/m, z31.b, #7
+uqshl	z31.d, p0/m, z31.d, #63
+uqshl	z31.d, p7/m, z31.d, z30.d
+uqshl	z31.h, p0/m, z31.h, #15
+uqshl	z31.s, p0/m, z31.s, #31
+uqshlr	z0.b, p0/m, z0.b, z1.b
+uqshlr	z0.h, p0/m, z0.h, z1.h
+uqshlr	z29.s, p7/m, z29.s, z30.s
+uqshlr	z31.d, p7/m, z31.d, z30.d
+uqshrnb	z0.b, z0.h, #1
+uqshrnb	z0.h, z0.s, #1
+uqshrnb	z0.s, z0.d, #1
+uqshrnb	z31.b, z31.h, #8
+uqshrnb	z31.h, z31.s, #16
+uqshrnb	z31.s, z31.d, #32
+uqshrnt	z0.b, z0.h, #1
+uqshrnt	z0.h, z0.s, #1
+uqshrnt	z0.s, z0.d, #1
+uqshrnt	z31.b, z31.h, #8
+uqshrnt	z31.h, z31.s, #16
+uqshrnt	z31.s, z31.d, #32
+uqsub	z0.b, p0/m, z0.b, z1.b
+uqsub	z0.b, z0.b, #0
+uqsub	z0.b, z0.b, z0.b
+uqsub	z0.d, z0.d, #0
+uqsub	z0.d, z0.d, #0, lsl #8
+uqsub	z0.d, z0.d, z0.d
+uqsub	z0.h, p0/m, z0.h, z1.h
+uqsub	z0.h, z0.h, #0
+uqsub	z0.h, z0.h, #0, lsl #8
+uqsub	z0.h, z0.h, z0.h
+uqsub	z0.s, z0.s, #0
+uqsub	z0.s, z0.s, #0, lsl #8
+uqsub	z0.s, z0.s, z0.s
+uqsub	z29.s, p7/m, z29.s, z30.s
+uqsub	z31.b, z31.b, #255
+uqsub	z31.d, p7/m, z31.d, z30.d
+uqsub	z31.d, z31.d, #65280
+uqsub	z31.h, z31.h, #65280
+uqsub	z31.s, z31.s, #65280
+uqsubr	z0.b, p0/m, z0.b, z1.b
+uqsubr	z0.h, p0/m, z0.h, z1.h
+uqsubr	z29.s, p7/m, z29.s, z30.s
+uqsubr	z31.d, p7/m, z31.d, z30.d
+uqxtnb	z0.b, z31.h
+uqxtnb	z0.h, z31.s
+uqxtnb	z0.s, z31.d
+uqxtnt	z0.b, z31.h
+uqxtnt	z0.h, z31.s
+uqxtnt	z0.s, z31.d
+urecpe	z31.s, p7/m, z31.s
+urhadd	z0.b, p0/m, z0.b, z1.b
+urhadd	z0.h, p0/m, z0.h, z1.h
+urhadd	z29.s, p7/m, z29.s, z30.s
+urhadd	z31.d, p7/m, z31.d, z30.d
+urshl	z0.b, p0/m, z0.b, z1.b
+urshl	z0.h, p0/m, z0.h, z1.h
+urshl	z29.s, p7/m, z29.s, z30.s
+urshl	z31.d, p7/m, z31.d, z30.d
+urshlr	z0.b, p0/m, z0.b, z1.b
+urshlr	z0.h, p0/m, z0.h, z1.h
+urshlr	z29.s, p7/m, z29.s, z30.s
+urshlr	z31.d, p7/m, z31.d, z30.d
+urshr	z0.b, p0/m, z0.b, #1
+urshr	z0.d, p0/m, z0.d, #1
+urshr	z0.h, p0/m, z0.h, #1
+urshr	z0.s, p0/m, z0.s, #1
+urshr	z31.b, p0/m, z31.b, #8
+urshr	z31.d, p0/m, z31.d, #64
+urshr	z31.h, p0/m, z31.h, #16
+urshr	z31.s, p0/m, z31.s, #32
+ursqrte	z31.s, p7/m, z31.s
+ursra	z0.b, z0.b, #1
+ursra	z0.d, z0.d, #1
+ursra	z0.h, z0.h, #1
+ursra	z0.s, z0.s, #1
+ursra	z31.b, z31.b, #8
+ursra	z31.d, z31.d, #64
+ursra	z31.h, z31.h, #16
+ursra	z31.s, z31.s, #32
+ushllb	z0.d, z0.s, #0
+ushllb	z0.h, z0.b, #0
+ushllb	z0.s, z0.h, #0
+ushllb	z31.d, z31.s, #31
+ushllb	z31.h, z31.b, #7
+ushllb	z31.s, z31.h, #15
+ushllt	z0.d, z0.s, #0
+ushllt	z0.h, z0.b, #0
+ushllt	z0.s, z0.h, #0
+ushllt	z31.d, z31.s, #31
+ushllt	z31.h, z31.b, #7
+ushllt	z31.s, z31.h, #15
+usmmla	z0.s, z1.b, z2.b
+usqadd	z0.b, p0/m, z0.b, z1.b
+usqadd	z0.h, p0/m, z0.h, z1.h
+usqadd	z29.s, p7/m, z29.s, z30.s
+usqadd	z31.d, p7/m, z31.d, z30.d
+usra	z0.b, z0.b, #1
+usra	z0.d, z0.d, #1
+usra	z0.h, z0.h, #1
+usra	z0.s, z0.s, #1
+usra	z31.b, z31.b, #8
+usra	z31.d, z31.d, #64
+usra	z31.h, z31.h, #16
+usra	z31.s, z31.s, #32
+usublb	z0.h, z1.b, z2.b
+usublb	z29.s, z30.h, z31.h
+usublb	z31.d, z31.s, z31.s
+usublt	z0.h, z1.b, z2.b
+usublt	z29.s, z30.h, z31.h
+usublt	z31.d, z31.s, z31.s
+usubwb	z0.h, z1.h, z2.b
+usubwb	z29.s, z30.s, z31.h
+usubwb	z31.d, z31.d, z31.s
+usubwt	z0.h, z1.h, z2.b
+usubwt	z29.s, z30.s, z31.h
+usubwt	z31.d, z31.d, z31.s
+uunpkhi	z31.d, z31.s
+uunpkhi	z31.h, z31.b
+uunpkhi	z31.s, z31.h
+uunpklo	z31.d, z31.s
+uunpklo	z31.h, z31.b
+uunpklo	z31.s, z31.h
+uxtb	z0.d, p0/m, z0.d
+uxtb	z0.h, p0/m, z0.h
+uxtb	z0.s, p0/m, z0.s
+uxtb	z31.d, p7/m, z31.d
+uxtb	z31.h, p7/m, z31.h
+uxtb	z31.s, p7/m, z31.s
+uxth	z0.d, p0/m, z0.d
+uxth	z0.s, p0/m, z0.s
+uxth	z31.d, p7/m, z31.d
+uxth	z31.s, p7/m, z31.s
+uxtw	z0.d, p0/m, z0.d
+uxtw	z31.d, p7/m, z31.d
+uzp1	p15.b, p15.b, p15.b
+uzp1	p15.d, p15.d, p15.d
+uzp1	p15.h, p15.h, p15.h
+uzp1	p15.s, p15.s, p15.s
+uzp1	z31.b, z31.b, z31.b
+uzp1	z31.d, z31.d, z31.d
+uzp1	z31.h, z31.h, z31.h
+uzp1	z31.s, z31.s, z31.s
+uzp2	p15.b, p15.b, p15.b
+uzp2	p15.d, p15.d, p15.d
+uzp2	p15.h, p15.h, p15.h
+uzp2	p15.s, p15.s, p15.s
+uzp2	z31.b, z31.b, z31.b
+uzp2	z31.d, z31.d, z31.d
+uzp2	z31.h, z31.h, z31.h
+uzp2	z31.s, z31.s, z31.s
+whilege	p15.b, w0, wzr
+whilege	p15.b, wzr, w0
+whilege	p15.b, x0, xzr
+whilege	p15.b, xzr, x0
+whilege	p15.d, w0, wzr
+whilege	p15.d, x0, xzr
+whilege	p15.h, w0, wzr
+whilege	p15.h, x0, xzr
+whilege	p15.s, w0, wzr
+whilege	p15.s, x0, xzr
+whilerw	p15.b, x30, x30
+whilerw	p15.d, x30, x30
+whilerw	p15.h, x30, x30
+whilerw	p15.s, x30, x30
+whilewr	p15.b, x30, x30
+whilewr	p15.d, x30, x30
+whilewr	p15.h, x30, x30
+whilewr	p15.s, x30, x30
+wrffr	p0.b
+wrffr	p15.b
+xar	z0.b, z0.b, z1.b, #1
+xar	z0.d, z0.d, z1.d, #1
+xar	z0.h, z0.h, z1.h, #1
+xar	z0.s, z0.s, z1.s, #1
+xar	z31.b, z31.b, z30.b, #8
+xar	z31.d, z31.d, z30.d, #64
+xar	z31.h, z31.h, z30.h, #16
+xar	z31.s, z31.s, z30.s, #32
+zip1	p0.b, p0.b, p0.b
+zip1	p0.d, p0.d, p0.d
+zip1	p0.h, p0.h, p0.h
+zip1	p0.s, p0.s, p0.s
+zip1	p15.b, p15.b, p15.b
+zip1	p15.d, p15.d, p15.d
+zip1	p15.h, p15.h, p15.h
+zip1	p15.s, p15.s, p15.s
+zip1	z0.b, z0.b, z0.b
+zip1	z0.d, z0.d, z0.d
+zip1	z0.h, z0.h, z0.h
+zip1	z0.s, z0.s, z0.s
+zip1	z31.b, z31.b, z31.b
+zip1	z31.d, z31.d, z31.d
+zip1	z31.h, z31.h, z31.h
+zip1	z31.s, z31.s, z31.s
+zip2	p0.b, p0.b, p0.b
+zip2	p0.d, p0.d, p0.d
+zip2	p0.h, p0.h, p0.h
+zip2	p0.s, p0.s, p0.s
+zip2	p15.b, p15.b, p15.b
+zip2	p15.d, p15.d, p15.d
+zip2	p15.h, p15.h, p15.h
+zip2	p15.s, p15.s, p15.s
+zip2	z0.b, z0.b, z0.b
+zip2	z0.d, z0.d, z0.d
+zip2	z0.h, z0.h, z0.h
+zip2	z0.s, z0.s, z0.s
+zip2	z31.b, z31.b, z31.b
+zip2	z31.d, z31.d, z31.d
+zip2	z31.h, z31.h, z31.h
+zip2	z31.s, z31.s, z31.s
+
+# CHECK:      Instruction Info:
+# CHECK-NEXT: [1]: #uOps
+# CHECK-NEXT: [2]: Latency
+# CHECK-NEXT: [3]: RThroughput
+# CHECK-NEXT: [4]: MayLoad
+# CHECK-NEXT: [5]: MayStore
+# CHECK-NEXT: [6]: HasSideEffects (U)
+
+# CHECK:      [1]    [2]    [3]    [4]    [5]    [6]    Instructions:
+# CHECK-NEXT:  1      3     0.50                        abs	z0.b, p0/m, z0.b
+# CHECK-NEXT:  1      3     0.50                        abs	z0.d, p0/m, z0.d
+# CHECK-NEXT:  1      3     0.50                        abs	z0.h, p0/m, z0.h
+# CHECK-NEXT:  1      3     0.50                        abs	z0.s, p0/m, z0.s
+# CHECK-NEXT:  1      3     0.50                        abs	z31.b, p7/m, z31.b
+# CHECK-NEXT:  1      3     0.50                        abs	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      3     0.50                        abs	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      3     0.50                        abs	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     0.50                        adclb	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        adclb	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        adclt	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        adclt	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      3     0.50                        add	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  1      3     0.50                        add	z0.b, z0.b, #0
+# CHECK-NEXT:  1      3     0.50                        add	z0.b, z0.b, z0.b
+# CHECK-NEXT:  1      3     0.50                        add	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      3     0.50                        add	z0.d, z0.d, #0
+# CHECK-NEXT:  1      3     0.50                        add	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  1      3     0.50                        add	z0.d, z0.d, z0.d
+# CHECK-NEXT:  1      3     0.50                        add	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  1      3     0.50                        add	z0.h, z0.h, #0
+# CHECK-NEXT:  1      3     0.50                        add	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  1      3     0.50                        add	z0.h, z0.h, z0.h
+# CHECK-NEXT:  1      3     0.50                        add	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  1      3     0.50                        add	z0.s, z0.s, #0
+# CHECK-NEXT:  1      3     0.50                        add	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  1      3     0.50                        add	z0.s, z0.s, z0.s
+# CHECK-NEXT:  1      3     0.50                        add	z0.s, z1.s, z2.s
+# CHECK-NEXT:  1      3     0.50                        add	z21.b, p5/m, z21.b, z10.b
+# CHECK-NEXT:  1      3     0.50                        add	z21.b, z10.b, z21.b
+# CHECK-NEXT:  1      3     0.50                        add	z21.d, p5/m, z21.d, z10.d
+# CHECK-NEXT:  1      3     0.50                        add	z21.d, z10.d, z21.d
+# CHECK-NEXT:  1      3     0.50                        add	z21.h, p5/m, z21.h, z10.h
+# CHECK-NEXT:  1      3     0.50                        add	z21.h, z10.h, z21.h
+# CHECK-NEXT:  1      3     0.50                        add	z21.s, p5/m, z21.s, z10.s
+# CHECK-NEXT:  1      3     0.50                        add	z21.s, z10.s, z21.s
+# CHECK-NEXT:  1      3     0.50                        add	z23.b, p3/m, z23.b, z13.b
+# CHECK-NEXT:  1      3     0.50                        add	z23.b, z13.b, z8.b
+# CHECK-NEXT:  1      3     0.50                        add	z23.d, p3/m, z23.d, z13.d
+# CHECK-NEXT:  1      3     0.50                        add	z23.d, z13.d, z8.d
+# CHECK-NEXT:  1      3     0.50                        add	z23.h, p3/m, z23.h, z13.h
+# CHECK-NEXT:  1      3     0.50                        add	z23.h, z13.h, z8.h
+# CHECK-NEXT:  1      3     0.50                        add	z23.s, p3/m, z23.s, z13.s
+# CHECK-NEXT:  1      3     0.50                        add	z23.s, z13.s, z8.s
+# CHECK-NEXT:  1      3     0.50                        add	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  1      3     0.50                        add	z31.b, z31.b, #255
+# CHECK-NEXT:  1      3     0.50                        add	z31.b, z31.b, z31.b
+# CHECK-NEXT:  1      3     0.50                        add	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  1      3     0.50                        add	z31.d, z31.d, #65280
+# CHECK-NEXT:  1      3     0.50                        add	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      3     0.50                        add	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        add	z31.h, z31.h, #65280
+# CHECK-NEXT:  1      3     0.50                        add	z31.h, z31.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        add	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  1      3     0.50                        add	z31.s, z31.s, #65280
+# CHECK-NEXT:  1      3     0.50                        add	z31.s, z31.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        addhnb	z0.b, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        addhnb	z0.h, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        addhnb	z0.s, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        addhnt	z0.b, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        addhnt	z0.h, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        addhnt	z0.s, z1.d, z31.d
+# CHECK-NEXT:  1      3     0.50                        addp	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      3     0.50                        addp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      3     0.50                        addp	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      3     0.50                        addp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      1     0.33                        addpl	sp, sp, #31
+# CHECK-NEXT:  1      1     0.33                        addpl	x0, x0, #-32
+# CHECK-NEXT:  1      1     0.33                        addpl	x21, x21, #0
+# CHECK-NEXT:  1      1     0.33                        addpl	x23, x8, #-1
+# CHECK-NEXT:  1      1     0.33                        addvl	sp, sp, #31
+# CHECK-NEXT:  1      1     0.33                        addvl	x0, x0, #-32
+# CHECK-NEXT:  1      1     0.33                        addvl	x21, x21, #0
+# CHECK-NEXT:  1      1     0.33                        addvl	x23, x8, #-1
+# CHECK-NEXT:  1      3     0.50                        adr	z0.d, [z0.d, z0.d, lsl #1]
+# CHECK-NEXT:  1      3     0.50                        adr	z0.d, [z0.d, z0.d, lsl #2]
+# CHECK-NEXT:  1      3     0.50                        adr	z0.d, [z0.d, z0.d, lsl #3]
+# CHECK-NEXT:  1      3     0.50                        adr	z0.d, [z0.d, z0.d, sxtw #1]
+# CHECK-NEXT:  1      3     0.50                        adr	z0.d, [z0.d, z0.d, sxtw #2]
+# CHECK-NEXT:  1      3     0.50                        adr	z0.d, [z0.d, z0.d, sxtw #3]
+# CHECK-NEXT:  1      3     0.50                        adr	z0.d, [z0.d, z0.d, sxtw]
+# CHECK-NEXT:  1      3     0.50                        adr	z0.d, [z0.d, z0.d, uxtw #1]
+# CHECK-NEXT:  1      3     0.50                        adr	z0.d, [z0.d, z0.d, uxtw #2]
+# CHECK-NEXT:  1      3     0.50                        adr	z0.d, [z0.d, z0.d, uxtw #3]
+# CHECK-NEXT:  1      3     0.50                        adr	z0.d, [z0.d, z0.d, uxtw]
+# CHECK-NEXT:  1      3     0.50                        adr	z0.d, [z0.d, z0.d]
+# CHECK-NEXT:  1      3     0.50                        adr	z0.s, [z0.s, z0.s, lsl #1]
+# CHECK-NEXT:  1      3     0.50                        adr	z0.s, [z0.s, z0.s, lsl #2]
+# CHECK-NEXT:  1      3     0.50                        adr	z0.s, [z0.s, z0.s, lsl #3]
+# CHECK-NEXT:  1      3     0.50                        adr	z0.s, [z0.s, z0.s]
+# CHECK-NEXT:  1      3     0.50                        aesd	z0.b, z0.b, z31.b
+# CHECK-NEXT:  1      3     0.50                        aese	z0.b, z0.b, z31.b
+# CHECK-NEXT:  1      3     0.50                        aesimc	z0.b, z0.b
+# CHECK-NEXT:  1      3     0.50                        aesimc	z31.b, z31.b
+# CHECK-NEXT:  1      3     0.50                        aesmc	z0.b, z0.b
+# CHECK-NEXT:  1      3     0.50                        aesmc	z31.b, z31.b
+# CHECK-NEXT:  1      6     1.00                        and	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  1      3     0.50                        and	z0.d, z0.d, #0x6
+# CHECK-NEXT:  1      3     0.50                        and	z0.d, z0.d, #0xfffffffffffffff9
+# CHECK-NEXT:  1      3     0.50                        and	z0.d, z0.d, z0.d
+# CHECK-NEXT:  1      3     0.50                        and	z0.s, z0.s, #0x6
+# CHECK-NEXT:  1      3     0.50                        and	z0.s, z0.s, #0xfffffff9
+# CHECK-NEXT:  1      3     0.50                        and	z23.d, z13.d, z8.d
+# CHECK-NEXT:  1      3     0.50                        and	z23.h, z23.h, #0x6
+# CHECK-NEXT:  1      3     0.50                        and	z23.h, z23.h, #0xfff9
+# CHECK-NEXT:  1      3     0.50                        and	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  1      3     0.50                        and	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  1      3     0.50                        and	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        and	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  1      3     0.50                        and	z5.b, z5.b, #0x6
+# CHECK-NEXT:  1      3     0.50                        and	z5.b, z5.b, #0xf9
+# CHECK-NEXT:  1      6     1.00                        ands	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  1      4     1.00                        andv	b0, p7, z31.b
+# CHECK-NEXT:  1      4     1.00                        andv	d0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        andv	h0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        andv	s0, p7, z31.s
+# CHECK-NEXT:  1      3     0.50                        asr	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  1      3     0.50                        asr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  1      3     0.50                        asr	z0.b, p0/m, z0.b, z1.d
+# CHECK-NEXT:  1      3     0.50                        asr	z0.b, z0.b, #1
+# CHECK-NEXT:  1      3     0.50                        asr	z0.b, z1.b, z2.d
+# CHECK-NEXT:  1      3     0.50                        asr	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  1      3     0.50                        asr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      3     0.50                        asr	z0.d, z0.d, #1
+# CHECK-NEXT:  1      3     0.50                        asr	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  1      3     0.50                        asr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  1      3     0.50                        asr	z0.h, p0/m, z0.h, z1.d
+# CHECK-NEXT:  1      3     0.50                        asr	z0.h, z0.h, #1
+# CHECK-NEXT:  1      3     0.50                        asr	z0.h, z1.h, z2.d
+# CHECK-NEXT:  1      3     0.50                        asr	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  1      3     0.50                        asr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  1      3     0.50                        asr	z0.s, p0/m, z0.s, z1.d
+# CHECK-NEXT:  1      3     0.50                        asr	z0.s, z0.s, #1
+# CHECK-NEXT:  1      3     0.50                        asr	z0.s, z1.s, z2.d
+# CHECK-NEXT:  1      3     0.50                        asr	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  1      3     0.50                        asr	z31.b, z31.b, #8
+# CHECK-NEXT:  1      3     0.50                        asr	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  1      3     0.50                        asr	z31.d, z31.d, #64
+# CHECK-NEXT:  1      3     0.50                        asr	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  1      3     0.50                        asr	z31.h, z31.h, #16
+# CHECK-NEXT:  1      3     0.50                        asr	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  1      3     0.50                        asr	z31.s, z31.s, #32
+# CHECK-NEXT:  1      4     0.50                        asrd	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  1      4     0.50                        asrd	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  1      4     0.50                        asrd	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  1      4     0.50                        asrd	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  1      4     0.50                        asrd	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  1      4     0.50                        asrd	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  1      4     0.50                        asrd	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  1      4     0.50                        asrd	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  1      3     0.50                        asrr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  1      3     0.50                        asrr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      3     0.50                        asrr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  1      3     0.50                        asrr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  1      3     0.50                        bcax	z29.d, z29.d, z30.d, z31.d
+# CHECK-NEXT:  1      14    13.00                       bdep	z0.b, z1.b, z31.b
+# CHECK-NEXT:  1      70    69.00                       bdep	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      22    21.00                       bdep	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      38    37.00                       bdep	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      14    13.00                       bext	z0.b, z1.b, z31.b
+# CHECK-NEXT:  1      70    69.00                       bext	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      22    21.00                       bext	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      38    37.00                       bext	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        bfcvt	z0.h, p0/m, z1.s
+# CHECK-NEXT:  1      4     0.50                        bfcvtnt	z0.h, p0/m, z1.s
+# CHECK-NEXT:  2      10    0.50                        bfdot	z0.s, z1.h, z2.h
+# CHECK-NEXT:  2      10    0.50                        bfdot	z0.s, z1.h, z2.h[0]
+# CHECK-NEXT:  2      10    0.50                        bfdot	z0.s, z1.h, z2.h[3]
+# CHECK-NEXT:  1      4     0.50                        bfmlalb	z0.s, z1.h, z2.h
+# CHECK-NEXT:  1      4     0.50                        bfmlalb	z0.s, z1.h, z2.h[0]
+# CHECK-NEXT:  1      4     0.50                        bfmlalb	z0.s, z1.h, z2.h[7]
+# CHECK-NEXT:  1      4     0.50                        bfmlalb	z10.s, z21.h, z14.h
+# CHECK-NEXT:  1      4     0.50                        bfmlalb	z21.s, z14.h, z3.h[2]
+# CHECK-NEXT:  1      4     0.50                        bfmlalt	z0.s, z1.h, z2.h
+# CHECK-NEXT:  1      4     0.50                        bfmlalt	z0.s, z1.h, z2.h[0]
+# CHECK-NEXT:  1      4     0.50                        bfmlalt	z0.s, z1.h, z2.h[7]
+# CHECK-NEXT:  1      4     0.50                        bfmlalt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        bfmlalt	z14.s, z10.h, z21.h
+# CHECK-NEXT:  2      15    0.50                        bfmmla	z0.s, z1.h, z2.h
+# CHECK-NEXT:  1      14    13.00                       bgrp	z0.b, z1.b, z31.b
+# CHECK-NEXT:  1      70    69.00                       bgrp	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      22    21.00                       bgrp	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      38    37.00                       bgrp	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      6     1.00                        bic	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  1      6     1.00                        bic	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      3     0.50                        bic	z0.d, z0.d, z0.d
+# CHECK-NEXT:  1      3     0.50                        bic	z23.d, z13.d, z8.d
+# CHECK-NEXT:  1      3     0.50                        bic	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  1      3     0.50                        bic	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  1      3     0.50                        bic	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        bic	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  1      6     1.00                        bics	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  1      6     1.00                        bics	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      6     1.00                        brka	p0.b, p15/m, p15.b
+# CHECK-NEXT:  1      6     1.00                        brka	p0.b, p15/z, p15.b
+# CHECK-NEXT:  1      6     1.00                        brkas	p0.b, p15/z, p15.b
+# CHECK-NEXT:  1      6     1.00                        brkb	p0.b, p15/m, p15.b
+# CHECK-NEXT:  1      6     1.00                        brkb	p0.b, p15/z, p15.b
+# CHECK-NEXT:  1      6     1.00                        brkbs	p0.b, p15/z, p15.b
+# CHECK-NEXT:  1      6     1.00                        brkn	p0.b, p15/z, p1.b, p0.b
+# CHECK-NEXT:  1      6     1.00                        brkn	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      6     1.00                        brkns	p0.b, p15/z, p1.b, p0.b
+# CHECK-NEXT:  1      6     1.00                        brkns	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      6     1.00                        brkpa	p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  1      6     1.00                        brkpa	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      6     1.00                        brkpas	p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  1      6     1.00                        brkpas	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      6     1.00                        brkpb	p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  1      6     1.00                        brkpb	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      6     1.00                        brkpbs	p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  1      6     1.00                        brkpbs	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      3     0.50                        bsl	z0.d, z0.d, z1.d, z2.d
+# CHECK-NEXT:  1      3     0.50                        bsl1n	z0.d, z0.d, z1.d, z2.d
+# CHECK-NEXT:  1      3     0.50                        bsl2n	z0.d, z0.d, z1.d, z2.d
+# CHECK-NEXT:  1      3     0.50                        cadd	z0.b, z0.b, z0.b, #90
+# CHECK-NEXT:  1      3     0.50                        cadd	z0.d, z0.d, z0.d, #90
+# CHECK-NEXT:  1      3     0.50                        cadd	z0.h, z0.h, z0.h, #90
+# CHECK-NEXT:  1      3     0.50                        cadd	z0.s, z0.s, z0.s, #90
+# CHECK-NEXT:  1      3     0.50                        cadd	z31.b, z31.b, z31.b, #270
+# CHECK-NEXT:  1      3     0.50                        cadd	z31.d, z31.d, z31.d, #270
+# CHECK-NEXT:  1      3     0.50                        cadd	z31.h, z31.h, z31.h, #270
+# CHECK-NEXT:  1      3     0.50                        cadd	z31.s, z31.s, z31.s, #270
+# CHECK-NEXT:  1      4     0.50                        cdot	z0.d, z1.h, z15.h[1], #0
+# CHECK-NEXT:  1      4     0.50                        cdot	z0.d, z1.h, z31.h, #0
+# CHECK-NEXT:  1      4     0.50                        cdot	z0.d, z1.h, z31.h, #180
+# CHECK-NEXT:  1      4     0.50                        cdot	z0.d, z1.h, z31.h, #270
+# CHECK-NEXT:  1      4     0.50                        cdot	z0.d, z1.h, z31.h, #90
+# CHECK-NEXT:  1      4     0.50                        cdot	z0.s, z1.b, z31.b, #0
+# CHECK-NEXT:  1      4     0.50                        cdot	z0.s, z1.b, z7.b[3], #0
+# CHECK-NEXT:  1      4     0.50                        cdot	z29.d, z30.h, z0.h[0], #180
+# CHECK-NEXT:  1      4     0.50                        cdot	z31.d, z30.h, z7.h[1], #270
+# CHECK-NEXT:  1      4     0.50                        cdot	z5.d, z6.h, z3.h[0], #90
+# CHECK-NEXT:  1      4     0.50                        clasta	b0, p7, b0, z31.b
+# CHECK-NEXT:  1      4     0.50                        clasta	d0, p7, d0, z31.d
+# CHECK-NEXT:  1      4     0.50                        clasta	h0, p7, h0, z31.h
+# CHECK-NEXT:  1      4     0.50                        clasta	s0, p7, s0, z31.s
+# CHECK-NEXT:  1      8     1.00                        clasta	w0, p7, w0, z31.b
+# CHECK-NEXT:  1      8     1.00                        clasta	w0, p7, w0, z31.h
+# CHECK-NEXT:  1      8     1.00                        clasta	w0, p7, w0, z31.s
+# CHECK-NEXT:  1      8     1.00                        clasta	x0, p7, x0, z31.d
+# CHECK-NEXT:  1      4     0.50                        clasta	z0.b, p7, z0.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        clasta	z0.d, p7, z0.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        clasta	z0.h, p7, z0.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        clasta	z0.s, p7, z0.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        clastb	b0, p7, b0, z31.b
+# CHECK-NEXT:  1      4     0.50                        clastb	d0, p7, d0, z31.d
+# CHECK-NEXT:  1      4     0.50                        clastb	h0, p7, h0, z31.h
+# CHECK-NEXT:  1      4     0.50                        clastb	s0, p7, s0, z31.s
+# CHECK-NEXT:  1      8     1.00                        clastb	w0, p7, w0, z31.b
+# CHECK-NEXT:  1      8     1.00                        clastb	w0, p7, w0, z31.h
+# CHECK-NEXT:  1      8     1.00                        clastb	w0, p7, w0, z31.s
+# CHECK-NEXT:  1      8     1.00                        clastb	x0, p7, x0, z31.d
+# CHECK-NEXT:  1      4     0.50                        clastb	z0.b, p7, z0.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        clastb	z0.d, p7, z0.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        clastb	z0.h, p7, z0.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        clastb	z0.s, p7, z0.s, z31.s
+# CHECK-NEXT:  1      3     0.50                        cls	z31.b, p7/m, z31.b
+# CHECK-NEXT:  1      3     0.50                        cls	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      3     0.50                        cls	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      3     0.50                        cls	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      3     0.50                        clz	z31.b, p7/m, z31.b
+# CHECK-NEXT:  1      3     0.50                        clz	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      3     0.50                        clz	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      3     0.50                        clz	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     0.50                        cmla	z0.b, z1.b, z2.b, #0
+# CHECK-NEXT:  1      4     0.50                        cmla	z0.d, z1.d, z2.d, #0
+# CHECK-NEXT:  1      4     0.50                        cmla	z0.h, z1.h, z2.h, #0
+# CHECK-NEXT:  1      4     0.50                        cmla	z0.h, z1.h, z2.h[0], #0
+# CHECK-NEXT:  1      4     0.50                        cmla	z0.s, z1.s, z2.s, #0
+# CHECK-NEXT:  1      4     0.50                        cmla	z0.s, z1.s, z2.s[0], #0
+# CHECK-NEXT:  1      4     0.50                        cmla	z15.b, z16.b, z17.b, #270
+# CHECK-NEXT:  1      4     0.50                        cmla	z15.d, z16.d, z17.d, #270
+# CHECK-NEXT:  1      4     0.50                        cmla	z15.h, z16.h, z17.h, #270
+# CHECK-NEXT:  1      4     0.50                        cmla	z15.s, z16.s, z17.s, #270
+# CHECK-NEXT:  1      4     0.50                        cmla	z29.b, z30.b, z31.b, #90
+# CHECK-NEXT:  1      4     0.50                        cmla	z29.d, z30.d, z31.d, #90
+# CHECK-NEXT:  1      4     0.50                        cmla	z29.h, z30.h, z31.h, #90
+# CHECK-NEXT:  1      4     0.50                        cmla	z29.s, z30.s, z31.s, #90
+# CHECK-NEXT:  1      4     0.50                        cmla	z31.b, z31.b, z31.b, #180
+# CHECK-NEXT:  1      4     0.50                        cmla	z31.d, z31.d, z31.d, #180
+# CHECK-NEXT:  1      4     0.50                        cmla	z31.h, z30.h, z7.h[0], #180
+# CHECK-NEXT:  1      4     0.50                        cmla	z31.h, z31.h, z31.h, #180
+# CHECK-NEXT:  1      4     0.50                        cmla	z31.s, z30.s, z7.s[0], #180
+# CHECK-NEXT:  1      4     0.50                        cmla	z31.s, z31.s, z31.s, #180
+# CHECK-NEXT:  1      3     0.50                        cmpeq	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  1      3     0.50                        cmpeq	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  1      3     0.50                        cmpeq	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  1      3     0.50                        cmpeq	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmpeq	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  1      3     0.50                        cmpeq	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  1      3     0.50                        cmpeq	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmpeq	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  1      3     0.50                        cmpeq	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  1      3     0.50                        cmpeq	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmpeq	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  1      3     0.50                        cmpeq	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  1      3     0.50                        cmpeq	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  1      3     0.50                        cmpeq	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmpeq	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  1      3     0.50                        cmpge	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  1      3     0.50                        cmpge	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  1      3     0.50                        cmpge	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  1      3     0.50                        cmpge	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmpge	p0.b, p0/z, z1.b, z0.b
+# CHECK-NEXT:  1      3     0.50                        cmpge	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  1      3     0.50                        cmpge	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  1      3     0.50                        cmpge	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmpge	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmpge	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  1      3     0.50                        cmpge	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  1      3     0.50                        cmpge	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmpge	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  1      3     0.50                        cmpge	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  1      3     0.50                        cmpge	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  1      3     0.50                        cmpge	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  1      3     0.50                        cmpge	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmpge	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  1      3     0.50                        cmpge	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  1      3     0.50                        cmpgt	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  1      3     0.50                        cmpgt	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  1      3     0.50                        cmpgt	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  1      3     0.50                        cmpgt	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmpgt	p0.b, p0/z, z1.b, z0.b
+# CHECK-NEXT:  1      3     0.50                        cmpgt	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  1      3     0.50                        cmpgt	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  1      3     0.50                        cmpgt	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmpgt	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmpgt	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  1      3     0.50                        cmpgt	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  1      3     0.50                        cmpgt	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmpgt	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  1      3     0.50                        cmpgt	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  1      3     0.50                        cmpgt	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  1      3     0.50                        cmpgt	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  1      3     0.50                        cmpgt	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmpgt	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  1      3     0.50                        cmpgt	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  1      3     0.50                        cmphi	p0.b, p0/z, z0.b, #0
+# CHECK-NEXT:  1      3     0.50                        cmphi	p0.b, p0/z, z0.b, #127
+# CHECK-NEXT:  1      3     0.50                        cmphi	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  1      3     0.50                        cmphi	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmphi	p0.b, p0/z, z1.b, z0.b
+# CHECK-NEXT:  1      3     0.50                        cmphi	p0.d, p0/z, z0.d, #0
+# CHECK-NEXT:  1      3     0.50                        cmphi	p0.d, p0/z, z0.d, #127
+# CHECK-NEXT:  1      3     0.50                        cmphi	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmphi	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmphi	p0.h, p0/z, z0.h, #0
+# CHECK-NEXT:  1      3     0.50                        cmphi	p0.h, p0/z, z0.h, #127
+# CHECK-NEXT:  1      3     0.50                        cmphi	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmphi	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  1      3     0.50                        cmphi	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  1      3     0.50                        cmphi	p0.s, p0/z, z0.s, #0
+# CHECK-NEXT:  1      3     0.50                        cmphi	p0.s, p0/z, z0.s, #127
+# CHECK-NEXT:  1      3     0.50                        cmphi	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmphi	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  1      3     0.50                        cmphi	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  1      3     0.50                        cmphs	p0.b, p0/z, z0.b, #0
+# CHECK-NEXT:  1      3     0.50                        cmphs	p0.b, p0/z, z0.b, #127
+# CHECK-NEXT:  1      3     0.50                        cmphs	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  1      3     0.50                        cmphs	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmphs	p0.b, p0/z, z1.b, z0.b
+# CHECK-NEXT:  1      3     0.50                        cmphs	p0.d, p0/z, z0.d, #0
+# CHECK-NEXT:  1      3     0.50                        cmphs	p0.d, p0/z, z0.d, #127
+# CHECK-NEXT:  1      3     0.50                        cmphs	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmphs	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmphs	p0.h, p0/z, z0.h, #0
+# CHECK-NEXT:  1      3     0.50                        cmphs	p0.h, p0/z, z0.h, #127
+# CHECK-NEXT:  1      3     0.50                        cmphs	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmphs	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  1      3     0.50                        cmphs	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  1      3     0.50                        cmphs	p0.s, p0/z, z0.s, #0
+# CHECK-NEXT:  1      3     0.50                        cmphs	p0.s, p0/z, z0.s, #127
+# CHECK-NEXT:  1      3     0.50                        cmphs	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmphs	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  1      3     0.50                        cmphs	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  1      3     0.50                        cmple	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  1      3     0.50                        cmple	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  1      3     0.50                        cmple	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmple	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  1      3     0.50                        cmple	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  1      3     0.50                        cmple	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  1      3     0.50                        cmple	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  1      3     0.50                        cmple	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmple	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  1      3     0.50                        cmple	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  1      3     0.50                        cmple	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmplo	p0.b, p0/z, z0.b, #0
+# CHECK-NEXT:  1      3     0.50                        cmplo	p0.b, p0/z, z0.b, #127
+# CHECK-NEXT:  1      3     0.50                        cmplo	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmplo	p0.d, p0/z, z0.d, #0
+# CHECK-NEXT:  1      3     0.50                        cmplo	p0.d, p0/z, z0.d, #127
+# CHECK-NEXT:  1      3     0.50                        cmplo	p0.h, p0/z, z0.h, #0
+# CHECK-NEXT:  1      3     0.50                        cmplo	p0.h, p0/z, z0.h, #127
+# CHECK-NEXT:  1      3     0.50                        cmplo	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmplo	p0.s, p0/z, z0.s, #0
+# CHECK-NEXT:  1      3     0.50                        cmplo	p0.s, p0/z, z0.s, #127
+# CHECK-NEXT:  1      3     0.50                        cmplo	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmpls	p0.b, p0/z, z0.b, #0
+# CHECK-NEXT:  1      3     0.50                        cmpls	p0.b, p0/z, z0.b, #127
+# CHECK-NEXT:  1      3     0.50                        cmpls	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmpls	p0.d, p0/z, z0.d, #0
+# CHECK-NEXT:  1      3     0.50                        cmpls	p0.d, p0/z, z0.d, #127
+# CHECK-NEXT:  1      3     0.50                        cmpls	p0.h, p0/z, z0.h, #0
+# CHECK-NEXT:  1      3     0.50                        cmpls	p0.h, p0/z, z0.h, #127
+# CHECK-NEXT:  1      3     0.50                        cmpls	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmpls	p0.s, p0/z, z0.s, #0
+# CHECK-NEXT:  1      3     0.50                        cmpls	p0.s, p0/z, z0.s, #127
+# CHECK-NEXT:  1      3     0.50                        cmpls	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmplt	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  1      3     0.50                        cmplt	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  1      3     0.50                        cmplt	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmplt	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  1      3     0.50                        cmplt	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  1      3     0.50                        cmplt	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  1      3     0.50                        cmplt	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  1      3     0.50                        cmplt	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmplt	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  1      3     0.50                        cmplt	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  1      3     0.50                        cmplt	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmpne	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  1      3     0.50                        cmpne	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  1      3     0.50                        cmpne	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  1      3     0.50                        cmpne	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmpne	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  1      3     0.50                        cmpne	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  1      3     0.50                        cmpne	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmpne	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  1      3     0.50                        cmpne	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  1      3     0.50                        cmpne	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmpne	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  1      3     0.50                        cmpne	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  1      3     0.50                        cmpne	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  1      3     0.50                        cmpne	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  1      3     0.50                        cmpne	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  1      3     0.50                        cnot	z31.b, p7/m, z31.b
+# CHECK-NEXT:  1      3     0.50                        cnot	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      3     0.50                        cnot	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      3     0.50                        cnot	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     0.50                        cnt	z31.b, p7/m, z31.b
+# CHECK-NEXT:  1      12    0.50                        cnt	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     0.50                        cnt	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      8     0.50                        cnt	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      1     0.33                        cntb	x0
+# CHECK-NEXT:  1      1     0.33                        cntb	x0, #28
+# CHECK-NEXT:  1      1     0.33                        cntb	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        cntb	x0, pow2
+# CHECK-NEXT:  1      1     0.33                        cntd	x0
+# CHECK-NEXT:  1      1     0.33                        cntd	x0, #28
+# CHECK-NEXT:  1      1     0.33                        cntd	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        cntd	x0, pow2
+# CHECK-NEXT:  1      1     0.33                        cnth	x0
+# CHECK-NEXT:  1      1     0.33                        cnth	x0, #28
+# CHECK-NEXT:  1      1     0.33                        cnth	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        cnth	x0, pow2
+# CHECK-NEXT:  1      6     1.00                        cntp	x0, p15, p0.b
+# CHECK-NEXT:  1      6     1.00                        cntp	x0, p15, p0.d
+# CHECK-NEXT:  1      6     1.00                        cntp	x0, p15, p0.h
+# CHECK-NEXT:  1      6     1.00                        cntp	x0, p15, p0.s
+# CHECK-NEXT:  1      1     0.33                        cntw	x0
+# CHECK-NEXT:  1      1     0.33                        cntw	x0, #28
+# CHECK-NEXT:  1      1     0.33                        cntw	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        cntw	x0, pow2
+# CHECK-NEXT:  1      4     0.50                        compact	z31.d, p7, z31.d
+# CHECK-NEXT:  1      4     0.50                        compact	z31.s, p7, z31.s
+# CHECK-NEXT:  1      1     0.33                        ctermeq	w30, wzr
+# CHECK-NEXT:  1      1     0.33                        ctermeq	wzr, w30
+# CHECK-NEXT:  1      1     0.33                        ctermeq	x30, xzr
+# CHECK-NEXT:  1      1     0.33                        ctermeq	xzr, x30
+# CHECK-NEXT:  1      1     0.33                        ctermne	w30, wzr
+# CHECK-NEXT:  1      1     0.33                        ctermne	wzr, w30
+# CHECK-NEXT:  1      1     0.33                        ctermne	x30, xzr
+# CHECK-NEXT:  1      1     0.33                        ctermne	xzr, x30
+# CHECK-NEXT:  1      1     0.33                        decb	x0
+# CHECK-NEXT:  1      1     0.33                        decb	x0, #14
+# CHECK-NEXT:  1      1     0.33                        decb	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        decb	x0, pow2
+# CHECK-NEXT:  1      1     0.33                        decb	x0, vl1
+# CHECK-NEXT:  1      1     0.33                        decd	x0
+# CHECK-NEXT:  1      1     0.33                        decd	x0, #14
+# CHECK-NEXT:  1      1     0.33                        decd	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        decd	x0, pow2
+# CHECK-NEXT:  1      1     0.33                        decd	x0, vl1
+# CHECK-NEXT:  1      1     0.33                        dech	x0
+# CHECK-NEXT:  1      1     0.33                        dech	x0, #14
+# CHECK-NEXT:  1      1     0.33                        dech	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        dech	x0, pow2
+# CHECK-NEXT:  1      1     0.33                        dech	x0, vl1
+# CHECK-NEXT:  1      6     1.00                        decp	x0, p0.b
+# CHECK-NEXT:  1      6     1.00                        decp	x0, p0.d
+# CHECK-NEXT:  1      6     1.00                        decp	x0, p0.h
+# CHECK-NEXT:  1      6     1.00                        decp	x0, p0.s
+# CHECK-NEXT:  1      6     1.00                        decp	xzr, p15.b
+# CHECK-NEXT:  1      6     1.00                        decp	xzr, p15.d
+# CHECK-NEXT:  1      6     1.00                        decp	xzr, p15.h
+# CHECK-NEXT:  1      6     1.00                        decp	xzr, p15.s
+# CHECK-NEXT:  1      4     0.50                        decp	z31.d, p15.d
+# CHECK-NEXT:  1      4     0.50                        decp	z31.h, p15.h
+# CHECK-NEXT:  1      4     0.50                        decp	z31.s, p15.s
+# CHECK-NEXT:  1      1     0.33                        decw	x0
+# CHECK-NEXT:  1      1     0.33                        decw	x0, #14
+# CHECK-NEXT:  1      1     0.33                        decw	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        decw	x0, pow2
+# CHECK-NEXT:  1      1     0.33                        decw	x0, vl1
+# CHECK-NEXT:  1      4     0.50                        dupm	z0.d, #0xfffffffffffffff9
+# CHECK-NEXT:  1      4     0.50                        dupm	z0.s, #0xfffffff9
+# CHECK-NEXT:  1      4     0.50                        dupm	z23.h, #0xfff9
+# CHECK-NEXT:  1      4     0.50                        dupm	z5.b, #0xf9
+# CHECK-NEXT:  1      6     1.00                        eor	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  1      3     0.50                        eor	z0.d, z0.d, #0x6
+# CHECK-NEXT:  1      3     0.50                        eor	z0.d, z0.d, #0xfffffffffffffff9
+# CHECK-NEXT:  1      3     0.50                        eor	z0.d, z0.d, z0.d
+# CHECK-NEXT:  1      3     0.50                        eor	z0.s, z0.s, #0x6
+# CHECK-NEXT:  1      3     0.50                        eor	z0.s, z0.s, #0xfffffff9
+# CHECK-NEXT:  1      3     0.50                        eor	z23.d, z13.d, z8.d
+# CHECK-NEXT:  1      3     0.50                        eor	z23.h, z23.h, #0x6
+# CHECK-NEXT:  1      3     0.50                        eor	z23.h, z23.h, #0xfff9
+# CHECK-NEXT:  1      3     0.50                        eor	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  1      3     0.50                        eor	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  1      3     0.50                        eor	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        eor	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  1      3     0.50                        eor	z5.b, z5.b, #0x6
+# CHECK-NEXT:  1      3     0.50                        eor	z5.b, z5.b, #0xf9
+# CHECK-NEXT:  1      3     0.50                        eor3	z29.d, z29.d, z30.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        eorbt	z0.b, z1.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        eorbt	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        eorbt	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        eorbt	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      6     1.00                        eors	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  1      4     0.50                        eortb	z0.b, z1.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        eortb	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        eortb	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        eortb	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        eorv	b0, p7, z31.b
+# CHECK-NEXT:  1      4     1.00                        eorv	d0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        eorv	h0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        eorv	s0, p7, z31.s
+# CHECK-NEXT:  1      3     0.50                        ext	z0.b, { z1.b, z2.b }, #0
+# CHECK-NEXT:  1      3     0.50                        ext	z31.b, z31.b, z0.b, #0
+# CHECK-NEXT:  1      3     0.50                        ext	z31.b, z31.b, z0.b, #255
+# CHECK-NEXT:  1      3     0.50                        ext	z31.b, { z30.b, z31.b }, #255
+# CHECK-NEXT:  1      4     0.50                        fabd	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        fabd	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fabd	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        fabs	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     0.50                        fabs	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     0.50                        fabs	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     0.50                        facge	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  1      4     0.50                        facge	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  1      4     0.50                        facge	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        facge	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  1      4     0.50                        facge	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  1      4     0.50                        facge	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  1      4     0.50                        facgt	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  1      4     0.50                        facgt	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  1      4     0.50                        facgt	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        facgt	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  1      4     0.50                        facgt	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  1      4     0.50                        facgt	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  1      4     0.50                        fadd	z0.d, p0/m, z0.d, #0.5
+# CHECK-NEXT:  1      4     0.50                        fadd	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        fadd	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        fadd	z0.h, p0/m, z0.h, #0.5
+# CHECK-NEXT:  1      4     0.50                        fadd	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fadd	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fadd	z0.s, p0/m, z0.s, #0.5
+# CHECK-NEXT:  1      4     0.50                        fadd	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        fadd	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        fadd	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  1      4     0.50                        fadd	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  1      4     0.50                        fadd	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  1      8     2.50                        fadda	d0, p7, d0, z31.d
+# CHECK-NEXT:  1      32    14.50                       fadda	h0, p7, h0, z31.h
+# CHECK-NEXT:  1      16    6.50                        fadda	s0, p7, s0, z31.s
+# CHECK-NEXT:  1      4     0.50                        faddp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        faddp	z29.s, p3/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        faddp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        faddv	d0, p7, z31.d
+# CHECK-NEXT:  1      12    11.00                       faddv	h0, p7, z31.h
+# CHECK-NEXT:  1      8     5.00                        faddv	s0, p7, z31.s
+# CHECK-NEXT:  1      4     0.50                        fcadd	z0.d, p0/m, z0.d, z0.d, #90
+# CHECK-NEXT:  1      4     0.50                        fcadd	z0.h, p0/m, z0.h, z0.h, #90
+# CHECK-NEXT:  1      4     0.50                        fcadd	z0.s, p0/m, z0.s, z0.s, #90
+# CHECK-NEXT:  1      4     0.50                        fcadd	z31.d, p7/m, z31.d, z31.d, #270
+# CHECK-NEXT:  1      4     0.50                        fcadd	z31.h, p7/m, z31.h, z31.h, #270
+# CHECK-NEXT:  1      4     0.50                        fcadd	z31.s, p7/m, z31.s, z31.s, #270
+# CHECK-NEXT:  1      4     0.50                        fcmeq	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmeq	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  1      4     0.50                        fcmeq	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmeq	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        fcmeq	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmeq	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  1      4     0.50                        fcmge	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmge	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  1      4     0.50                        fcmge	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  1      4     0.50                        fcmge	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmge	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        fcmge	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  1      4     0.50                        fcmge	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmge	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  1      4     0.50                        fcmge	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  1      4     0.50                        fcmgt	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmgt	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  1      4     0.50                        fcmgt	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  1      4     0.50                        fcmgt	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmgt	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        fcmgt	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  1      4     0.50                        fcmgt	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmgt	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  1      4     0.50                        fcmgt	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  1      4     0.50                        fcmla	z0.d, p0/m, z0.d, z0.d, #0
+# CHECK-NEXT:  1      4     0.50                        fcmla	z0.d, p0/m, z1.d, z2.d, #90
+# CHECK-NEXT:  1      4     0.50                        fcmla	z0.h, p0/m, z0.h, z0.h, #0
+# CHECK-NEXT:  1      4     0.50                        fcmla	z0.h, p0/m, z1.h, z2.h, #90
+# CHECK-NEXT:  1      4     0.50                        fcmla	z0.h, z0.h, z0.h[0], #0
+# CHECK-NEXT:  1      4     0.50                        fcmla	z0.s, p0/m, z0.s, z0.s, #0
+# CHECK-NEXT:  1      4     0.50                        fcmla	z0.s, p0/m, z1.s, z2.s, #90
+# CHECK-NEXT:  1      4     0.50                        fcmla	z21.s, z10.s, z5.s[1], #90
+# CHECK-NEXT:  1      4     0.50                        fcmla	z23.s, z13.s, z8.s[0], #270
+# CHECK-NEXT:  1      4     0.50                        fcmla	z29.d, p7/m, z30.d, z31.d, #180
+# CHECK-NEXT:  1      4     0.50                        fcmla	z29.h, p7/m, z30.h, z31.h, #180
+# CHECK-NEXT:  1      4     0.50                        fcmla	z29.s, p7/m, z30.s, z31.s, #180
+# CHECK-NEXT:  1      4     0.50                        fcmla	z31.d, p7/m, z31.d, z31.d, #270
+# CHECK-NEXT:  1      4     0.50                        fcmla	z31.h, p7/m, z31.h, z31.h, #270
+# CHECK-NEXT:  1      4     0.50                        fcmla	z31.h, z31.h, z7.h[3], #270
+# CHECK-NEXT:  1      4     0.50                        fcmla	z31.s, p7/m, z31.s, z31.s, #270
+# CHECK-NEXT:  1      4     0.50                        fcmle	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmle	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmle	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmlt	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmlt	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmlt	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmne	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmne	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  1      4     0.50                        fcmne	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmne	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        fcmne	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  1      4     0.50                        fcmne	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  1      4     0.50                        fcmuo	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  1      4     0.50                        fcmuo	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        fcmuo	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  1      4     0.50                        fcvt	z0.d, p0/m, z0.h
+# CHECK-NEXT:  1      4     0.50                        fcvt	z0.d, p0/m, z0.s
+# CHECK-NEXT:  1      4     0.50                        fcvt	z0.h, p0/m, z0.d
+# CHECK-NEXT:  1      4     0.50                        fcvt	z0.h, p0/m, z0.s
+# CHECK-NEXT:  1      4     0.50                        fcvt	z0.s, p0/m, z0.d
+# CHECK-NEXT:  1      4     0.50                        fcvt	z0.s, p0/m, z0.h
+# CHECK-NEXT:  1      4     0.50                        fcvtlt	z0.s, p0/m, z1.h
+# CHECK-NEXT:  1      4     0.50                        fcvtlt	z30.d, p7/m, z31.s
+# CHECK-NEXT:  1      4     0.50                        fcvtnt	z0.h, p0/m, z1.s
+# CHECK-NEXT:  1      4     0.50                        fcvtnt	z30.s, p7/m, z31.d
+# CHECK-NEXT:  1      4     0.50                        fcvtx	z0.s, p0/m, z0.d
+# CHECK-NEXT:  1      4     0.50                        fcvtx	z30.s, p7/m, z31.d
+# CHECK-NEXT:  1      4     0.50                        fcvtxnt	z0.s, p0/m, z1.d
+# CHECK-NEXT:  1      4     0.50                        fcvtxnt	z30.s, p7/m, z31.d
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	z0.d, p0/m, z0.d
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	z0.d, p0/m, z0.h
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	z0.d, p0/m, z0.s
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	z0.h, p0/m, z0.h
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	z0.s, p0/m, z0.d
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	z0.s, p0/m, z0.h
+# CHECK-NEXT:  1      4     0.50                        fcvtzs	z0.s, p0/m, z0.s
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	z0.d, p0/m, z0.d
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	z0.d, p0/m, z0.h
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	z0.d, p0/m, z0.s
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	z0.h, p0/m, z0.h
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	z0.s, p0/m, z0.d
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	z0.s, p0/m, z0.h
+# CHECK-NEXT:  1      4     0.50                        fcvtzu	z0.s, p0/m, z0.s
+# CHECK-NEXT:  1      22    19.00                       fdiv	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      8     5.00                        fdiv	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      13    10.00                       fdiv	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      22    19.00                       fdivr	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      8     5.00                        fdivr	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      13    10.00                       fdivr	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        fexpa	z0.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        fexpa	z0.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fexpa	z0.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        flogb	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     0.50                        flogb	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     0.50                        flogb	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     0.50                        fmad	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        fmad	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fmad	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        fmax	z0.d, p0/m, z0.d, #0.0
+# CHECK-NEXT:  1      4     0.50                        fmax	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        fmax	z0.h, p0/m, z0.h, #0.0
+# CHECK-NEXT:  1      4     0.50                        fmax	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fmax	z0.s, p0/m, z0.s, #0.0
+# CHECK-NEXT:  1      4     0.50                        fmax	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        fmax	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  1      4     0.50                        fmax	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  1      4     0.50                        fmax	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  1      4     0.50                        fmaxnm	z0.d, p0/m, z0.d, #0.0
+# CHECK-NEXT:  1      4     0.50                        fmaxnm	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        fmaxnm	z0.h, p0/m, z0.h, #0.0
+# CHECK-NEXT:  1      4     0.50                        fmaxnm	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fmaxnm	z0.s, p0/m, z0.s, #0.0
+# CHECK-NEXT:  1      4     0.50                        fmaxnm	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        fmaxnm	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  1      4     0.50                        fmaxnm	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  1      4     0.50                        fmaxnm	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  1      4     0.50                        fmaxnmp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        fmaxnmp	z29.s, p3/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        fmaxnmp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        fmaxnmv	d0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        fmaxnmv	h0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        fmaxnmv	s0, p7, z31.s
+# CHECK-NEXT:  1      4     0.50                        fmaxp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        fmaxp	z29.s, p3/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        fmaxp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        fmaxv	d0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        fmaxv	h0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        fmaxv	s0, p7, z31.s
+# CHECK-NEXT:  1      4     0.50                        fmin	z0.d, p0/m, z0.d, #0.0
+# CHECK-NEXT:  1      4     0.50                        fmin	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        fmin	z0.h, p0/m, z0.h, #0.0
+# CHECK-NEXT:  1      4     0.50                        fmin	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fmin	z0.s, p0/m, z0.s, #0.0
+# CHECK-NEXT:  1      4     0.50                        fmin	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        fmin	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  1      4     0.50                        fmin	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  1      4     0.50                        fmin	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  1      4     0.50                        fminnm	z0.d, p0/m, z0.d, #0.0
+# CHECK-NEXT:  1      4     0.50                        fminnm	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        fminnm	z0.h, p0/m, z0.h, #0.0
+# CHECK-NEXT:  1      4     0.50                        fminnm	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fminnm	z0.s, p0/m, z0.s, #0.0
+# CHECK-NEXT:  1      4     0.50                        fminnm	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        fminnm	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  1      4     0.50                        fminnm	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  1      4     0.50                        fminnm	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  1      4     0.50                        fminnmp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        fminnmp	z29.s, p3/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        fminnmp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        fminnmv	d0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        fminnmv	h0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        fminnmv	s0, p7, z31.s
+# CHECK-NEXT:  1      4     0.50                        fminp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        fminp	z29.s, p3/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        fminp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        fminv	d0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        fminv	h0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        fminv	s0, p7, z31.s
+# CHECK-NEXT:  1      4     0.50                        fmla	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        fmla	z0.d, z1.d, z7.d[1]
+# CHECK-NEXT:  1      4     0.50                        fmla	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fmla	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        fmla	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        fmla	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  1      4     0.50                        fmlalb	z0.s, z1.h, z7.h[0]
+# CHECK-NEXT:  1      4     0.50                        fmlalb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fmlalb	z30.s, z31.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        fmlalt	z0.s, z1.h, z7.h[0]
+# CHECK-NEXT:  1      4     0.50                        fmlalt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fmlalt	z30.s, z31.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        fmls	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        fmls	z0.d, z1.d, z7.d[1]
+# CHECK-NEXT:  1      4     0.50                        fmls	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fmls	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        fmls	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        fmls	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  1      4     0.50                        fmlslb	z0.s, z1.h, z7.h[0]
+# CHECK-NEXT:  1      4     0.50                        fmlslb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fmlslb	z30.s, z31.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        fmlslt	z0.s, z1.h, z7.h[0]
+# CHECK-NEXT:  1      4     0.50                        fmlslt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fmlslt	z30.s, z31.h, z7.h[7]
+# CHECK-NEXT:  1      3     1.00                        fmov	z0.d, #-10.00000000
+# CHECK-NEXT:  1      3     1.00                        fmov	z0.d, #0.12500000
+# CHECK-NEXT:  1      3     1.00                        fmov	z0.d, p0/m, #-10.00000000
+# CHECK-NEXT:  1      3     1.00                        fmov	z0.d, p0/m, #0.12500000
+# CHECK-NEXT:  1      3     1.00                        fmov	z0.h, #-0.12500000
+# CHECK-NEXT:  1      3     1.00                        fmov	z0.h, p0/m, #-0.12500000
+# CHECK-NEXT:  1      3     1.00                        fmov	z0.s, #-0.12500000
+# CHECK-NEXT:  1      3     1.00                        fmov	z0.s, p0/m, #-0.12500000
+# CHECK-NEXT:  1      4     0.50                        fmsb	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        fmsb	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fmsb	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        fmul	z0.d, p0/m, z0.d, #0.5
+# CHECK-NEXT:  1      4     0.50                        fmul	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        fmul	z0.d, z0.d, z0.d[0]
+# CHECK-NEXT:  1      4     0.50                        fmul	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        fmul	z0.h, p0/m, z0.h, #0.5
+# CHECK-NEXT:  1      4     0.50                        fmul	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fmul	z0.h, z0.h, z0.h[0]
+# CHECK-NEXT:  1      4     0.50                        fmul	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fmul	z0.s, p0/m, z0.s, #0.5
+# CHECK-NEXT:  1      4     0.50                        fmul	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        fmul	z0.s, z0.s, z0.s[0]
+# CHECK-NEXT:  1      4     0.50                        fmul	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        fmul	z31.d, p7/m, z31.d, #2.0
+# CHECK-NEXT:  1      4     0.50                        fmul	z31.d, z31.d, z15.d[1]
+# CHECK-NEXT:  1      4     0.50                        fmul	z31.h, p7/m, z31.h, #2.0
+# CHECK-NEXT:  1      4     0.50                        fmul	z31.h, z31.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        fmul	z31.s, p7/m, z31.s, #2.0
+# CHECK-NEXT:  1      4     0.50                        fmul	z31.s, z31.s, z7.s[3]
+# CHECK-NEXT:  1      4     0.50                        fmulx	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        fmulx	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fmulx	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        fneg	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     0.50                        fneg	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     0.50                        fneg	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     0.50                        fnmad	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        fnmad	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fnmad	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        fnmla	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        fnmla	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fnmla	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        fnmls	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        fnmls	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fnmls	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        fnmsb	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        fnmsb	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fnmsb	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        frecpe	z0.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        frecpe	z0.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        frecpe	z0.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        frecps	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        frecps	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        frecps	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        frecpx	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     0.50                        frecpx	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     0.50                        frecpx	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     0.50                        frinta	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     0.50                        frinta	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     0.50                        frinta	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     0.50                        frinti	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     0.50                        frinti	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     0.50                        frinti	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     0.50                        frintm	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     0.50                        frintm	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     0.50                        frintm	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     0.50                        frintn	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     0.50                        frintn	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     0.50                        frintn	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     0.50                        frintp	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     0.50                        frintp	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     0.50                        frintp	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     0.50                        frintx	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     0.50                        frintx	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     0.50                        frintx	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     0.50                        frintz	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     0.50                        frintz	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     0.50                        frintz	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     0.50                        frsqrte	z0.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        frsqrte	z0.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        frsqrte	z0.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        frsqrts	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        frsqrts	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        frsqrts	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        fscale	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        fscale	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fscale	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      22    19.00                       fsqrt	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      8     5.00                        fsqrt	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      12    9.00                        fsqrt	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     0.50                        fsub	z0.d, p0/m, z0.d, #0.5
+# CHECK-NEXT:  1      4     0.50                        fsub	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        fsub	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        fsub	z0.h, p0/m, z0.h, #0.5
+# CHECK-NEXT:  1      4     0.50                        fsub	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fsub	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fsub	z0.s, p0/m, z0.s, #0.5
+# CHECK-NEXT:  1      4     0.50                        fsub	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        fsub	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        fsub	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  1      4     0.50                        fsub	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  1      4     0.50                        fsub	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  1      4     0.50                        fsubr	z0.d, p0/m, z0.d, #0.5
+# CHECK-NEXT:  1      4     0.50                        fsubr	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        fsubr	z0.h, p0/m, z0.h, #0.5
+# CHECK-NEXT:  1      4     0.50                        fsubr	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        fsubr	z0.s, p0/m, z0.s, #0.5
+# CHECK-NEXT:  1      4     0.50                        fsubr	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        fsubr	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  1      4     0.50                        fsubr	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  1      4     0.50                        fsubr	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  1      4     0.50                        ftmad	z0.d, z0.d, z31.d, #7
+# CHECK-NEXT:  1      4     0.50                        ftmad	z0.h, z0.h, z31.h, #7
+# CHECK-NEXT:  1      4     0.50                        ftmad	z0.s, z0.s, z31.s, #7
+# CHECK-NEXT:  1      4     0.50                        ftsmul	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        ftsmul	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        ftsmul	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        ftssel	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        ftssel	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        ftssel	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      8     2.00                        histcnt	z0.s, p0/z, z1.s, z2.s
+# CHECK-NEXT:  1      8     2.00                        histcnt	z29.d, p7/z, z30.d, z31.d
+# CHECK-NEXT:  1      8     2.00                        histseg	z0.b, z1.b, z31.b
+# CHECK-NEXT:  1      1     0.33                        incb	x0
+# CHECK-NEXT:  1      1     0.33                        incb	x0, #14
+# CHECK-NEXT:  1      1     0.33                        incb	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        incb	x0, pow2
+# CHECK-NEXT:  1      1     0.33                        incb	x0, vl1
+# CHECK-NEXT:  1      1     0.33                        incd	x0
+# CHECK-NEXT:  1      1     0.33                        incd	x0, #14
+# CHECK-NEXT:  1      1     0.33                        incd	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        incd	x0, pow2
+# CHECK-NEXT:  1      1     0.33                        incd	x0, vl1
+# CHECK-NEXT:  1      4     0.50                        incd	z0.d
+# CHECK-NEXT:  1      4     0.50                        incd	z0.d, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        inch	x0
+# CHECK-NEXT:  1      1     0.33                        inch	x0, #14
+# CHECK-NEXT:  1      1     0.33                        inch	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        inch	x0, pow2
+# CHECK-NEXT:  1      1     0.33                        inch	x0, vl1
+# CHECK-NEXT:  1      4     0.50                        inch	z0.h
+# CHECK-NEXT:  1      4     0.50                        inch	z0.h, all, mul #16
+# CHECK-NEXT:  1      6     1.00                        incp	x0, p0.b
+# CHECK-NEXT:  1      6     1.00                        incp	x0, p0.d
+# CHECK-NEXT:  1      6     1.00                        incp	x0, p0.h
+# CHECK-NEXT:  1      6     1.00                        incp	x0, p0.s
+# CHECK-NEXT:  1      6     1.00                        incp	xzr, p15.b
+# CHECK-NEXT:  1      6     1.00                        incp	xzr, p15.d
+# CHECK-NEXT:  1      6     1.00                        incp	xzr, p15.h
+# CHECK-NEXT:  1      6     1.00                        incp	xzr, p15.s
+# CHECK-NEXT:  1      4     0.50                        incp	z31.d, p15.d
+# CHECK-NEXT:  1      4     0.50                        incp	z31.h, p15.h
+# CHECK-NEXT:  1      4     0.50                        incp	z31.s, p15.s
+# CHECK-NEXT:  1      1     0.33                        incw	x0
+# CHECK-NEXT:  1      1     0.33                        incw	x0, #14
+# CHECK-NEXT:  1      1     0.33                        incw	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        incw	x0, pow2
+# CHECK-NEXT:  1      1     0.33                        incw	x0, vl1
+# CHECK-NEXT:  1      4     0.50                        incw	z0.s
+# CHECK-NEXT:  1      4     0.50                        incw	z0.s, all, mul #16
+# CHECK-NEXT:  1      4     0.50                        index	z0.b, #0, #0
+# CHECK-NEXT:  1      4     0.50                        index	z0.d, #0, #0
+# CHECK-NEXT:  1      4     0.50                        index	z0.h, #0, #0
+# CHECK-NEXT:  1      4     0.50                        index	z0.h, w0, w0
+# CHECK-NEXT:  1      4     0.50                        index	z0.s, #0, #0
+# CHECK-NEXT:  1      4     0.50                        index	z21.b, w10, w21
+# CHECK-NEXT:  1      4     0.50                        index	z21.d, x10, x21
+# CHECK-NEXT:  1      4     0.50                        index	z21.s, w10, w21
+# CHECK-NEXT:  1      4     0.50                        index	z23.b, #13, w8
+# CHECK-NEXT:  1      4     0.50                        index	z23.b, w13, #8
+# CHECK-NEXT:  1      4     0.50                        index	z23.d, #13, x8
+# CHECK-NEXT:  1      4     0.50                        index	z23.d, x13, #8
+# CHECK-NEXT:  1      4     0.50                        index	z23.h, #13, w8
+# CHECK-NEXT:  1      4     0.50                        index	z23.h, w13, #8
+# CHECK-NEXT:  1      4     0.50                        index	z23.s, #13, w8
+# CHECK-NEXT:  1      4     0.50                        index	z23.s, w13, #8
+# CHECK-NEXT:  1      4     0.50                        index	z31.b, #-1, #-1
+# CHECK-NEXT:  1      4     0.50                        index	z31.b, #-1, wzr
+# CHECK-NEXT:  1      4     0.50                        index	z31.b, wzr, #-1
+# CHECK-NEXT:  1      4     0.50                        index	z31.b, wzr, wzr
+# CHECK-NEXT:  1      4     0.50                        index	z31.d, #-1, #-1
+# CHECK-NEXT:  1      4     0.50                        index	z31.d, #-1, xzr
+# CHECK-NEXT:  1      4     0.50                        index	z31.d, xzr, #-1
+# CHECK-NEXT:  1      4     0.50                        index	z31.d, xzr, xzr
+# CHECK-NEXT:  1      4     0.50                        index	z31.h, #-1, #-1
+# CHECK-NEXT:  1      4     0.50                        index	z31.h, #-1, wzr
+# CHECK-NEXT:  1      4     0.50                        index	z31.h, wzr, #-1
+# CHECK-NEXT:  1      4     0.50                        index	z31.h, wzr, wzr
+# CHECK-NEXT:  1      4     0.50                        index	z31.s, #-1, #-1
+# CHECK-NEXT:  1      4     0.50                        index	z31.s, #-1, wzr
+# CHECK-NEXT:  1      4     0.50                        index	z31.s, wzr, #-1
+# CHECK-NEXT:  1      4     0.50                        index	z31.s, wzr, wzr
+# CHECK-NEXT:  1      8     2.00                        insr	z0.b, w0
+# CHECK-NEXT:  1      8     2.00                        insr	z0.d, x0
+# CHECK-NEXT:  1      8     2.00                        insr	z0.h, w0
+# CHECK-NEXT:  1      8     2.00                        insr	z0.s, w0
+# CHECK-NEXT:  1      4     0.50                        insr	z31.b, b31
+# CHECK-NEXT:  1      8     2.00                        insr	z31.b, wzr
+# CHECK-NEXT:  1      4     0.50                        insr	z31.d, d31
+# CHECK-NEXT:  1      8     2.00                        insr	z31.d, xzr
+# CHECK-NEXT:  1      4     0.50                        insr	z31.h, h31
+# CHECK-NEXT:  1      8     2.00                        insr	z31.h, wzr
+# CHECK-NEXT:  1      4     0.50                        insr	z31.s, s31
+# CHECK-NEXT:  1      8     2.00                        insr	z31.s, wzr
+# CHECK-NEXT:  1      4     0.50                        lasta	b0, p7, z31.b
+# CHECK-NEXT:  1      4     0.50                        lasta	d0, p7, z31.d
+# CHECK-NEXT:  1      4     0.50                        lasta	h0, p7, z31.h
+# CHECK-NEXT:  1      4     0.50                        lasta	s0, p7, z31.s
+# CHECK-NEXT:  1      8     2.00                        lasta	w0, p7, z31.b
+# CHECK-NEXT:  1      8     2.00                        lasta	w0, p7, z31.h
+# CHECK-NEXT:  1      8     2.00                        lasta	w0, p7, z31.s
+# CHECK-NEXT:  1      8     2.00                        lasta	x0, p7, z31.d
+# CHECK-NEXT:  1      4     0.50                        lastb	b0, p7, z31.b
+# CHECK-NEXT:  1      4     0.50                        lastb	d0, p7, z31.d
+# CHECK-NEXT:  1      4     0.50                        lastb	h0, p7, z31.h
+# CHECK-NEXT:  1      4     0.50                        lastb	s0, p7, z31.s
+# CHECK-NEXT:  1      8     2.00                        lastb	w0, p7, z31.b
+# CHECK-NEXT:  1      8     2.00                        lastb	w0, p7, z31.h
+# CHECK-NEXT:  1      8     2.00                        lastb	w0, p7, z31.s
+# CHECK-NEXT:  1      8     2.00                        lastb	x0, p7, z31.d
+# CHECK-NEXT:  1      3     0.50    *                   ld1b	{ z0.b }, p0/z, [sp, x0]
+# CHECK-NEXT:  1      3     0.50    *                   ld1b	{ z0.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  1      3     0.50    *                   ld1b	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *                   ld1b	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      7     7.00    *                   ld1b	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      3     0.50    *                   ld1b	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      9     4.50    *                   ld1b	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      9     4.50    *                   ld1b	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      3     0.50    *                   ld1b	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      9     9.00    *                   ld1b	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      3     0.50    *                   ld1b	{ z21.b }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     0.50    *                   ld1b	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1b	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *                   ld1b	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      3     0.50    *                   ld1b	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     0.50    *                   ld1b	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     0.50    *                   ld1b	{ z21.s }, p5/z, [x10, x21]
+# CHECK-NEXT:  1      3     0.50    *                   ld1b	{ z23.d }, p3/z, [x13, x8]
+# CHECK-NEXT:  1      3     0.50    *                   ld1b	{ z31.b }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     0.50    *                   ld1b	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1b	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      7     7.00    *                   ld1b	{ z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  1      3     0.50    *                   ld1b	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     0.50    *                   ld1b	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      9     9.00    *                   ld1b	{ z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  1      3     0.50    *                   ld1b	{ z5.h }, p3/z, [x17, x16]
+# CHECK-NEXT:  1      7     7.00    *                   ld1d	{ z0.d }, p0/z, [x0, z0.d, sxtw #3]
+# CHECK-NEXT:  1      7     7.00    *                   ld1d	{ z0.d }, p0/z, [x0, z0.d, uxtw #3]
+# CHECK-NEXT:  1      3     0.50    *                   ld1d	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      7     7.00    *                   ld1d	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      3     0.50    *                   ld1d	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1d	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *                   ld1d	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      3     0.50    *                   ld1d	{ z23.d }, p3/z, [sp, x8, lsl #3]
+# CHECK-NEXT:  1      3     0.50    *                   ld1d	{ z23.d }, p3/z, [x13, x8, lsl #3]
+# CHECK-NEXT:  1      7     7.00    *                   ld1d	{ z23.d }, p3/z, [x13, z8.d, lsl #3]
+# CHECK-NEXT:  1      3     0.50    *                   ld1d	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1d	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      7     7.00    *                   ld1d	{ z31.d }, p7/z, [z31.d, #248]
+# CHECK-NEXT:  1      7     7.00    *                   ld1h	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  1      7     7.00    *                   ld1h	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  1      3     0.50    *                   ld1h	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      7     7.00    *                   ld1h	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      3     0.50    *                   ld1h	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      9     4.50    *                   ld1h	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      9     4.50    *                   ld1h	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      3     0.50    *                   ld1h	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      9     9.00    *                   ld1h	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      3     0.50    *                   ld1h	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1h	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *                   ld1h	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      3     0.50    *                   ld1h	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     0.50    *                   ld1h	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     0.50    *                   ld1h	{ z21.s }, p5/z, [x10, x21, lsl #1]
+# CHECK-NEXT:  1      3     0.50    *                   ld1h	{ z23.d }, p3/z, [x13, x8, lsl #1]
+# CHECK-NEXT:  1      7     7.00    *                   ld1h	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  1      3     0.50    *                   ld1h	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1h	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      7     7.00    *                   ld1h	{ z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  1      3     0.50    *                   ld1h	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     0.50    *                   ld1h	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      9     4.50    *                   ld1h	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  1      9     4.50    *                   ld1h	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  1      9     9.00    *                   ld1h	{ z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  1      3     0.50    *                   ld1h	{ z5.h }, p3/z, [sp, x16, lsl #1]
+# CHECK-NEXT:  1      3     0.50    *                   ld1h	{ z5.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rb	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rb	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rb	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rb	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rb	{ z31.b }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rb	{ z31.d }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rb	{ z31.h }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rb	{ z31.s }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rd	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rd	{ z31.d }, p7/z, [sp, #504]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rh	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rh	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rh	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rh	{ z31.d }, p7/z, [sp, #126]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rh	{ z31.h }, p7/z, [sp, #126]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rh	{ z31.s }, p7/z, [sp, #126]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rqb	{ z0.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rqb	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rqb	{ z21.b }, p5/z, [x10, #112]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rqb	{ z23.b }, p3/z, [x13, #-128]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rqb	{ z31.b }, p7/z, [sp, #-16]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rqd	{ z0.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rqd	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rqd	{ z23.d }, p3/z, [x13, #-128]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rqd	{ z23.d }, p3/z, [x13, #112]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rqd	{ z31.d }, p7/z, [sp, #-16]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rqh	{ z0.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rqh	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rqh	{ z23.h }, p3/z, [x13, #-128]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rqh	{ z23.h }, p3/z, [x13, #112]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rqh	{ z31.h }, p7/z, [sp, #-16]
+# CHECK-NEXT:  1      3     1.00    *                   ld1rqw	{ z0.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rqw	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rqw	{ z23.s }, p3/z, [x13, #-128]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rqw	{ z23.s }, p3/z, [x13, #112]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rqw	{ z31.s }, p7/z, [sp, #-16]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rsb	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rsb	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rsb	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rsb	{ z31.d }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rsb	{ z31.h }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rsb	{ z31.s }, p7/z, [sp, #63]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rsh	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rsh	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rsh	{ z31.d }, p7/z, [sp, #126]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rsh	{ z31.s }, p7/z, [sp, #126]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rsw	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rsw	{ z31.d }, p7/z, [sp, #252]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rw	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rw	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rw	{ z31.d }, p7/z, [sp, #252]
+# CHECK-NEXT:  1      3     0.50    *                   ld1rw	{ z31.s }, p7/z, [sp, #252]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sb	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sb	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sb	{ z0.h }, p0/z, [sp, x0]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sb	{ z0.h }, p0/z, [x0, x0]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sb	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      9     4.50    *                   ld1sb	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sb	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      9     9.00    *                   ld1sb	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sb	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sb	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sb	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sb	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sb	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sb	{ z21.s }, p5/z, [x10, x21]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sb	{ z23.d }, p3/z, [x13, x8]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sb	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sb	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sb	{ z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sb	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sb	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      9     9.00    *                   ld1sb	{ z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sh	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sh	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sh	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sh	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      9     4.50    *                   ld1sh	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      9     4.50    *                   ld1sh	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sh	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      9     9.00    *                   ld1sh	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sh	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sh	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sh	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sh	{ z21.s }, p5/z, [sp, x21, lsl #1]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sh	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sh	{ z21.s }, p5/z, [x10, x21, lsl #1]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sh	{ z23.d }, p3/z, [x13, x8, lsl #1]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sh	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sh	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sh	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sh	{ z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sh	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      9     4.50    *                   ld1sh	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  1      9     4.50    *                   ld1sh	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  1      9     9.00    *                   ld1sh	{ z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sw	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sw	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sw	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sw	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sw	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sw	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sw	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sw	{ z23.d }, p3/z, [sp, x8, lsl #2]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sw	{ z23.d }, p3/z, [x13, x8, lsl #2]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sw	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  1      3     0.50    *                   ld1sw	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sw	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      7     7.00    *                   ld1sw	{ z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  1      7     7.00    *                   ld1w	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  1      7     7.00    *                   ld1w	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  1      3     0.50    *                   ld1w	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      7     7.00    *                   ld1w	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      9     4.50    *                   ld1w	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      9     4.50    *                   ld1w	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      3     0.50    *                   ld1w	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      9     9.00    *                   ld1w	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      3     0.50    *                   ld1w	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1w	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *                   ld1w	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      3     0.50    *                   ld1w	{ z21.s }, p5/z, [sp, x21, lsl #2]
+# CHECK-NEXT:  1      3     0.50    *                   ld1w	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     0.50    *                   ld1w	{ z21.s }, p5/z, [x10, x21, lsl #2]
+# CHECK-NEXT:  1      3     0.50    *                   ld1w	{ z23.d }, p3/z, [x13, x8, lsl #2]
+# CHECK-NEXT:  1      7     7.00    *                   ld1w	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  1      3     0.50    *                   ld1w	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ld1w	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      7     7.00    *                   ld1w	{ z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  1      3     0.50    *                   ld1w	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      9     4.50    *                   ld1w	{ z31.s }, p7/z, [sp, z31.s, sxtw #2]
+# CHECK-NEXT:  1      9     4.50    *                   ld1w	{ z31.s }, p7/z, [sp, z31.s, uxtw #2]
+# CHECK-NEXT:  1      9     9.00    *                   ld1w	{ z31.s }, p7/z, [z31.s, #124]
+# CHECK-NEXT:  1      3     2.00    *                   ld2b	{ z0.b, z1.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld2b	{ z0.b, z1.b }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld2b	{ z21.b, z22.b }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ld2b	{ z23.b, z24.b }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  1      3     2.00    *                   ld2b	{ z5.b, z6.b }, p3/z, [x17, x16]
+# CHECK-NEXT:  1      3     2.00    *                   ld2d	{ z0.d, z1.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  1      3     1.00    *                   ld2d	{ z0.d, z1.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld2d	{ z21.d, z22.d }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ld2d	{ z23.d, z24.d }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  1      3     2.00    *                   ld2d	{ z5.d, z6.d }, p3/z, [x17, x16, lsl #3]
+# CHECK-NEXT:  1      3     2.00    *                   ld2h	{ z0.h, z1.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      3     1.00    *                   ld2h	{ z0.h, z1.h }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld2h	{ z21.h, z22.h }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ld2h	{ z23.h, z24.h }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  1      3     2.00    *                   ld2h	{ z5.h, z6.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  1      3     2.00    *                   ld2w	{ z0.s, z1.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      3     1.00    *                   ld2w	{ z0.s, z1.s }, p0/z, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ld2w	{ z21.s, z22.s }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ld2w	{ z23.s, z24.s }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  1      3     2.00    *                   ld2w	{ z5.s, z6.s }, p3/z, [x17, x16, lsl #2]
+# CHECK-NEXT:  1      5     3.00    *                   ld3b	{ z0.b - z2.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  1      5     3.00    *                   ld3b	{ z0.b - z2.b }, p0/z, [x0]
+# CHECK-NEXT:  1      5     3.00    *                   ld3b	{ z21.b - z23.b }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld3b	{ z23.b - z25.b }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld3b	{ z5.b - z7.b }, p3/z, [x17, x16]
+# CHECK-NEXT:  1      5     3.00    *                   ld3d	{ z0.d - z2.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  1      5     3.00    *                   ld3d	{ z0.d - z2.d }, p0/z, [x0]
+# CHECK-NEXT:  1      5     3.00    *                   ld3d	{ z21.d - z23.d }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld3d	{ z23.d - z25.d }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld3d	{ z5.d - z7.d }, p3/z, [x17, x16, lsl #3]
+# CHECK-NEXT:  1      5     3.00    *                   ld3h	{ z0.h - z2.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      5     3.00    *                   ld3h	{ z0.h - z2.h }, p0/z, [x0]
+# CHECK-NEXT:  1      5     3.00    *                   ld3h	{ z21.h - z23.h }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld3h	{ z23.h - z25.h }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld3h	{ z5.h - z7.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  1      5     3.00    *                   ld3w	{ z0.s - z2.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      5     3.00    *                   ld3w	{ z0.s - z2.s }, p0/z, [x0]
+# CHECK-NEXT:  1      5     3.00    *                   ld3w	{ z21.s - z23.s }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld3w	{ z23.s - z25.s }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld3w	{ z5.s - z7.s }, p3/z, [x17, x16, lsl #2]
+# CHECK-NEXT:  1      5     3.00    *                   ld4b	{ z0.b - z3.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  1      5     3.00    *                   ld4b	{ z0.b - z3.b }, p0/z, [x0]
+# CHECK-NEXT:  1      5     3.00    *                   ld4b	{ z21.b - z24.b }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld4b	{ z23.b - z26.b }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld4b	{ z5.b - z8.b }, p3/z, [x17, x16]
+# CHECK-NEXT:  1      5     3.00    *                   ld4d	{ z0.d - z3.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  1      5     3.00    *                   ld4d	{ z0.d - z3.d }, p0/z, [x0]
+# CHECK-NEXT:  1      5     3.00    *                   ld4d	{ z21.d - z24.d }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld4d	{ z23.d - z26.d }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld4d	{ z5.d - z8.d }, p3/z, [x17, x16, lsl #3]
+# CHECK-NEXT:  1      5     3.00    *                   ld4h	{ z0.h - z3.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      5     3.00    *                   ld4h	{ z0.h - z3.h }, p0/z, [x0]
+# CHECK-NEXT:  1      5     3.00    *                   ld4h	{ z21.h - z24.h }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld4h	{ z23.h - z26.h }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld4h	{ z5.h - z8.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  1      5     3.00    *                   ld4w	{ z0.s - z3.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      5     3.00    *                   ld4w	{ z0.s - z3.s }, p0/z, [x0]
+# CHECK-NEXT:  1      5     3.00    *                   ld4w	{ z21.s - z24.s }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld4w	{ z23.s - z26.s }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  1      5     3.00    *                   ld4w	{ z5.s - z8.s }, p3/z, [x17, x16, lsl #2]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1b	{ z0.d }, p0/z, [x0, x0]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1b	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1b	{ z0.h }, p0/z, [x0, x0]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1b	{ z0.s }, p0/z, [x0, x0]
+# CHECK-NEXT:  1      9     4.50    *             U     ldff1b	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      9     4.50    *             U     ldff1b	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      9     9.00    *             U     ldff1b	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1b	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1b	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1b	{ z31.b }, p7/z, [sp]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1b	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1b	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1b	{ z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1b	{ z31.h }, p7/z, [sp]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1b	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  1      9     9.00    *             U     ldff1b	{ z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1d	{ z0.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1d	{ z0.d }, p0/z, [x0, z0.d, sxtw #3]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1d	{ z0.d }, p0/z, [x0, z0.d, uxtw #3]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1d	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1d	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1d	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1d	{ z23.d }, p3/z, [x13, z8.d, lsl #3]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1d	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1d	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1d	{ z31.d }, p7/z, [z31.d, #248]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1h	{ z0.d }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1h	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1h	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1h	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1h	{ z0.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1h	{ z0.s }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      9     4.50    *             U     ldff1h	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      9     4.50    *             U     ldff1h	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      9     9.00    *             U     ldff1h	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1h	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1h	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1h	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1h	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1h	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1h	{ z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1h	{ z31.h }, p7/z, [sp]
+# CHECK-NEXT:  1      9     4.50    *             U     ldff1h	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  1      9     4.50    *             U     ldff1h	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1h	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  1      9     9.00    *             U     ldff1h	{ z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1sb	{ z0.d }, p0/z, [x0, x0]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sb	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1sb	{ z0.h }, p0/z, [x0, x0]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1sb	{ z0.s }, p0/z, [x0, x0]
+# CHECK-NEXT:  1      9     4.50    *             U     ldff1sb	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      9     4.50    *             U     ldff1sb	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      9     9.00    *             U     ldff1sb	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sb	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sb	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sb	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1sb	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sb	{ z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1sb	{ z31.h }, p7/z, [sp]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1sb	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  1      9     9.00    *             U     ldff1sb	{ z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1sh	{ z0.d }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sh	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sh	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sh	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1sh	{ z0.s }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      9     4.50    *             U     ldff1sh	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      9     4.50    *             U     ldff1sh	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      9     9.00    *             U     ldff1sh	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sh	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sh	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sh	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sh	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1sh	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sh	{ z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  1      9     4.50    *             U     ldff1sh	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  1      9     4.50    *             U     ldff1sh	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1sh	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  1      9     9.00    *             U     ldff1sh	{ z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1sw	{ z0.d }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sw	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sw	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sw	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sw	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sw	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sw	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sw	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1sw	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1sw	{ z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1w	{ z0.d }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1w	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1w	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1w	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1w	{ z0.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      9     4.50    *             U     ldff1w	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      9     4.50    *             U     ldff1w	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      9     9.00    *             U     ldff1w	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1w	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1w	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1w	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1w	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1w	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  1      7     7.00    *             U     ldff1w	{ z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  1      9     4.50    *             U     ldff1w	{ z31.s }, p7/z, [sp, z31.s, sxtw #2]
+# CHECK-NEXT:  1      9     4.50    *             U     ldff1w	{ z31.s }, p7/z, [sp, z31.s, uxtw #2]
+# CHECK-NEXT:  1      3     0.50    *             U     ldff1w	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  1      9     9.00    *             U     ldff1w	{ z31.s }, p7/z, [z31.s, #124]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1b	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1b	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1b	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1b	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1b	{ z21.b }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1b	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1b	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1b	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1b	{ z31.b }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1b	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1b	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1b	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1d	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1d	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1d	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1h	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1h	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1h	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1h	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1h	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1h	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1h	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1h	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1h	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1sb	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1sb	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1sb	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1sb	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1sb	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1sb	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1sb	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1sb	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1sb	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1sh	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1sh	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1sh	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1sh	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1sh	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1sh	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1sw	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1sw	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1sw	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1w	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1w	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1w	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1w	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1w	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     0.50    *             U     ldnf1w	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1b	{ z0.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1b	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1b	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1b	{ z0.s }, p0/z, [z1.s]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1b	{ z21.b }, p5/z, [x10, #7, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1b	{ z23.b }, p3/z, [x13, #-8, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1b	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1b	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1b	{ z31.s }, p7/z, [z31.s, x0]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1b	{ z31.s }, p7/z, [z31.s]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1d	{ z0.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1d	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1d	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1d	{ z21.d }, p5/z, [x10, #7, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1d	{ z23.d }, p3/z, [x13, #-8, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1d	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1d	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1h	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1h	{ z0.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1h	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1h	{ z0.s }, p0/z, [z1.s]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1h	{ z21.h }, p5/z, [x10, #7, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1h	{ z23.h }, p3/z, [x13, #-8, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1h	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1h	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1h	{ z31.s }, p7/z, [z31.s, x0]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1h	{ z31.s }, p7/z, [z31.s]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1sb	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1sb	{ z0.s }, p0/z, [z1.s]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1sb	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1sb	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1sb	{ z31.s }, p7/z, [z31.s, x0]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1sb	{ z31.s }, p7/z, [z31.s]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1sh	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1sh	{ z0.s }, p0/z, [z1.s]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1sh	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1sh	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1sh	{ z31.s }, p7/z, [z31.s, x0]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1sh	{ z31.s }, p7/z, [z31.s]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1sw	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1sw	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1sw	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1w	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1w	{ z0.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1w	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1w	{ z0.s }, p0/z, [z1.s]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1w	{ z21.s }, p5/z, [x10, #7, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ldnt1w	{ z23.s }, p3/z, [x13, #-8, mul vl]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1w	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  1      7     7.00    *                   ldnt1w	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1w	{ z31.s }, p7/z, [z31.s, x0]
+# CHECK-NEXT:  1      9     9.00    *                   ldnt1w	{ z31.s }, p7/z, [z31.s]
+# CHECK-NEXT:  1      3     1.00    *                   ldr	p0, [x0]
+# CHECK-NEXT:  1      3     1.00    *                   ldr	p5, [x10, #255, mul vl]
+# CHECK-NEXT:  1      3     1.00    *                   ldr	p7, [x13, #-256, mul vl]
+# CHECK-NEXT:  1      3     0.50    *                   ldr	z0, [x0]
+# CHECK-NEXT:  1      3     0.50    *                   ldr	z23, [x13, #255, mul vl]
+# CHECK-NEXT:  1      3     0.50    *                   ldr	z31, [sp, #-256, mul vl]
+# CHECK-NEXT:  1      3     0.50                        lsl	z0.b, p0/m, z0.b, #0
+# CHECK-NEXT:  1      3     0.50                        lsl	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  1      3     0.50                        lsl	z0.b, p0/m, z0.b, z1.d
+# CHECK-NEXT:  1      3     0.50                        lsl	z0.b, z0.b, #0
+# CHECK-NEXT:  1      3     0.50                        lsl	z0.b, z1.b, z2.d
+# CHECK-NEXT:  1      3     0.50                        lsl	z0.d, p0/m, z0.d, #0
+# CHECK-NEXT:  1      3     0.50                        lsl	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      3     0.50                        lsl	z0.d, z0.d, #0
+# CHECK-NEXT:  1      3     0.50                        lsl	z0.h, p0/m, z0.h, #0
+# CHECK-NEXT:  1      3     0.50                        lsl	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  1      3     0.50                        lsl	z0.h, p0/m, z0.h, z1.d
+# CHECK-NEXT:  1      3     0.50                        lsl	z0.h, z0.h, #0
+# CHECK-NEXT:  1      3     0.50                        lsl	z0.h, z1.h, z2.d
+# CHECK-NEXT:  1      3     0.50                        lsl	z0.s, p0/m, z0.s, #0
+# CHECK-NEXT:  1      3     0.50                        lsl	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  1      3     0.50                        lsl	z0.s, p0/m, z0.s, z1.d
+# CHECK-NEXT:  1      3     0.50                        lsl	z0.s, z0.s, #0
+# CHECK-NEXT:  1      3     0.50                        lsl	z0.s, z1.s, z2.d
+# CHECK-NEXT:  1      3     0.50                        lsl	z31.b, p0/m, z31.b, #7
+# CHECK-NEXT:  1      3     0.50                        lsl	z31.b, z31.b, #7
+# CHECK-NEXT:  1      3     0.50                        lsl	z31.d, p0/m, z31.d, #63
+# CHECK-NEXT:  1      3     0.50                        lsl	z31.d, z31.d, #63
+# CHECK-NEXT:  1      3     0.50                        lsl	z31.h, p0/m, z31.h, #15
+# CHECK-NEXT:  1      3     0.50                        lsl	z31.h, z31.h, #15
+# CHECK-NEXT:  1      3     0.50                        lsl	z31.s, p0/m, z31.s, #31
+# CHECK-NEXT:  1      3     0.50                        lsl	z31.s, z31.s, #31
+# CHECK-NEXT:  1      3     0.50                        lslr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  1      3     0.50                        lslr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      3     0.50                        lslr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  1      3     0.50                        lslr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  1      3     0.50                        lsr	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  1      3     0.50                        lsr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  1      3     0.50                        lsr	z0.b, p0/m, z0.b, z1.d
+# CHECK-NEXT:  1      3     0.50                        lsr	z0.b, z0.b, #1
+# CHECK-NEXT:  1      3     0.50                        lsr	z0.b, z1.b, z2.d
+# CHECK-NEXT:  1      3     0.50                        lsr	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  1      3     0.50                        lsr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      3     0.50                        lsr	z0.d, z0.d, #1
+# CHECK-NEXT:  1      3     0.50                        lsr	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  1      3     0.50                        lsr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  1      3     0.50                        lsr	z0.h, p0/m, z0.h, z1.d
+# CHECK-NEXT:  1      3     0.50                        lsr	z0.h, z0.h, #1
+# CHECK-NEXT:  1      3     0.50                        lsr	z0.h, z1.h, z2.d
+# CHECK-NEXT:  1      3     0.50                        lsr	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  1      3     0.50                        lsr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  1      3     0.50                        lsr	z0.s, p0/m, z0.s, z1.d
+# CHECK-NEXT:  1      3     0.50                        lsr	z0.s, z0.s, #1
+# CHECK-NEXT:  1      3     0.50                        lsr	z0.s, z1.s, z2.d
+# CHECK-NEXT:  1      3     0.50                        lsr	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  1      3     0.50                        lsr	z31.b, z31.b, #8
+# CHECK-NEXT:  1      3     0.50                        lsr	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  1      3     0.50                        lsr	z31.d, z31.d, #64
+# CHECK-NEXT:  1      3     0.50                        lsr	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  1      3     0.50                        lsr	z31.h, z31.h, #16
+# CHECK-NEXT:  1      3     0.50                        lsr	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  1      3     0.50                        lsr	z31.s, z31.s, #32
+# CHECK-NEXT:  1      3     0.50                        lsrr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  1      3     0.50                        lsrr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      3     0.50                        lsrr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  1      3     0.50                        lsrr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  1      4     0.50                        mad	z0.b, p7/m, z1.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        mad	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        mad	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        mad	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  1      7     1.00                        match	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  1      7     1.00                        match	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  1      7     1.00                        match	p15.b, p7/z, z30.b, z31.b
+# CHECK-NEXT:  1      7     1.00                        match	p15.h, p7/z, z30.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        mla	z0.b, p7/m, z1.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        mla	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        mla	z0.d, z1.d, z7.d[1]
+# CHECK-NEXT:  1      4     0.50                        mla	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        mla	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        mla	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        mla	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  1      4     0.50                        mls	z0.b, p7/m, z1.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        mls	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        mls	z0.d, z1.d, z7.d[1]
+# CHECK-NEXT:  1      4     0.50                        mls	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        mls	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        mls	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        mls	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  1      6     1.00                        mov	p0.b, p0.b
+# CHECK-NEXT:  1      6     1.00                        mov	p0.b, p0/m, p0.b
+# CHECK-NEXT:  1      6     1.00                        mov	p0.b, p0/z, p0.b
+# CHECK-NEXT:  1      6     1.00                        mov	p15.b, p15.b
+# CHECK-NEXT:  1      6     1.00                        mov	p15.b, p15/m, p15.b
+# CHECK-NEXT:  1      6     1.00                        mov	p15.b, p15/z, p15.b
+# CHECK-NEXT:  1      3     0.50                        mov	z0.b, #127
+# CHECK-NEXT:  1      3     0.50                        mov	z0.b, b0
+# CHECK-NEXT:  1      3     0.50                        mov	z0.b, p0/m, b0
+# CHECK-NEXT:  1      3     1.00                        mov	z0.b, p0/m, w0
+# CHECK-NEXT:  1      3     0.50                        mov	z0.b, p0/z, #127
+# CHECK-NEXT:  1      3     0.50                        mov	z0.b, w0
+# CHECK-NEXT:  1      3     0.50                        mov	z0.d, #0
+# CHECK-NEXT:  1      4     0.50                        mov	z0.d, #0xe0000000000003ff
+# CHECK-NEXT:  1      4     0.50                        mov	z0.d, #0xffffffffffff7fff
+# CHECK-NEXT:  1      4     0.50                        mov	z0.d, #32768
+# CHECK-NEXT:  1      3     0.50                        mov	z0.d, d0
+# CHECK-NEXT:  1      3     0.50                        mov	z0.d, p0/m, d0
+# CHECK-NEXT:  1      3     1.00                        mov	z0.d, p0/m, x0
+# CHECK-NEXT:  1      3     0.50                        mov	z0.d, x0
+# CHECK-NEXT:  1      3     0.50                        mov	z0.d, z0.d
+# CHECK-NEXT:  1      3     0.50                        mov	z0.h, #-256
+# CHECK-NEXT:  1      3     0.50                        mov	z0.h, #-32768
+# CHECK-NEXT:  1      3     0.50                        mov	z0.h, #0
+# CHECK-NEXT:  1      3     0.50                        mov	z0.h, #32512
+# CHECK-NEXT:  1      4     0.50                        mov	z0.h, #32767
+# CHECK-NEXT:  1      3     0.50                        mov	z0.h, h0
+# CHECK-NEXT:  1      3     0.50                        mov	z0.h, p0/m, h0
+# CHECK-NEXT:  1      3     1.00                        mov	z0.h, p0/m, w0
+# CHECK-NEXT:  1      3     0.50                        mov	z0.h, p0/z, #32512
+# CHECK-NEXT:  1      3     0.50                        mov	z0.h, w0
+# CHECK-NEXT:  1      3     0.50                        mov	z0.q, q0
+# CHECK-NEXT:  1      3     0.50                        mov	z0.s, #0
+# CHECK-NEXT:  1      4     0.50                        mov	z0.s, #0xffff7fff
+# CHECK-NEXT:  1      4     0.50                        mov	z0.s, #32768
+# CHECK-NEXT:  1      3     0.50                        mov	z0.s, p0/m, s0
+# CHECK-NEXT:  1      3     1.00                        mov	z0.s, p0/m, w0
+# CHECK-NEXT:  1      3     0.50                        mov	z0.s, s0
+# CHECK-NEXT:  1      3     0.50                        mov	z0.s, w0
+# CHECK-NEXT:  1      3     0.50                        mov	z21.d, #-128
+# CHECK-NEXT:  1      3     0.50                        mov	z21.d, #-32768
+# CHECK-NEXT:  1      3     0.50                        mov	z21.d, #127
+# CHECK-NEXT:  1      3     0.50                        mov	z21.d, #32512
+# CHECK-NEXT:  1      3     0.50                        mov	z21.d, p0/z, #-128
+# CHECK-NEXT:  1      3     0.50                        mov	z21.d, p0/z, #-32768
+# CHECK-NEXT:  1      3     0.50                        mov	z21.d, p0/z, #127
+# CHECK-NEXT:  1      3     0.50                        mov	z21.d, p0/z, #32512
+# CHECK-NEXT:  1      3     0.50                        mov	z21.d, p15/m, #-128
+# CHECK-NEXT:  1      3     0.50                        mov	z21.d, p15/m, #-32768
+# CHECK-NEXT:  1      3     0.50                        mov	z21.h, #-128
+# CHECK-NEXT:  1      3     0.50                        mov	z21.h, #-32768
+# CHECK-NEXT:  1      3     0.50                        mov	z21.h, #127
+# CHECK-NEXT:  1      3     0.50                        mov	z21.h, #32512
+# CHECK-NEXT:  1      3     0.50                        mov	z21.h, p0/z, #-128
+# CHECK-NEXT:  1      3     0.50                        mov	z21.h, p0/z, #-32768
+# CHECK-NEXT:  1      3     0.50                        mov	z21.h, p0/z, #127
+# CHECK-NEXT:  1      3     0.50                        mov	z21.h, p0/z, #32512
+# CHECK-NEXT:  1      3     0.50                        mov	z21.h, p15/m, #-128
+# CHECK-NEXT:  1      3     0.50                        mov	z21.h, p15/m, #-32768
+# CHECK-NEXT:  1      3     0.50                        mov	z21.s, #-128
+# CHECK-NEXT:  1      3     0.50                        mov	z21.s, #-32768
+# CHECK-NEXT:  1      3     0.50                        mov	z21.s, #127
+# CHECK-NEXT:  1      3     0.50                        mov	z21.s, #32512
+# CHECK-NEXT:  1      3     0.50                        mov	z21.s, p0/z, #-128
+# CHECK-NEXT:  1      3     0.50                        mov	z21.s, p0/z, #-32768
+# CHECK-NEXT:  1      3     0.50                        mov	z21.s, p0/z, #127
+# CHECK-NEXT:  1      3     0.50                        mov	z21.s, p0/z, #32512
+# CHECK-NEXT:  1      3     0.50                        mov	z21.s, p15/m, #-128
+# CHECK-NEXT:  1      3     0.50                        mov	z21.s, p15/m, #-32768
+# CHECK-NEXT:  1      3     0.50                        mov	z31.b, p15/m, z31.b
+# CHECK-NEXT:  1      3     0.50                        mov	z31.b, p7/m, b31
+# CHECK-NEXT:  1      3     0.50                        movprfx	z31, z6
+# CHECK-NEXT:  1      3     1.00                        mov	z31.b, p7/m, wsp
+# CHECK-NEXT:  1      3     0.50                        mov	z31.b, wsp
+# CHECK-NEXT:  1      3     0.50                        mov	z31.b, z31.b[63]
+# CHECK-NEXT:  1      3     0.50                        mov	z31.d, p15/m, z31.d
+# CHECK-NEXT:  1      3     0.50                        mov	z31.d, p7/m, d31
+# CHECK-NEXT:  1      3     0.50                        movprfx	z31.d, p7/z, z6.d
+# CHECK-NEXT:  1      3     1.00                        mov	z31.d, p7/m, sp
+# CHECK-NEXT:  1      3     0.50                        mov	z31.d, sp
+# CHECK-NEXT:  1      3     0.50                        mov	z31.d, z0.d
+# CHECK-NEXT:  1      3     0.50                        mov	z31.d, z31.d[7]
+# CHECK-NEXT:  1      3     0.50                        mov	z31.h, p15/m, z31.h
+# CHECK-NEXT:  1      3     0.50                        mov	z31.h, p7/m, h31
+# CHECK-NEXT:  1      3     1.00                        mov	z31.h, p7/m, wsp
+# CHECK-NEXT:  1      3     0.50                        mov	z31.h, wsp
+# CHECK-NEXT:  1      3     0.50                        mov	z31.h, z31.h[31]
+# CHECK-NEXT:  1      3     0.50                        mov	z31.s, p15/m, z31.s
+# CHECK-NEXT:  1      3     0.50                        mov	z31.s, p7/m, s31
+# CHECK-NEXT:  1      3     1.00                        mov	z31.s, p7/m, wsp
+# CHECK-NEXT:  1      3     0.50                        mov	z31.s, wsp
+# CHECK-NEXT:  1      3     0.50                        mov	z31.s, z31.s[15]
+# CHECK-NEXT:  1      3     0.50                        mov	z5.b, #-1
+# CHECK-NEXT:  1      3     0.50                        mov	z5.b, #-128
+# CHECK-NEXT:  1      3     0.50                        mov	z5.b, #127
+# CHECK-NEXT:  1      3     0.50                        mov	z5.b, p0/z, #-1
+# CHECK-NEXT:  1      3     0.50                        mov	z5.b, p0/z, #-128
+# CHECK-NEXT:  1      3     0.50                        mov	z5.b, p0/z, #127
+# CHECK-NEXT:  1      3     0.50                        mov	z5.b, p15/m, #-128
+# CHECK-NEXT:  1      3     0.50                        mov	z5.d, #-6
+# CHECK-NEXT:  1      3     0.50                        mov	z5.h, #-6
+# CHECK-NEXT:  1      3     0.50                        mov	z5.q, z17.q[3]
+# CHECK-NEXT:  1      3     0.50                        mov	z5.s, #-6
+# CHECK-NEXT:  1      6     1.00                        movs	p0.b, p0.b
+# CHECK-NEXT:  1      6     1.00                        movs	p0.b, p0/z, p0.b
+# CHECK-NEXT:  1      6     1.00                        movs	p15.b, p15.b
+# CHECK-NEXT:  1      6     1.00                        movs	p15.b, p15/z, p15.b
+# CHECK-NEXT:  1      1     1.00                  U     mrs	x3, ID_AA64ZFR0_EL1
+# CHECK-NEXT:  1      1     1.00                  U     mrs	x3, ZCR_EL1
+# CHECK-NEXT:  1      1     1.00                  U     mrs	x3, ZCR_EL12
+# CHECK-NEXT:  1      1     1.00                  U     mrs	x3, ZCR_EL2
+# CHECK-NEXT:  1      1     1.00                  U     mrs	x3, ZCR_EL3
+# CHECK-NEXT:  1      4     0.50                        msb	z0.b, p7/m, z1.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        msb	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        msb	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        msb	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  1      1     1.00                  U     msr	ZCR_EL1, x3
+# CHECK-NEXT:  1      1     1.00                  U     msr	ZCR_EL12, x3
+# CHECK-NEXT:  1      1     1.00                  U     msr	ZCR_EL2, x3
+# CHECK-NEXT:  1      1     1.00                  U     msr	ZCR_EL3, x3
+# CHECK-NEXT:  1      4     0.50                        mul	z0.b, p7/m, z0.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        mul	z0.b, z1.b, z2.b
+# CHECK-NEXT:  1      4     0.50                        mul	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        mul	z0.d, z1.d, z15.d[1]
+# CHECK-NEXT:  1      4     0.50                        mul	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        mul	z0.h, z1.h, z2.h
+# CHECK-NEXT:  1      4     0.50                        mul	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        mul	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        mul	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  1      4     0.50                        mul	z29.s, z30.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        mul	z31.b, z31.b, #-128
+# CHECK-NEXT:  1      4     0.50                        mul	z31.b, z31.b, #127
+# CHECK-NEXT:  1      4     0.50                        mul	z31.d, z31.d, #-128
+# CHECK-NEXT:  1      4     0.50                        mul	z31.d, z31.d, #127
+# CHECK-NEXT:  1      4     0.50                        mul	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        mul	z31.h, z31.h, #-128
+# CHECK-NEXT:  1      4     0.50                        mul	z31.h, z31.h, #127
+# CHECK-NEXT:  1      4     0.50                        mul	z31.s, z31.s, #-128
+# CHECK-NEXT:  1      4     0.50                        mul	z31.s, z31.s, #127
+# CHECK-NEXT:  1      6     1.00                        nand	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  1      6     1.00                        nand	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      6     1.00                        nands	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  1      6     1.00                        nands	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      3     0.50                        nbsl	z0.d, z0.d, z1.d, z2.d
+# CHECK-NEXT:  1      3     0.50                        neg	z0.b, p0/m, z0.b
+# CHECK-NEXT:  1      3     0.50                        neg	z0.d, p0/m, z0.d
+# CHECK-NEXT:  1      3     0.50                        neg	z0.h, p0/m, z0.h
+# CHECK-NEXT:  1      3     0.50                        neg	z0.s, p0/m, z0.s
+# CHECK-NEXT:  1      3     0.50                        neg	z31.b, p7/m, z31.b
+# CHECK-NEXT:  1      3     0.50                        neg	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      3     0.50                        neg	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      3     0.50                        neg	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      7     1.00                        nmatch	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  1      7     1.00                        nmatch	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  1      7     1.00                        nmatch	p15.b, p7/z, z30.b, z31.b
+# CHECK-NEXT:  1      7     1.00                        nmatch	p15.h, p7/z, z30.h, z31.h
+# CHECK-NEXT:  1      6     1.00                        nor	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  1      6     1.00                        nor	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      6     1.00                        nors	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  1      6     1.00                        nors	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      6     1.00                        not	p0.b, p0/z, p0.b
+# CHECK-NEXT:  1      6     1.00                        not	p15.b, p15/z, p15.b
+# CHECK-NEXT:  1      3     0.50                        not	z31.b, p7/m, z31.b
+# CHECK-NEXT:  1      3     0.50                        not	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      3     0.50                        not	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      3     0.50                        not	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      6     1.00                        nots	p0.b, p0/z, p0.b
+# CHECK-NEXT:  1      6     1.00                        nots	p15.b, p15/z, p15.b
+# CHECK-NEXT:  1      6     1.00                        orn	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  1      6     1.00                        orn	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      6     1.00                        orns	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  1      6     1.00                        orns	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  1      6     1.00                        orr	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  1      3     0.50                        orr	z0.d, z0.d, #0x6
+# CHECK-NEXT:  1      3     0.50                        orr	z0.d, z0.d, #0xfffffffffffffff9
+# CHECK-NEXT:  1      3     0.50                        orr	z0.s, z0.s, #0x6
+# CHECK-NEXT:  1      3     0.50                        orr	z0.s, z0.s, #0xfffffff9
+# CHECK-NEXT:  1      3     0.50                        orr	z23.d, z13.d, z8.d
+# CHECK-NEXT:  1      3     0.50                        orr	z23.h, z23.h, #0x6
+# CHECK-NEXT:  1      3     0.50                        orr	z23.h, z23.h, #0xfff9
+# CHECK-NEXT:  1      3     0.50                        orr	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  1      3     0.50                        orr	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  1      3     0.50                        orr	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        orr	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  1      3     0.50                        orr	z5.b, z5.b, #0x6
+# CHECK-NEXT:  1      3     0.50                        orr	z5.b, z5.b, #0xf9
+# CHECK-NEXT:  1      6     1.00                        orrs	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  1      4     1.00                        orv	b0, p7, z31.b
+# CHECK-NEXT:  1      4     1.00                        orv	d0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        orv	h0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        orv	s0, p7, z31.s
+# CHECK-NEXT:  1      6     1.00                        pfalse	p15.b
+# CHECK-NEXT:  1      6     1.00                        pfirst	p0.b, p15, p0.b
+# CHECK-NEXT:  1      6     1.00                        pfirst	p15.b, p15, p15.b
+# CHECK-NEXT:  1      4     0.50                        pmul	z0.b, z1.b, z2.b
+# CHECK-NEXT:  1      4     0.50                        pmul	z29.b, z30.b, z31.b
+# CHECK-NEXT:  1      6     1.00                        pmullb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      6     1.00                        pmullb	z29.q, z30.d, z31.d
+# CHECK-NEXT:  1      6     1.00                        pmullb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      6     1.00                        pmullt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      6     1.00                        pmullt	z29.q, z30.d, z31.d
+# CHECK-NEXT:  1      6     1.00                        pmullt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      6     1.00                        pnext	p0.b, p15, p0.b
+# CHECK-NEXT:  1      6     1.00                        pnext	p0.d, p15, p0.d
+# CHECK-NEXT:  1      6     1.00                        pnext	p0.h, p15, p0.h
+# CHECK-NEXT:  1      6     1.00                        pnext	p0.s, p15, p0.s
+# CHECK-NEXT:  1      6     1.00                        pnext	p15.b, p15, p15.b
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	#14, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	#15, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	#6, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	#7, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	#7, p3, [z13.s, #31]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	#7, p3, [z13.s]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	pldl1keep, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	pldl1keep, p0, [x0, z0.d]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	pldl1keep, p0, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	pldl1keep, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	pldl1strm, p0, [x0, #-32, mul vl]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	pldl1strm, p0, [x0, #31, mul vl]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	pldl1strm, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	pldl2keep, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	pldl2strm, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	pldl3keep, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	pldl3strm, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	pldl3strm, p5, [x10, z21.d, sxtw]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	pldl3strm, p5, [x10, z21.s, uxtw]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	pldl3strm, p5, [z10.d, #31]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	pldl3strm, p5, [z10.d]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	pstl1keep, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	pstl1strm, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	pstl2keep, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	pstl2strm, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	pstl3keep, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfb	pstl3strm, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	#14, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	#15, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	#15, p7, [z31.d, #248]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	#15, p7, [z31.d]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	#15, p7, [z31.s, #248]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	#15, p7, [z31.s]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	#6, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	#7, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	pldl1keep, p0, [x0, z0.d, lsl #3]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	pldl1keep, p0, [x0, z0.d, sxtw #3]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	pldl1keep, p0, [x0, z0.d, uxtw #3]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	pldl1keep, p0, [x0, z0.s, sxtw #3]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	pldl1keep, p0, [x0, z0.s, uxtw #3]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	pldl1keep, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	pldl1strm, p0, [x0, #-32, mul vl]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	pldl1strm, p0, [x0, #31, mul vl]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	pldl1strm, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	pldl2keep, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	pldl2strm, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	pldl3keep, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	pldl3strm, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	pstl1keep, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	pstl1strm, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	pstl2keep, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	pstl2strm, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	pstl3keep, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfd	pstl3strm, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	#14, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	#15, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	#15, p7, [z31.d, #62]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	#15, p7, [z31.d]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	#15, p7, [z31.s, #62]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	#15, p7, [z31.s]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	#6, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	#7, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	pldl1keep, p0, [x0, z0.d, lsl #1]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	pldl1keep, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	pldl1strm, p0, [x0, #-32, mul vl]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	pldl1strm, p0, [x0, #31, mul vl]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	pldl1strm, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	pldl2keep, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	pldl2strm, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	pldl3keep, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	pldl3strm, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	pldl3strm, p5, [x10, z21.d, sxtw #1]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	pldl3strm, p5, [x10, z21.d, uxtw #1]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	pldl3strm, p5, [x10, z21.s, sxtw #1]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	pldl3strm, p5, [x10, z21.s, uxtw #1]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	pstl1keep, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	pstl1strm, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	pstl2keep, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	pstl2strm, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	pstl3keep, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfh	pstl3strm, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	#14, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	#15, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	#15, p7, [z31.d, #124]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	#15, p7, [z31.d]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	#15, p7, [z31.s, #124]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	#15, p7, [z31.s]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	#6, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	#7, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	#7, p3, [x13, z8.d, uxtw #2]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	pldl1keep, p0, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	pldl1keep, p0, [x0, z0.s, uxtw #2]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	pldl1keep, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	pldl1strm, p0, [x0, #-32, mul vl]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	pldl1strm, p0, [x0, #31, mul vl]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	pldl1strm, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	pldl2keep, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	pldl2strm, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	pldl3keep, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	pldl3strm, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	pldl3strm, p5, [x10, z21.d, lsl #2]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	pldl3strm, p5, [x10, z21.s, sxtw #2]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	pstl1keep, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	pstl1strm, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	pstl2keep, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	pstl2strm, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	pstl3keep, p0, [x0]
+# CHECK-NEXT:  1      0     0.50    *      *      U     prfw	pstl3strm, p0, [x0]
+# CHECK-NEXT:  1      6     1.00                        ptest	p15, p0.b
+# CHECK-NEXT:  1      6     1.00                        ptest	p15, p15.b
+# CHECK-NEXT:  1      6     1.00                        ptrue	p0.b, pow2
+# CHECK-NEXT:  1      6     1.00                        ptrue	p0.d, pow2
+# CHECK-NEXT:  1      6     1.00                        ptrue	p0.h, pow2
+# CHECK-NEXT:  1      6     1.00                        ptrue	p0.s, pow2
+# CHECK-NEXT:  1      6     1.00                        ptrue	p15.b
+# CHECK-NEXT:  1      6     1.00                        ptrue	p15.d
+# CHECK-NEXT:  1      6     1.00                        ptrue	p15.h
+# CHECK-NEXT:  1      6     1.00                        ptrue	p15.s
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, #14
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, #15
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, #16
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, #17
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, #18
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, #19
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, #20
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, #21
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, #22
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, #23
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, #24
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, #25
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, #26
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, #27
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, #28
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, mul3
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, mul4
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, vl1
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, vl128
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, vl16
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, vl2
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, vl256
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, vl3
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, vl32
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, vl4
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, vl5
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, vl6
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, vl64
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, vl7
+# CHECK-NEXT:  1      6     1.00                        ptrue	p7.s, vl8
+# CHECK-NEXT:  1      6     1.00                        ptrues	p0.b, pow2
+# CHECK-NEXT:  1      6     1.00                        ptrues	p0.d, pow2
+# CHECK-NEXT:  1      6     1.00                        ptrues	p0.h, pow2
+# CHECK-NEXT:  1      6     1.00                        ptrues	p0.s, pow2
+# CHECK-NEXT:  1      6     1.00                        ptrues	p15.b
+# CHECK-NEXT:  1      6     1.00                        ptrues	p15.d
+# CHECK-NEXT:  1      6     1.00                        ptrues	p15.h
+# CHECK-NEXT:  1      6     1.00                        ptrues	p15.s
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, #14
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, #15
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, #16
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, #17
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, #18
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, #19
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, #20
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, #21
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, #22
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, #23
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, #24
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, #25
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, #26
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, #27
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, #28
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, mul3
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, mul4
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, vl1
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, vl128
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, vl16
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, vl2
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, vl256
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, vl3
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, vl32
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, vl4
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, vl5
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, vl6
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, vl64
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, vl7
+# CHECK-NEXT:  1      6     1.00                        ptrues	p7.s, vl8
+# CHECK-NEXT:  1      6     1.00                        punpkhi	p0.h, p0.b
+# CHECK-NEXT:  1      6     1.00                        punpkhi	p15.h, p15.b
+# CHECK-NEXT:  1      6     1.00                        punpklo	p0.h, p0.b
+# CHECK-NEXT:  1      6     1.00                        punpklo	p15.h, p15.b
+# CHECK-NEXT:  1      4     0.50                        raddhnb	z0.b, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        raddhnb	z0.h, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        raddhnb	z0.s, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        raddhnt	z0.b, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        raddhnt	z0.h, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        raddhnt	z0.s, z1.d, z31.d
+# CHECK-NEXT:  1      8     1.00                        rax1	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      3     0.50                        rbit	z0.b, p7/m, z31.b
+# CHECK-NEXT:  1      3     0.50                        rbit	z0.d, p7/m, z31.d
+# CHECK-NEXT:  1      3     0.50                        rbit	z0.h, p7/m, z31.h
+# CHECK-NEXT:  1      3     0.50                        rbit	z0.s, p7/m, z31.s
+# CHECK-NEXT:  1      1     0.33                  U     rdffr	p0.b
+# CHECK-NEXT:  1      3     1.00                  U     rdffr	p0.b, p0/z
+# CHECK-NEXT:  1      1     0.33                  U     rdffr	p15.b
+# CHECK-NEXT:  1      3     1.00                  U     rdffr	p15.b, p15/z
+# CHECK-NEXT:  1      3     1.00                  U     rdffrs	p0.b, p0/z
+# CHECK-NEXT:  1      3     1.00                  U     rdffrs	p15.b, p15/z
+# CHECK-NEXT:  1      1     0.33                        rdvl	x0, #0
+# CHECK-NEXT:  1      1     0.33                        rdvl	x21, #-32
+# CHECK-NEXT:  1      1     0.33                        rdvl	x23, #31
+# CHECK-NEXT:  1      1     0.33                        rdvl	xzr, #-1
+# CHECK-NEXT:  1      4     0.50                        rev	z0.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        rev	z0.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        rev	z0.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        rev	z0.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        revb	z0.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     0.50                        revb	z0.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     0.50                        revb	z0.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     0.50                        revh	z0.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     0.50                        revh	z0.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     0.50                        revw	z0.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     0.50                        rshrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     0.50                        rshrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     0.50                        rshrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     0.50                        rshrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     0.50                        rshrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     0.50                        rshrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     0.50                        rshrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     0.50                        rshrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     0.50                        rshrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     0.50                        rshrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     0.50                        rshrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     0.50                        rshrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     0.50                        rsubhnb	z0.b, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        rsubhnb	z0.h, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        rsubhnb	z0.s, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        rsubhnt	z0.b, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        rsubhnt	z0.h, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        rsubhnt	z0.s, z1.d, z31.d
+# CHECK-NEXT:  1      8     1.00                        saba	z0.b, z1.b, z31.b
+# CHECK-NEXT:  1      8     1.00                        saba	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      8     1.00                        saba	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      8     1.00                        saba	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      8     1.00                        sabalb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      8     1.00                        sabalb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      8     1.00                        sabalb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      8     1.00                        sabalt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      8     1.00                        sabalt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      8     1.00                        sabalt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        sabd	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  1      3     0.50                        sabd	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  1      3     0.50                        sabd	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        sabd	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  1      3     0.50                        sabdlb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      3     0.50                        sabdlb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        sabdlb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      3     0.50                        sabdlt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      3     0.50                        sabdlt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        sabdlt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      7     1.00                        sadalp	z0.h, p0/m, z1.b
+# CHECK-NEXT:  1      7     1.00                        sadalp	z29.s, p0/m, z30.h
+# CHECK-NEXT:  1      7     1.00                        sadalp	z30.d, p7/m, z31.s
+# CHECK-NEXT:  1      3     0.50                        saddlb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      3     0.50                        saddlb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        saddlb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      3     0.50                        saddlbt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      3     0.50                        saddlbt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      3     0.50                        saddlbt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        saddlt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      3     0.50                        saddlt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        saddlt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        saddv	d0, p7, z31.b
+# CHECK-NEXT:  1      4     1.00                        saddv	d0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        saddv	d0, p7, z31.s
+# CHECK-NEXT:  1      3     0.50                        saddwb	z0.h, z1.h, z2.b
+# CHECK-NEXT:  1      3     0.50                        saddwb	z29.s, z30.s, z31.h
+# CHECK-NEXT:  1      3     0.50                        saddwb	z31.d, z31.d, z31.s
+# CHECK-NEXT:  1      3     0.50                        saddwt	z0.h, z1.h, z2.b
+# CHECK-NEXT:  1      3     0.50                        saddwt	z29.s, z30.s, z31.h
+# CHECK-NEXT:  1      3     0.50                        saddwt	z31.d, z31.d, z31.s
+# CHECK-NEXT:  1      4     0.50                        sbclb	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        sbclb	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        sbclt	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        sbclt	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        scvtf	z0.d, p0/m, z0.d
+# CHECK-NEXT:  1      4     0.50                        scvtf	z0.d, p0/m, z0.s
+# CHECK-NEXT:  1      4     0.50                        scvtf	z0.h, p0/m, z0.d
+# CHECK-NEXT:  1      4     0.50                        scvtf	z0.h, p0/m, z0.h
+# CHECK-NEXT:  1      4     0.50                        scvtf	z0.h, p0/m, z0.s
+# CHECK-NEXT:  1      4     0.50                        scvtf	z0.s, p0/m, z0.d
+# CHECK-NEXT:  1      4     0.50                        scvtf	z0.s, p0/m, z0.s
+# CHECK-NEXT:  1      26    23.00                       sdiv	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      15    12.00                       sdiv	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      26    23.00                       sdivr	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      15    12.00                       sdivr	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        sdot	z0.d, z1.h, z15.h[1]
+# CHECK-NEXT:  1      4     0.50                        sdot	z0.d, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        sdot	z0.s, z1.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        sdot	z0.s, z1.b, z7.b[3]
+# CHECK-NEXT:  1      3     0.50                        sel	z23.b, p11, z13.b, z8.b
+# CHECK-NEXT:  1      3     0.50                        sel	z23.d, p11, z13.d, z8.d
+# CHECK-NEXT:  1      3     0.50                        sel	z23.h, p11, z13.h, z8.h
+# CHECK-NEXT:  1      3     0.50                        sel	z23.s, p11, z13.s, z8.s
+# CHECK-NEXT:  1      1     0.33           *      U     setffr
+# CHECK-NEXT:  1      3     0.50                        shadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      3     0.50                        shadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      3     0.50                        shadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      3     0.50                        shadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      3     0.50                        shrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  1      3     0.50                        shrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  1      3     0.50                        shrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  1      3     0.50                        shrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  1      3     0.50                        shrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  1      3     0.50                        shrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  1      3     0.50                        shrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  1      3     0.50                        shrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  1      3     0.50                        shrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  1      3     0.50                        shrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  1      3     0.50                        shrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  1      3     0.50                        shrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  1      3     0.50                        shsub	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      3     0.50                        shsub	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      3     0.50                        shsub	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      3     0.50                        shsub	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      3     0.50                        shsubr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      3     0.50                        shsubr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      3     0.50                        shsubr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      3     0.50                        shsubr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      3     0.50                        sli	z0.b, z0.b, #0
+# CHECK-NEXT:  1      3     0.50                        sli	z0.d, z0.d, #0
+# CHECK-NEXT:  1      3     0.50                        sli	z0.h, z0.h, #0
+# CHECK-NEXT:  1      3     0.50                        sli	z0.s, z0.s, #0
+# CHECK-NEXT:  1      3     0.50                        sli	z31.b, z31.b, #7
+# CHECK-NEXT:  1      3     0.50                        sli	z31.d, z31.d, #63
+# CHECK-NEXT:  1      3     0.50                        sli	z31.h, z31.h, #15
+# CHECK-NEXT:  1      3     0.50                        sli	z31.s, z31.s, #31
+# CHECK-NEXT:  1      8     1.00                        sm4e	z0.s, z0.s, z31.s
+# CHECK-NEXT:  1      8     1.00                        sm4ekey	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      3     0.50                        smax	z0.b, z0.b, #-128
+# CHECK-NEXT:  1      3     0.50                        smax	z0.d, z0.d, #-128
+# CHECK-NEXT:  1      3     0.50                        smax	z0.h, z0.h, #-128
+# CHECK-NEXT:  1      3     0.50                        smax	z0.s, z0.s, #-128
+# CHECK-NEXT:  1      3     0.50                        smax	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  1      3     0.50                        smax	z31.b, z31.b, #127
+# CHECK-NEXT:  1      3     0.50                        smax	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  1      3     0.50                        smax	z31.d, z31.d, #127
+# CHECK-NEXT:  1      3     0.50                        smax	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        smax	z31.h, z31.h, #127
+# CHECK-NEXT:  1      3     0.50                        smax	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  1      3     0.50                        smax	z31.s, z31.s, #127
+# CHECK-NEXT:  1      3     0.50                        smaxp	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      3     0.50                        smaxp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      3     0.50                        smaxp	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      3     0.50                        smaxp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        smaxv	b0, p7, z31.b
+# CHECK-NEXT:  1      4     1.00                        smaxv	d0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        smaxv	h0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        smaxv	s0, p7, z31.s
+# CHECK-NEXT:  1      3     0.50                        smin	z0.b, z0.b, #-128
+# CHECK-NEXT:  1      3     0.50                        smin	z0.d, z0.d, #-128
+# CHECK-NEXT:  1      3     0.50                        smin	z0.h, z0.h, #-128
+# CHECK-NEXT:  1      3     0.50                        smin	z0.s, z0.s, #-128
+# CHECK-NEXT:  1      3     0.50                        smin	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  1      3     0.50                        smin	z31.b, z31.b, #127
+# CHECK-NEXT:  1      3     0.50                        smin	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  1      3     0.50                        smin	z31.d, z31.d, #127
+# CHECK-NEXT:  1      3     0.50                        smin	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        smin	z31.h, z31.h, #127
+# CHECK-NEXT:  1      3     0.50                        smin	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  1      3     0.50                        smin	z31.s, z31.s, #127
+# CHECK-NEXT:  1      3     0.50                        sminp	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      3     0.50                        sminp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      3     0.50                        sminp	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      3     0.50                        sminp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        sminv	b0, p7, z31.b
+# CHECK-NEXT:  1      4     1.00                        sminv	d0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        sminv	h0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        sminv	s0, p7, z31.s
+# CHECK-NEXT:  1      4     0.50                        smlalb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     0.50                        smlalb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        smlalb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        smlalb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        smlalb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        smlalt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     0.50                        smlalt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        smlalt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        smlalt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        smlalt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        smlslb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     0.50                        smlslb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        smlslb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        smlslb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        smlslb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        smlslt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     0.50                        smlslt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        smlslt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        smlslt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        smlslt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        smmla	z0.s, z1.b, z2.b
+# CHECK-NEXT:  1      4     0.50                        smulh	z0.b, p7/m, z0.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        smulh	z0.b, z1.b, z2.b
+# CHECK-NEXT:  1      4     0.50                        smulh	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        smulh	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        smulh	z0.h, z1.h, z2.h
+# CHECK-NEXT:  1      4     0.50                        smulh	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        smulh	z29.s, z30.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        smulh	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        smullb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     0.50                        smullb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      4     0.50                        smullb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        smullb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        smullb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        smullt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     0.50                        smullt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      4     0.50                        smullt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        smullt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        smullt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        splice	z29.b, p7, { z30.b, z31.b }
+# CHECK-NEXT:  1      4     0.50                        splice	z29.d, p7, { z30.d, z31.d }
+# CHECK-NEXT:  1      4     0.50                        splice	z29.h, p7, { z30.h, z31.h }
+# CHECK-NEXT:  1      4     0.50                        splice	z29.s, p7, { z30.s, z31.s }
+# CHECK-NEXT:  1      4     0.50                        splice	z31.b, p7, z31.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        splice	z31.d, p7, z31.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        splice	z31.h, p7, z31.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        splice	z31.s, p7, z31.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        sqabs	z31.b, p7/m, z31.b
+# CHECK-NEXT:  1      4     0.50                        sqabs	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     0.50                        sqabs	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     0.50                        sqabs	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     0.50                        sqadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     0.50                        sqadd	z0.b, z0.b, #0
+# CHECK-NEXT:  1      4     0.50                        sqadd	z0.b, z0.b, z0.b
+# CHECK-NEXT:  1      4     0.50                        sqadd	z0.d, z0.d, #0
+# CHECK-NEXT:  1      4     0.50                        sqadd	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  1      4     0.50                        sqadd	z0.d, z0.d, z0.d
+# CHECK-NEXT:  1      4     0.50                        sqadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        sqadd	z0.h, z0.h, #0
+# CHECK-NEXT:  1      4     0.50                        sqadd	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  1      4     0.50                        sqadd	z0.h, z0.h, z0.h
+# CHECK-NEXT:  1      4     0.50                        sqadd	z0.s, z0.s, #0
+# CHECK-NEXT:  1      4     0.50                        sqadd	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  1      4     0.50                        sqadd	z0.s, z0.s, z0.s
+# CHECK-NEXT:  1      4     0.50                        sqadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        sqadd	z31.b, z31.b, #255
+# CHECK-NEXT:  1      4     0.50                        sqadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     0.50                        sqadd	z31.d, z31.d, #65280
+# CHECK-NEXT:  1      4     0.50                        sqadd	z31.h, z31.h, #65280
+# CHECK-NEXT:  1      4     0.50                        sqadd	z31.s, z31.s, #65280
+# CHECK-NEXT:  1      4     0.50                        sqcadd	z0.b, z0.b, z0.b, #90
+# CHECK-NEXT:  1      4     0.50                        sqcadd	z0.d, z0.d, z0.d, #90
+# CHECK-NEXT:  1      4     0.50                        sqcadd	z0.h, z0.h, z0.h, #90
+# CHECK-NEXT:  1      4     0.50                        sqcadd	z0.s, z0.s, z0.s, #90
+# CHECK-NEXT:  1      4     0.50                        sqcadd	z31.b, z31.b, z31.b, #270
+# CHECK-NEXT:  1      4     0.50                        sqcadd	z31.d, z31.d, z31.d, #270
+# CHECK-NEXT:  1      4     0.50                        sqcadd	z31.h, z31.h, z31.h, #270
+# CHECK-NEXT:  1      4     0.50                        sqcadd	z31.s, z31.s, z31.s, #270
+# CHECK-NEXT:  1      1     0.33                        sqdecb	x0
+# CHECK-NEXT:  1      1     0.33                        sqdecb	x0, #14
+# CHECK-NEXT:  1      1     0.33                        sqdecb	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        sqdecb	x0, pow2
+# CHECK-NEXT:  1      1     0.33                        sqdecb	x0, vl1
+# CHECK-NEXT:  1      1     0.33                        sqdecb	x0, w0
+# CHECK-NEXT:  1      1     0.33                        sqdecb	x0, w0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        sqdecb	x0, w0, pow2
+# CHECK-NEXT:  1      1     0.33                        sqdecb	x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      1     0.33                        sqdecd	x0
+# CHECK-NEXT:  1      1     0.33                        sqdecd	x0, #14
+# CHECK-NEXT:  1      1     0.33                        sqdecd	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        sqdecd	x0, pow2
+# CHECK-NEXT:  1      1     0.33                        sqdecd	x0, vl1
+# CHECK-NEXT:  1      1     0.33                        sqdecd	x0, w0
+# CHECK-NEXT:  1      1     0.33                        sqdecd	x0, w0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        sqdecd	x0, w0, pow2
+# CHECK-NEXT:  1      1     0.33                        sqdecd	x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      4     0.50                        sqdecd	z0.d
+# CHECK-NEXT:  1      4     0.50                        sqdecd	z0.d, all, mul #16
+# CHECK-NEXT:  1      4     0.50                        sqdecd	z0.d, pow2
+# CHECK-NEXT:  1      4     0.50                        sqdecd	z0.d, pow2, mul #16
+# CHECK-NEXT:  1      1     0.33                        sqdech	x0
+# CHECK-NEXT:  1      1     0.33                        sqdech	x0, #14
+# CHECK-NEXT:  1      1     0.33                        sqdech	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        sqdech	x0, pow2
+# CHECK-NEXT:  1      1     0.33                        sqdech	x0, vl1
+# CHECK-NEXT:  1      1     0.33                        sqdech	x0, w0
+# CHECK-NEXT:  1      1     0.33                        sqdech	x0, w0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        sqdech	x0, w0, pow2
+# CHECK-NEXT:  1      1     0.33                        sqdech	x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      4     0.50                        sqdech	z0.h
+# CHECK-NEXT:  1      4     0.50                        sqdech	z0.h, all, mul #16
+# CHECK-NEXT:  1      4     0.50                        sqdech	z0.h, pow2
+# CHECK-NEXT:  1      4     0.50                        sqdech	z0.h, pow2, mul #16
+# CHECK-NEXT:  1      8     1.00                        sqdecp	x0, p0.b
+# CHECK-NEXT:  1      8     1.00                        sqdecp	x0, p0.d
+# CHECK-NEXT:  1      8     1.00                        sqdecp	x0, p0.h
+# CHECK-NEXT:  1      8     1.00                        sqdecp	x0, p0.s
+# CHECK-NEXT:  1      8     1.00                        sqdecp	xzr, p15.b, wzr
+# CHECK-NEXT:  1      8     1.00                        sqdecp	xzr, p15.d, wzr
+# CHECK-NEXT:  1      8     1.00                        sqdecp	xzr, p15.h, wzr
+# CHECK-NEXT:  1      8     1.00                        sqdecp	xzr, p15.s, wzr
+# CHECK-NEXT:  1      4     0.50                        sqdecp	z0.d, p0.d
+# CHECK-NEXT:  1      4     0.50                        sqdecp	z0.h, p0.h
+# CHECK-NEXT:  1      4     0.50                        sqdecp	z0.s, p0.s
+# CHECK-NEXT:  1      1     0.33                        sqdecw	x0
+# CHECK-NEXT:  1      1     0.33                        sqdecw	x0, #14
+# CHECK-NEXT:  1      1     0.33                        sqdecw	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        sqdecw	x0, pow2
+# CHECK-NEXT:  1      1     0.33                        sqdecw	x0, vl1
+# CHECK-NEXT:  1      1     0.33                        sqdecw	x0, w0
+# CHECK-NEXT:  1      1     0.33                        sqdecw	x0, w0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        sqdecw	x0, w0, pow2
+# CHECK-NEXT:  1      1     0.33                        sqdecw	x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      4     0.50                        sqdecw	z0.s
+# CHECK-NEXT:  1      4     0.50                        sqdecw	z0.s, all, mul #16
+# CHECK-NEXT:  1      4     0.50                        sqdecw	z0.s, pow2
+# CHECK-NEXT:  1      4     0.50                        sqdecw	z0.s, pow2, mul #16
+# CHECK-NEXT:  1      4     0.50                        sqdmlalb	z0.d, z1.s, z15.s[3]
+# CHECK-NEXT:  1      4     0.50                        sqdmlalb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        sqdmlalb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        sqdmlalb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        sqdmlalb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        sqdmlalbt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        sqdmlalbt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        sqdmlalbt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        sqdmlalt	z0.d, z1.s, z15.s[3]
+# CHECK-NEXT:  1      4     0.50                        sqdmlalt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        sqdmlalt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        sqdmlalt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        sqdmlalt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        sqdmlslb	z0.d, z1.s, z15.s[3]
+# CHECK-NEXT:  1      4     0.50                        sqdmlslb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        sqdmlslb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        sqdmlslb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        sqdmlslb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        sqdmlslbt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        sqdmlslbt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        sqdmlslbt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        sqdmlslt	z0.d, z1.s, z15.s[3]
+# CHECK-NEXT:  1      4     0.50                        sqdmlslt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        sqdmlslt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        sqdmlslt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        sqdmlslt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        sqdmulh	z0.b, z1.b, z2.b
+# CHECK-NEXT:  1      4     0.50                        sqdmulh	z0.d, z1.d, z15.d[1]
+# CHECK-NEXT:  1      4     0.50                        sqdmulh	z0.h, z1.h, z2.h
+# CHECK-NEXT:  1      4     0.50                        sqdmulh	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        sqdmulh	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  1      4     0.50                        sqdmulh	z29.s, z30.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        sqdmulh	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        sqdmullb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     0.50                        sqdmullb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      4     0.50                        sqdmullb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        sqdmullb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        sqdmullb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        sqdmullt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     0.50                        sqdmullt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      4     0.50                        sqdmullt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        sqdmullt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        sqdmullt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      1     0.33                        sqincb	x0
+# CHECK-NEXT:  1      1     0.33                        sqincb	x0, #14
+# CHECK-NEXT:  1      1     0.33                        sqincb	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        sqincb	x0, pow2
+# CHECK-NEXT:  1      1     0.33                        sqincb	x0, vl1
+# CHECK-NEXT:  1      1     0.33                        sqincb	x0, w0
+# CHECK-NEXT:  1      1     0.33                        sqincb	x0, w0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        sqincb	x0, w0, pow2
+# CHECK-NEXT:  1      1     0.33                        sqincb	x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      1     0.33                        sqincd	x0
+# CHECK-NEXT:  1      1     0.33                        sqincd	x0, #14
+# CHECK-NEXT:  1      1     0.33                        sqincd	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        sqincd	x0, pow2
+# CHECK-NEXT:  1      1     0.33                        sqincd	x0, vl1
+# CHECK-NEXT:  1      1     0.33                        sqincd	x0, w0
+# CHECK-NEXT:  1      1     0.33                        sqincd	x0, w0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        sqincd	x0, w0, pow2
+# CHECK-NEXT:  1      1     0.33                        sqincd	x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      4     0.50                        sqincd	z0.d
+# CHECK-NEXT:  1      4     0.50                        sqincd	z0.d, all, mul #16
+# CHECK-NEXT:  1      4     0.50                        sqincd	z0.d, pow2
+# CHECK-NEXT:  1      4     0.50                        sqincd	z0.d, pow2, mul #16
+# CHECK-NEXT:  1      1     0.33                        sqinch	x0
+# CHECK-NEXT:  1      1     0.33                        sqinch	x0, #14
+# CHECK-NEXT:  1      1     0.33                        sqinch	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        sqinch	x0, pow2
+# CHECK-NEXT:  1      1     0.33                        sqinch	x0, vl1
+# CHECK-NEXT:  1      1     0.33                        sqinch	x0, w0
+# CHECK-NEXT:  1      1     0.33                        sqinch	x0, w0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        sqinch	x0, w0, pow2
+# CHECK-NEXT:  1      1     0.33                        sqinch	x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      4     0.50                        sqinch	z0.h
+# CHECK-NEXT:  1      4     0.50                        sqinch	z0.h, all, mul #16
+# CHECK-NEXT:  1      4     0.50                        sqinch	z0.h, pow2
+# CHECK-NEXT:  1      4     0.50                        sqinch	z0.h, pow2, mul #16
+# CHECK-NEXT:  1      8     1.00                        sqincp	x0, p0.b
+# CHECK-NEXT:  1      8     1.00                        sqincp	x0, p0.d
+# CHECK-NEXT:  1      8     1.00                        sqincp	x0, p0.h
+# CHECK-NEXT:  1      8     1.00                        sqincp	x0, p0.s
+# CHECK-NEXT:  1      8     1.00                        sqincp	xzr, p15.b, wzr
+# CHECK-NEXT:  1      8     1.00                        sqincp	xzr, p15.d, wzr
+# CHECK-NEXT:  1      8     1.00                        sqincp	xzr, p15.h, wzr
+# CHECK-NEXT:  1      8     1.00                        sqincp	xzr, p15.s, wzr
+# CHECK-NEXT:  1      4     0.50                        sqincp	z0.d, p0.d
+# CHECK-NEXT:  1      4     0.50                        sqincp	z0.h, p0.h
+# CHECK-NEXT:  1      4     0.50                        sqincp	z0.s, p0.s
+# CHECK-NEXT:  1      1     0.33                        sqincw	x0
+# CHECK-NEXT:  1      1     0.33                        sqincw	x0, #14
+# CHECK-NEXT:  1      1     0.33                        sqincw	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        sqincw	x0, pow2
+# CHECK-NEXT:  1      1     0.33                        sqincw	x0, vl1
+# CHECK-NEXT:  1      1     0.33                        sqincw	x0, w0
+# CHECK-NEXT:  1      1     0.33                        sqincw	x0, w0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        sqincw	x0, w0, pow2
+# CHECK-NEXT:  1      1     0.33                        sqincw	x0, w0, pow2, mul #16
+# CHECK-NEXT:  1      4     0.50                        sqincw	z0.s
+# CHECK-NEXT:  1      4     0.50                        sqincw	z0.s, all, mul #16
+# CHECK-NEXT:  1      4     0.50                        sqincw	z0.s, pow2
+# CHECK-NEXT:  1      4     0.50                        sqincw	z0.s, pow2, mul #16
+# CHECK-NEXT:  1      4     0.50                        sqneg	z31.b, p7/m, z31.b
+# CHECK-NEXT:  1      4     0.50                        sqneg	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     0.50                        sqneg	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      4     0.50                        sqneg	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     0.50                        sqrdcmlah	z0.b, z1.b, z2.b, #0
+# CHECK-NEXT:  1      4     0.50                        sqrdcmlah	z0.d, z1.d, z2.d, #0
+# CHECK-NEXT:  1      4     0.50                        sqrdcmlah	z0.h, z1.h, z2.h, #0
+# CHECK-NEXT:  1      4     0.50                        sqrdcmlah	z0.h, z1.h, z2.h[0], #0
+# CHECK-NEXT:  1      4     0.50                        sqrdcmlah	z0.s, z1.s, z2.s, #0
+# CHECK-NEXT:  1      4     0.50                        sqrdcmlah	z0.s, z1.s, z2.s[0], #0
+# CHECK-NEXT:  1      4     0.50                        sqrdcmlah	z15.b, z16.b, z17.b, #270
+# CHECK-NEXT:  1      4     0.50                        sqrdcmlah	z15.d, z16.d, z17.d, #270
+# CHECK-NEXT:  1      4     0.50                        sqrdcmlah	z15.h, z16.h, z17.h, #270
+# CHECK-NEXT:  1      4     0.50                        sqrdcmlah	z15.s, z16.s, z17.s, #270
+# CHECK-NEXT:  1      4     0.50                        sqrdcmlah	z29.b, z30.b, z31.b, #90
+# CHECK-NEXT:  1      4     0.50                        sqrdcmlah	z29.d, z30.d, z31.d, #90
+# CHECK-NEXT:  1      4     0.50                        sqrdcmlah	z29.h, z30.h, z31.h, #90
+# CHECK-NEXT:  1      4     0.50                        sqrdcmlah	z29.s, z30.s, z31.s, #90
+# CHECK-NEXT:  1      4     0.50                        sqrdcmlah	z31.b, z31.b, z31.b, #180
+# CHECK-NEXT:  1      4     0.50                        sqrdcmlah	z31.d, z31.d, z31.d, #180
+# CHECK-NEXT:  1      4     0.50                        sqrdcmlah	z31.h, z30.h, z7.h[0], #180
+# CHECK-NEXT:  1      4     0.50                        sqrdcmlah	z31.h, z31.h, z31.h, #180
+# CHECK-NEXT:  1      4     0.50                        sqrdcmlah	z31.s, z30.s, z7.s[0], #180
+# CHECK-NEXT:  1      4     0.50                        sqrdcmlah	z31.s, z31.s, z31.s, #180
+# CHECK-NEXT:  1      4     0.50                        sqrdmlah	z0.b, z1.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        sqrdmlah	z0.d, z1.d, z15.d[1]
+# CHECK-NEXT:  1      4     0.50                        sqrdmlah	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        sqrdmlah	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        sqrdmlah	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        sqrdmlah	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        sqrdmlah	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  1      4     0.50                        sqrdmlsh	z0.b, z1.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        sqrdmlsh	z0.d, z1.d, z15.d[1]
+# CHECK-NEXT:  1      4     0.50                        sqrdmlsh	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        sqrdmlsh	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        sqrdmlsh	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        sqrdmlsh	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        sqrdmlsh	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  1      4     0.50                        sqrdmulh	z0.b, z1.b, z2.b
+# CHECK-NEXT:  1      4     0.50                        sqrdmulh	z0.d, z1.d, z15.d[1]
+# CHECK-NEXT:  1      4     0.50                        sqrdmulh	z0.h, z1.h, z2.h
+# CHECK-NEXT:  1      4     0.50                        sqrdmulh	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        sqrdmulh	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  1      4     0.50                        sqrdmulh	z29.s, z30.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        sqrdmulh	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        sqrshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     0.50                        sqrshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        sqrshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        sqrshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     0.50                        sqrshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     0.50                        sqrshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        sqrshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        sqrshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     0.50                        sqrshrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     0.50                        sqrshrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     0.50                        sqrshrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     0.50                        sqrshrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     0.50                        sqrshrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     0.50                        sqrshrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     0.50                        sqrshrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     0.50                        sqrshrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     0.50                        sqrshrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     0.50                        sqrshrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     0.50                        sqrshrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     0.50                        sqrshrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     0.50                        sqrshrunb	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     0.50                        sqrshrunb	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     0.50                        sqrshrunb	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     0.50                        sqrshrunb	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     0.50                        sqrshrunb	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     0.50                        sqrshrunb	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     0.50                        sqrshrunt	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     0.50                        sqrshrunt	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     0.50                        sqrshrunt	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     0.50                        sqrshrunt	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     0.50                        sqrshrunt	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     0.50                        sqrshrunt	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     0.50                        sqshl	z0.b, p0/m, z0.b, #0
+# CHECK-NEXT:  1      4     0.50                        sqshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     0.50                        sqshl	z0.d, p0/m, z0.d, #0
+# CHECK-NEXT:  1      4     0.50                        sqshl	z0.h, p0/m, z0.h, #0
+# CHECK-NEXT:  1      4     0.50                        sqshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        sqshl	z0.s, p0/m, z0.s, #0
+# CHECK-NEXT:  1      4     0.50                        sqshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        sqshl	z31.b, p0/m, z31.b, #7
+# CHECK-NEXT:  1      4     0.50                        sqshl	z31.d, p0/m, z31.d, #63
+# CHECK-NEXT:  1      4     0.50                        sqshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     0.50                        sqshl	z31.h, p0/m, z31.h, #15
+# CHECK-NEXT:  1      4     0.50                        sqshl	z31.s, p0/m, z31.s, #31
+# CHECK-NEXT:  1      4     0.50                        sqshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     0.50                        sqshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        sqshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        sqshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     0.50                        sqshlu	z0.b, p0/m, z0.b, #0
+# CHECK-NEXT:  1      4     0.50                        sqshlu	z0.d, p0/m, z0.d, #0
+# CHECK-NEXT:  1      4     0.50                        sqshlu	z0.h, p0/m, z0.h, #0
+# CHECK-NEXT:  1      4     0.50                        sqshlu	z0.s, p0/m, z0.s, #0
+# CHECK-NEXT:  1      4     0.50                        sqshlu	z31.b, p0/m, z31.b, #7
+# CHECK-NEXT:  1      4     0.50                        sqshlu	z31.d, p0/m, z31.d, #63
+# CHECK-NEXT:  1      4     0.50                        sqshlu	z31.h, p0/m, z31.h, #15
+# CHECK-NEXT:  1      4     0.50                        sqshlu	z31.s, p0/m, z31.s, #31
+# CHECK-NEXT:  1      4     0.50                        sqshrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     0.50                        sqshrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     0.50                        sqshrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     0.50                        sqshrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     0.50                        sqshrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     0.50                        sqshrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     0.50                        sqshrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     0.50                        sqshrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     0.50                        sqshrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     0.50                        sqshrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     0.50                        sqshrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     0.50                        sqshrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     0.50                        sqshrunb	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     0.50                        sqshrunb	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     0.50                        sqshrunb	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     0.50                        sqshrunb	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     0.50                        sqshrunb	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     0.50                        sqshrunb	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     0.50                        sqshrunt	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     0.50                        sqshrunt	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     0.50                        sqshrunt	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     0.50                        sqshrunt	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     0.50                        sqshrunt	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     0.50                        sqshrunt	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     0.50                        sqsub	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     0.50                        sqsub	z0.b, z0.b, #0
+# CHECK-NEXT:  1      4     0.50                        sqsub	z0.b, z0.b, z0.b
+# CHECK-NEXT:  1      4     0.50                        sqsub	z0.d, z0.d, #0
+# CHECK-NEXT:  1      4     0.50                        sqsub	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  1      4     0.50                        sqsub	z0.d, z0.d, z0.d
+# CHECK-NEXT:  1      4     0.50                        sqsub	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        sqsub	z0.h, z0.h, #0
+# CHECK-NEXT:  1      4     0.50                        sqsub	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  1      4     0.50                        sqsub	z0.h, z0.h, z0.h
+# CHECK-NEXT:  1      4     0.50                        sqsub	z0.s, z0.s, #0
+# CHECK-NEXT:  1      4     0.50                        sqsub	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  1      4     0.50                        sqsub	z0.s, z0.s, z0.s
+# CHECK-NEXT:  1      4     0.50                        sqsub	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        sqsub	z31.b, z31.b, #255
+# CHECK-NEXT:  1      4     0.50                        sqsub	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     0.50                        sqsub	z31.d, z31.d, #65280
+# CHECK-NEXT:  1      4     0.50                        sqsub	z31.h, z31.h, #65280
+# CHECK-NEXT:  1      4     0.50                        sqsub	z31.s, z31.s, #65280
+# CHECK-NEXT:  1      4     0.50                        sqsubr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     0.50                        sqsubr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        sqsubr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        sqsubr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     0.50                        sqxtnb	z0.b, z31.h
+# CHECK-NEXT:  1      4     0.50                        sqxtnb	z0.h, z31.s
+# CHECK-NEXT:  1      4     0.50                        sqxtnb	z0.s, z31.d
+# CHECK-NEXT:  1      4     0.50                        sqxtnt	z0.b, z31.h
+# CHECK-NEXT:  1      4     0.50                        sqxtnt	z0.h, z31.s
+# CHECK-NEXT:  1      4     0.50                        sqxtnt	z0.s, z31.d
+# CHECK-NEXT:  1      4     0.50                        sqxtunb	z0.b, z31.h
+# CHECK-NEXT:  1      4     0.50                        sqxtunb	z0.h, z31.s
+# CHECK-NEXT:  1      4     0.50                        sqxtunb	z0.s, z31.d
+# CHECK-NEXT:  1      4     0.50                        sqxtunt	z0.b, z31.h
+# CHECK-NEXT:  1      4     0.50                        sqxtunt	z0.h, z31.s
+# CHECK-NEXT:  1      4     0.50                        sqxtunt	z0.s, z31.d
+# CHECK-NEXT:  1      4     0.50                        srhadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     0.50                        srhadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        srhadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        srhadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      3     0.50                        sri	z0.b, z0.b, #1
+# CHECK-NEXT:  1      3     0.50                        sri	z0.d, z0.d, #1
+# CHECK-NEXT:  1      3     0.50                        sri	z0.h, z0.h, #1
+# CHECK-NEXT:  1      3     0.50                        sri	z0.s, z0.s, #1
+# CHECK-NEXT:  1      3     0.50                        sri	z31.b, z31.b, #8
+# CHECK-NEXT:  1      3     0.50                        sri	z31.d, z31.d, #64
+# CHECK-NEXT:  1      3     0.50                        sri	z31.h, z31.h, #16
+# CHECK-NEXT:  1      3     0.50                        sri	z31.s, z31.s, #32
+# CHECK-NEXT:  1      4     0.50                        srshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     0.50                        srshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        srshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        srshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     0.50                        srshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     0.50                        srshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        srshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        srshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     0.50                        srshr	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  1      4     0.50                        srshr	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  1      4     0.50                        srshr	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  1      4     0.50                        srshr	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  1      4     0.50                        srshr	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  1      4     0.50                        srshr	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  1      4     0.50                        srshr	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  1      4     0.50                        srshr	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  1      7     1.00                        srsra	z0.b, z0.b, #1
+# CHECK-NEXT:  1      7     1.00                        srsra	z0.d, z0.d, #1
+# CHECK-NEXT:  1      7     1.00                        srsra	z0.h, z0.h, #1
+# CHECK-NEXT:  1      7     1.00                        srsra	z0.s, z0.s, #1
+# CHECK-NEXT:  1      7     1.00                        srsra	z31.b, z31.b, #8
+# CHECK-NEXT:  1      7     1.00                        srsra	z31.d, z31.d, #64
+# CHECK-NEXT:  1      7     1.00                        srsra	z31.h, z31.h, #16
+# CHECK-NEXT:  1      7     1.00                        srsra	z31.s, z31.s, #32
+# CHECK-NEXT:  1      3     0.50                        sshllb	z0.d, z0.s, #0
+# CHECK-NEXT:  1      3     0.50                        sshllb	z0.h, z0.b, #0
+# CHECK-NEXT:  1      3     0.50                        sshllb	z0.s, z0.h, #0
+# CHECK-NEXT:  1      3     0.50                        sshllb	z31.d, z31.s, #31
+# CHECK-NEXT:  1      3     0.50                        sshllb	z31.h, z31.b, #7
+# CHECK-NEXT:  1      3     0.50                        sshllb	z31.s, z31.h, #15
+# CHECK-NEXT:  1      3     0.50                        sshllt	z0.d, z0.s, #0
+# CHECK-NEXT:  1      3     0.50                        sshllt	z0.h, z0.b, #0
+# CHECK-NEXT:  1      3     0.50                        sshllt	z0.s, z0.h, #0
+# CHECK-NEXT:  1      3     0.50                        sshllt	z31.d, z31.s, #31
+# CHECK-NEXT:  1      3     0.50                        sshllt	z31.h, z31.b, #7
+# CHECK-NEXT:  1      3     0.50                        sshllt	z31.s, z31.h, #15
+# CHECK-NEXT:  1      4     0.50                        ssra	z0.b, z0.b, #1
+# CHECK-NEXT:  1      4     0.50                        ssra	z0.d, z0.d, #1
+# CHECK-NEXT:  1      4     0.50                        ssra	z0.h, z0.h, #1
+# CHECK-NEXT:  1      4     0.50                        ssra	z0.s, z0.s, #1
+# CHECK-NEXT:  1      4     0.50                        ssra	z31.b, z31.b, #8
+# CHECK-NEXT:  1      4     0.50                        ssra	z31.d, z31.d, #64
+# CHECK-NEXT:  1      4     0.50                        ssra	z31.h, z31.h, #16
+# CHECK-NEXT:  1      4     0.50                        ssra	z31.s, z31.s, #32
+# CHECK-NEXT:  1      3     0.50                        ssublb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      3     0.50                        ssublb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        ssublb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      3     0.50                        ssublbt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      3     0.50                        ssublbt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      3     0.50                        ssublbt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        ssublt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      3     0.50                        ssublt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        ssublt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      3     0.50                        ssubltb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      3     0.50                        ssubltb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      3     0.50                        ssubltb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        ssubwb	z0.h, z1.h, z2.b
+# CHECK-NEXT:  1      3     0.50                        ssubwb	z29.s, z30.s, z31.h
+# CHECK-NEXT:  1      3     0.50                        ssubwb	z31.d, z31.d, z31.s
+# CHECK-NEXT:  1      3     0.50                        ssubwt	z0.h, z1.h, z2.b
+# CHECK-NEXT:  1      3     0.50                        ssubwt	z29.s, z30.s, z31.h
+# CHECK-NEXT:  1      3     0.50                        ssubwt	z31.d, z31.d, z31.s
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z0.b }, p0, [x0, x0]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z0.b }, p0, [x0]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z0.d }, p0, [x0, x0]
+# CHECK-NEXT:  1      1     8.00           *            st1b	{ z0.d }, p0, [x0, z0.d, sxtw]
+# CHECK-NEXT:  1      1     8.00           *            st1b	{ z0.d }, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  1      1     8.00           *            st1b	{ z0.d }, p0, [x0, z0.d]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  1      1     7.00           *            st1b	{ z0.d }, p7, [z0.d]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z0.h }, p0, [x0, x0]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z0.h }, p0, [x0]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z0.s }, p0, [x0, x0]
+# CHECK-NEXT:  1      1     8.00           *            st1b	{ z0.s }, p0, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      1     8.00           *            st1b	{ z0.s }, p0, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z0.s }, p0, [x0]
+# CHECK-NEXT:  1      1     9.00           *            st1b	{ z0.s }, p7, [z0.s]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z21.b }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z21.d }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z21.h }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z21.s }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z31.b }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z31.d }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      1     7.00           *            st1b	{ z31.d }, p7, [z31.d, #31]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z31.h }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            st1b	{ z31.s }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      1     9.00           *            st1b	{ z31.s }, p7, [z31.s, #31]
+# CHECK-NEXT:  1      1     1.00           *            st1d	{ z0.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  1      1     8.00           *            st1d	{ z0.d }, p0, [x0, z0.d, lsl #3]
+# CHECK-NEXT:  1      1     8.00           *            st1d	{ z0.d }, p0, [x0, z0.d, sxtw #3]
+# CHECK-NEXT:  1      1     8.00           *            st1d	{ z0.d }, p0, [x0, z0.d, sxtw]
+# CHECK-NEXT:  1      1     8.00           *            st1d	{ z0.d }, p0, [x0, z0.d, uxtw #3]
+# CHECK-NEXT:  1      1     8.00           *            st1d	{ z0.d }, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  1      1     8.00           *            st1d	{ z0.d }, p0, [x0, z0.d]
+# CHECK-NEXT:  1      1     1.00           *            st1d	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  1      1     7.00           *            st1d	{ z0.d }, p7, [z0.d]
+# CHECK-NEXT:  1      1     1.00           *            st1d	{ z21.d }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            st1d	{ z31.d }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      1     7.00           *            st1d	{ z31.d }, p7, [z31.d, #248]
+# CHECK-NEXT:  1      1     1.00           *            st1h	{ z0.d }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      1     8.00           *            st1h	{ z0.d }, p0, [x0, z0.d, lsl #1]
+# CHECK-NEXT:  1      1     8.00           *            st1h	{ z0.d }, p0, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  1      1     8.00           *            st1h	{ z0.d }, p0, [x0, z0.d, sxtw]
+# CHECK-NEXT:  1      1     8.00           *            st1h	{ z0.d }, p0, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  1      1     8.00           *            st1h	{ z0.d }, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  1      1     8.00           *            st1h	{ z0.d }, p0, [x0, z0.d]
+# CHECK-NEXT:  1      1     1.00           *            st1h	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  1      1     7.00           *            st1h	{ z0.d }, p7, [z0.d]
+# CHECK-NEXT:  1      1     1.00           *            st1h	{ z0.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      1     1.00           *            st1h	{ z0.h }, p0, [x0]
+# CHECK-NEXT:  1      1     1.00           *            st1h	{ z0.s }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      1     8.00           *            st1h	{ z0.s }, p0, [x0, z0.s, sxtw #1]
+# CHECK-NEXT:  1      1     8.00           *            st1h	{ z0.s }, p0, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      1     8.00           *            st1h	{ z0.s }, p0, [x0, z0.s, uxtw #1]
+# CHECK-NEXT:  1      1     8.00           *            st1h	{ z0.s }, p0, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      1     1.00           *            st1h	{ z0.s }, p0, [x0]
+# CHECK-NEXT:  1      1     9.00           *            st1h	{ z0.s }, p7, [z0.s]
+# CHECK-NEXT:  1      1     1.00           *            st1h	{ z21.d }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            st1h	{ z21.h }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            st1h	{ z21.s }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            st1h	{ z31.d }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      1     7.00           *            st1h	{ z31.d }, p7, [z31.d, #62]
+# CHECK-NEXT:  1      1     1.00           *            st1h	{ z31.h }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            st1h	{ z31.s }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      1     9.00           *            st1h	{ z31.s }, p7, [z31.s, #62]
+# CHECK-NEXT:  1      1     1.00           *            st1w	{ z0.d }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      1     8.00           *            st1w	{ z0.d }, p0, [x0, z0.d, lsl #2]
+# CHECK-NEXT:  1      1     8.00           *            st1w	{ z0.d }, p0, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  1      1     8.00           *            st1w	{ z0.d }, p0, [x0, z0.d, sxtw]
+# CHECK-NEXT:  1      1     8.00           *            st1w	{ z0.d }, p0, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  1      1     8.00           *            st1w	{ z0.d }, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  1      1     8.00           *            st1w	{ z0.d }, p0, [x0, z0.d]
+# CHECK-NEXT:  1      1     1.00           *            st1w	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  1      1     7.00           *            st1w	{ z0.d }, p7, [z0.d]
+# CHECK-NEXT:  1      1     1.00           *            st1w	{ z0.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      1     8.00           *            st1w	{ z0.s }, p0, [x0, z0.s, sxtw #2]
+# CHECK-NEXT:  1      1     8.00           *            st1w	{ z0.s }, p0, [x0, z0.s, sxtw]
+# CHECK-NEXT:  1      1     8.00           *            st1w	{ z0.s }, p0, [x0, z0.s, uxtw #2]
+# CHECK-NEXT:  1      1     8.00           *            st1w	{ z0.s }, p0, [x0, z0.s, uxtw]
+# CHECK-NEXT:  1      1     1.00           *            st1w	{ z0.s }, p0, [x0]
+# CHECK-NEXT:  1      1     9.00           *            st1w	{ z0.s }, p7, [z0.s]
+# CHECK-NEXT:  1      1     1.00           *            st1w	{ z21.d }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            st1w	{ z21.s }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            st1w	{ z31.d }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      1     7.00           *            st1w	{ z31.d }, p7, [z31.d, #124]
+# CHECK-NEXT:  1      1     1.00           *            st1w	{ z31.s }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  1      1     9.00           *            st1w	{ z31.s }, p7, [z31.s, #124]
+# CHECK-NEXT:  1      1     11.00          *            st2b	{ z0.b, z1.b }, p0, [x0, x0]
+# CHECK-NEXT:  1      1     11.00          *            st2b	{ z0.b, z1.b }, p0, [x0]
+# CHECK-NEXT:  1      1     11.00          *            st2b	{ z21.b, z22.b }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  1      1     11.00          *            st2b	{ z23.b, z24.b }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  1      1     11.00          *            st2b	{ z5.b, z6.b }, p3, [x17, x16]
+# CHECK-NEXT:  1      1     11.00          *            st2d	{ z0.d, z1.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  1      1     11.00          *            st2d	{ z0.d, z1.d }, p0, [x0]
+# CHECK-NEXT:  1      1     11.00          *            st2d	{ z21.d, z22.d }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  1      1     11.00          *            st2d	{ z23.d, z24.d }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  1      1     11.00          *            st2d	{ z5.d, z6.d }, p3, [x17, x16, lsl #3]
+# CHECK-NEXT:  1      1     11.00          *            st2h	{ z0.h, z1.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      1     11.00          *            st2h	{ z0.h, z1.h }, p0, [x0]
+# CHECK-NEXT:  1      1     11.00          *            st2h	{ z21.h, z22.h }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  1      1     11.00          *            st2h	{ z23.h, z24.h }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  1      1     11.00          *            st2h	{ z5.h, z6.h }, p3, [x17, x16, lsl #1]
+# CHECK-NEXT:  1      1     11.00          *            st2w	{ z0.s, z1.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      1     11.00          *            st2w	{ z0.s, z1.s }, p0, [x0]
+# CHECK-NEXT:  1      1     11.00          *            st2w	{ z21.s, z22.s }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  1      1     11.00          *            st2w	{ z23.s, z24.s }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  1      1     11.00          *            st2w	{ z5.s, z6.s }, p3, [x17, x16, lsl #2]
+# CHECK-NEXT:  1      1     25.00          *            st3b	{ z0.b - z2.b }, p0, [x0, x0]
+# CHECK-NEXT:  1      1     25.00          *            st3b	{ z0.b - z2.b }, p0, [x0]
+# CHECK-NEXT:  1      1     25.00          *            st3b	{ z21.b - z23.b }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  1      1     25.00          *            st3b	{ z23.b - z25.b }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  1      1     25.00          *            st3b	{ z5.b - z7.b }, p3, [x17, x16]
+# CHECK-NEXT:  1      1     14.00          *            st3d	{ z0.d - z2.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  1      1     14.00          *            st3d	{ z0.d - z2.d }, p0, [x0]
+# CHECK-NEXT:  1      1     14.00          *            st3d	{ z21.d - z23.d }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  1      1     14.00          *            st3d	{ z23.d - z25.d }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  1      1     14.00          *            st3d	{ z5.d - z7.d }, p3, [x17, x16, lsl #3]
+# CHECK-NEXT:  1      1     25.00          *            st3h	{ z0.h - z2.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      1     25.00          *            st3h	{ z0.h - z2.h }, p0, [x0]
+# CHECK-NEXT:  1      1     25.00          *            st3h	{ z21.h - z23.h }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  1      1     25.00          *            st3h	{ z23.h - z25.h }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  1      1     25.00          *            st3h	{ z5.h - z7.h }, p3, [x17, x16, lsl #1]
+# CHECK-NEXT:  1      1     25.00          *            st3w	{ z0.s - z2.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      1     25.00          *            st3w	{ z0.s - z2.s }, p0, [x0]
+# CHECK-NEXT:  1      1     25.00          *            st3w	{ z21.s - z23.s }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  1      1     25.00          *            st3w	{ z23.s - z25.s }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  1      1     25.00          *            st3w	{ z5.s - z7.s }, p3, [x17, x16, lsl #2]
+# CHECK-NEXT:  1      1     50.00          *            st4b	{ z0.b - z3.b }, p0, [x0, x0]
+# CHECK-NEXT:  1      1     50.00          *            st4b	{ z0.b - z3.b }, p0, [x0]
+# CHECK-NEXT:  1      1     50.00          *            st4b	{ z21.b - z24.b }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  1      1     50.00          *            st4b	{ z23.b - z26.b }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  1      1     50.00          *            st4b	{ z5.b - z8.b }, p3, [x17, x16]
+# CHECK-NEXT:  1      1     25.00          *            st4d	{ z0.d - z3.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  1      1     25.00          *            st4d	{ z0.d - z3.d }, p0, [x0]
+# CHECK-NEXT:  1      1     25.00          *            st4d	{ z21.d - z24.d }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  1      1     25.00          *            st4d	{ z23.d - z26.d }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  1      1     25.00          *            st4d	{ z5.d - z8.d }, p3, [x17, x16, lsl #3]
+# CHECK-NEXT:  1      1     50.00          *            st4h	{ z0.h - z3.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      1     50.00          *            st4h	{ z0.h - z3.h }, p0, [x0]
+# CHECK-NEXT:  1      1     50.00          *            st4h	{ z21.h - z24.h }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  1      1     50.00          *            st4h	{ z23.h - z26.h }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  1      1     50.00          *            st4h	{ z5.h - z8.h }, p3, [x17, x16, lsl #1]
+# CHECK-NEXT:  1      1     50.00          *            st4w	{ z0.s - z3.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      1     50.00          *            st4w	{ z0.s - z3.s }, p0, [x0]
+# CHECK-NEXT:  1      1     50.00          *            st4w	{ z21.s - z24.s }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  1      1     50.00          *            st4w	{ z23.s - z26.s }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  1      1     50.00          *            st4w	{ z5.s - z8.s }, p3, [x17, x16, lsl #2]
+# CHECK-NEXT:  1      1     1.00           *            stnt1b	{ z0.b }, p0, [x0, x0]
+# CHECK-NEXT:  1      1     1.00           *            stnt1b	{ z0.b }, p0, [x0]
+# CHECK-NEXT:  1      1     7.00           *            stnt1b	{ z0.d }, p0, [z1.d]
+# CHECK-NEXT:  1      1     9.00           *            stnt1b	{ z0.s }, p0, [z1.s]
+# CHECK-NEXT:  1      1     1.00           *            stnt1b	{ z21.b }, p5, [x10, #7, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            stnt1b	{ z23.b }, p3, [x13, #-8, mul vl]
+# CHECK-NEXT:  1      1     7.00           *            stnt1b	{ z31.d }, p7, [z31.d, x0]
+# CHECK-NEXT:  1      1     7.00           *            stnt1b	{ z31.d }, p7, [z31.d]
+# CHECK-NEXT:  1      1     9.00           *            stnt1b	{ z31.s }, p7, [z31.s, x0]
+# CHECK-NEXT:  1      1     9.00           *            stnt1b	{ z31.s }, p7, [z31.s]
+# CHECK-NEXT:  1      1     1.00           *            stnt1d	{ z0.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  1      1     1.00           *            stnt1d	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  1      1     7.00           *            stnt1d	{ z0.d }, p0, [z1.d]
+# CHECK-NEXT:  1      1     1.00           *            stnt1d	{ z21.d }, p5, [x10, #7, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            stnt1d	{ z23.d }, p3, [x13, #-8, mul vl]
+# CHECK-NEXT:  1      1     7.00           *            stnt1d	{ z31.d }, p7, [z31.d, x0]
+# CHECK-NEXT:  1      1     7.00           *            stnt1d	{ z31.d }, p7, [z31.d]
+# CHECK-NEXT:  1      1     7.00           *            stnt1h	{ z0.d }, p0, [z1.d]
+# CHECK-NEXT:  1      1     1.00           *            stnt1h	{ z0.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  1      1     1.00           *            stnt1h	{ z0.h }, p0, [x0]
+# CHECK-NEXT:  1      1     9.00           *            stnt1h	{ z0.s }, p0, [z1.s]
+# CHECK-NEXT:  1      1     1.00           *            stnt1h	{ z21.h }, p5, [x10, #7, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            stnt1h	{ z23.h }, p3, [x13, #-8, mul vl]
+# CHECK-NEXT:  1      1     7.00           *            stnt1h	{ z31.d }, p7, [z31.d, x0]
+# CHECK-NEXT:  1      1     7.00           *            stnt1h	{ z31.d }, p7, [z31.d]
+# CHECK-NEXT:  1      1     9.00           *            stnt1h	{ z31.s }, p7, [z31.s, x0]
+# CHECK-NEXT:  1      1     9.00           *            stnt1h	{ z31.s }, p7, [z31.s]
+# CHECK-NEXT:  1      1     7.00           *            stnt1w	{ z0.d }, p0, [z1.d]
+# CHECK-NEXT:  1      1     1.00           *            stnt1w	{ z0.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  1      1     1.00           *            stnt1w	{ z0.s }, p0, [x0]
+# CHECK-NEXT:  1      1     9.00           *            stnt1w	{ z0.s }, p0, [z1.s]
+# CHECK-NEXT:  1      1     1.00           *            stnt1w	{ z21.s }, p5, [x10, #7, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            stnt1w	{ z23.s }, p3, [x13, #-8, mul vl]
+# CHECK-NEXT:  1      1     7.00           *            stnt1w	{ z31.d }, p7, [z31.d, x0]
+# CHECK-NEXT:  1      1     7.00           *            stnt1w	{ z31.d }, p7, [z31.d]
+# CHECK-NEXT:  1      1     9.00           *            stnt1w	{ z31.s }, p7, [z31.s, x0]
+# CHECK-NEXT:  1      1     9.00           *            stnt1w	{ z31.s }, p7, [z31.s]
+# CHECK-NEXT:  1      1     1.00           *            str	p0, [x0]
+# CHECK-NEXT:  1      1     1.00           *            str	p15, [sp, #-256, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            str	p5, [x10, #255, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            str	z0, [x0]
+# CHECK-NEXT:  1      1     1.00           *            str	z21, [x10, #-256, mul vl]
+# CHECK-NEXT:  1      1     1.00           *            str	z31, [sp, #255, mul vl]
+# CHECK-NEXT:  1      3     0.50                        sub	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  1      3     0.50                        sub	z0.b, z0.b, #0
+# CHECK-NEXT:  1      3     0.50                        sub	z0.b, z0.b, z0.b
+# CHECK-NEXT:  1      3     0.50                        sub	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      3     0.50                        sub	z0.d, z0.d, #0
+# CHECK-NEXT:  1      3     0.50                        sub	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  1      3     0.50                        sub	z0.d, z0.d, z0.d
+# CHECK-NEXT:  1      3     0.50                        sub	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  1      3     0.50                        sub	z0.h, z0.h, #0
+# CHECK-NEXT:  1      3     0.50                        sub	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  1      3     0.50                        sub	z0.h, z0.h, z0.h
+# CHECK-NEXT:  1      3     0.50                        sub	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  1      3     0.50                        sub	z0.s, z0.s, #0
+# CHECK-NEXT:  1      3     0.50                        sub	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  1      3     0.50                        sub	z0.s, z0.s, z0.s
+# CHECK-NEXT:  1      3     0.50                        sub	z21.b, p5/m, z21.b, z10.b
+# CHECK-NEXT:  1      3     0.50                        sub	z21.b, z10.b, z21.b
+# CHECK-NEXT:  1      3     0.50                        sub	z21.d, p5/m, z21.d, z10.d
+# CHECK-NEXT:  1      3     0.50                        sub	z21.d, z10.d, z21.d
+# CHECK-NEXT:  1      3     0.50                        sub	z21.h, p5/m, z21.h, z10.h
+# CHECK-NEXT:  1      3     0.50                        sub	z21.h, z10.h, z21.h
+# CHECK-NEXT:  1      3     0.50                        sub	z21.s, p5/m, z21.s, z10.s
+# CHECK-NEXT:  1      3     0.50                        sub	z21.s, z10.s, z21.s
+# CHECK-NEXT:  1      3     0.50                        sub	z23.b, p3/m, z23.b, z13.b
+# CHECK-NEXT:  1      3     0.50                        sub	z23.b, z13.b, z8.b
+# CHECK-NEXT:  1      3     0.50                        sub	z23.d, p3/m, z23.d, z13.d
+# CHECK-NEXT:  1      3     0.50                        sub	z23.d, z13.d, z8.d
+# CHECK-NEXT:  1      3     0.50                        sub	z23.h, p3/m, z23.h, z13.h
+# CHECK-NEXT:  1      3     0.50                        sub	z23.h, z13.h, z8.h
+# CHECK-NEXT:  1      3     0.50                        sub	z23.s, p3/m, z23.s, z13.s
+# CHECK-NEXT:  1      3     0.50                        sub	z23.s, z13.s, z8.s
+# CHECK-NEXT:  1      3     0.50                        sub	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  1      3     0.50                        sub	z31.b, z31.b, #255
+# CHECK-NEXT:  1      3     0.50                        sub	z31.b, z31.b, z31.b
+# CHECK-NEXT:  1      3     0.50                        sub	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  1      3     0.50                        sub	z31.d, z31.d, #65280
+# CHECK-NEXT:  1      3     0.50                        sub	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      3     0.50                        sub	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        sub	z31.h, z31.h, #65280
+# CHECK-NEXT:  1      3     0.50                        sub	z31.h, z31.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        sub	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  1      3     0.50                        sub	z31.s, z31.s, #65280
+# CHECK-NEXT:  1      3     0.50                        sub	z31.s, z31.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        subhnb	z0.b, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        subhnb	z0.h, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        subhnb	z0.s, z1.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        subhnt	z0.b, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        subhnt	z0.h, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        subhnt	z0.s, z1.d, z31.d
+# CHECK-NEXT:  1      3     0.50                        subr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  1      3     0.50                        subr	z0.b, z0.b, #0
+# CHECK-NEXT:  1      3     0.50                        subr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  1      3     0.50                        subr	z0.d, z0.d, #0
+# CHECK-NEXT:  1      3     0.50                        subr	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  1      3     0.50                        subr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  1      3     0.50                        subr	z0.h, z0.h, #0
+# CHECK-NEXT:  1      3     0.50                        subr	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  1      3     0.50                        subr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  1      3     0.50                        subr	z0.s, z0.s, #0
+# CHECK-NEXT:  1      3     0.50                        subr	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  1      3     0.50                        subr	z31.b, z31.b, #255
+# CHECK-NEXT:  1      3     0.50                        subr	z31.d, z31.d, #65280
+# CHECK-NEXT:  1      3     0.50                        subr	z31.h, z31.h, #65280
+# CHECK-NEXT:  1      3     0.50                        subr	z31.s, z31.s, #65280
+# CHECK-NEXT:  1      4     0.50                        sunpkhi	z31.d, z31.s
+# CHECK-NEXT:  1      4     0.50                        sunpkhi	z31.h, z31.b
+# CHECK-NEXT:  1      4     0.50                        sunpkhi	z31.s, z31.h
+# CHECK-NEXT:  1      4     0.50                        sunpklo	z31.d, z31.s
+# CHECK-NEXT:  1      4     0.50                        sunpklo	z31.h, z31.b
+# CHECK-NEXT:  1      4     0.50                        sunpklo	z31.s, z31.h
+# CHECK-NEXT:  1      4     0.50                        suqadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     0.50                        suqadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        suqadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        suqadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      3     0.50                        sxtb	z0.d, p0/m, z0.d
+# CHECK-NEXT:  1      3     0.50                        sxtb	z0.h, p0/m, z0.h
+# CHECK-NEXT:  1      3     0.50                        sxtb	z0.s, p0/m, z0.s
+# CHECK-NEXT:  1      3     0.50                        sxtb	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      3     0.50                        sxtb	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      3     0.50                        sxtb	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      3     0.50                        sxth	z0.d, p0/m, z0.d
+# CHECK-NEXT:  1      3     0.50                        sxth	z0.s, p0/m, z0.s
+# CHECK-NEXT:  1      3     0.50                        sxth	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      3     0.50                        sxth	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      3     0.50                        sxtw	z0.d, p0/m, z0.d
+# CHECK-NEXT:  1      3     0.50                        sxtw	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      4     0.50                        tbl	z28.b, { z29.b, z30.b }, z31.b
+# CHECK-NEXT:  1      4     0.50                        tbl	z28.d, { z29.d, z30.d }, z31.d
+# CHECK-NEXT:  1      4     0.50                        tbl	z28.h, { z29.h, z30.h }, z31.h
+# CHECK-NEXT:  1      4     0.50                        tbl	z28.s, { z29.s, z30.s }, z31.s
+# CHECK-NEXT:  1      4     0.50                        tbl	z31.b, { z31.b }, z31.b
+# CHECK-NEXT:  1      4     0.50                        tbl	z31.d, { z31.d }, z31.d
+# CHECK-NEXT:  1      4     0.50                        tbl	z31.h, { z31.h }, z31.h
+# CHECK-NEXT:  1      4     0.50                        tbl	z31.s, { z31.s }, z31.s
+# CHECK-NEXT:  1      4     0.50                        tbx	z31.b, z31.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        tbx	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        tbx	z31.h, z31.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        tbx	z31.s, z31.s, z31.s
+# CHECK-NEXT:  1      6     1.00                        trn1	p15.b, p15.b, p15.b
+# CHECK-NEXT:  1      6     1.00                        trn1	p15.d, p15.d, p15.d
+# CHECK-NEXT:  1      6     1.00                        trn1	p15.h, p15.h, p15.h
+# CHECK-NEXT:  1      6     1.00                        trn1	p15.s, p15.s, p15.s
+# CHECK-NEXT:  1      4     0.50                        trn1	z31.b, z31.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        trn1	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        trn1	z31.h, z31.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        trn1	z31.s, z31.s, z31.s
+# CHECK-NEXT:  1      6     1.00                        trn2	p15.b, p15.b, p15.b
+# CHECK-NEXT:  1      6     1.00                        trn2	p15.d, p15.d, p15.d
+# CHECK-NEXT:  1      6     1.00                        trn2	p15.h, p15.h, p15.h
+# CHECK-NEXT:  1      6     1.00                        trn2	p15.s, p15.s, p15.s
+# CHECK-NEXT:  1      4     0.50                        trn2	z31.b, z31.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        trn2	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        trn2	z31.h, z31.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        trn2	z31.s, z31.s, z31.s
+# CHECK-NEXT:  1      8     1.00                        uaba	z0.b, z1.b, z31.b
+# CHECK-NEXT:  1      8     1.00                        uaba	z0.d, z1.d, z31.d
+# CHECK-NEXT:  1      8     1.00                        uaba	z0.h, z1.h, z31.h
+# CHECK-NEXT:  1      8     1.00                        uaba	z0.s, z1.s, z31.s
+# CHECK-NEXT:  1      8     1.00                        uabalb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      8     1.00                        uabalb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      8     1.00                        uabalb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      8     1.00                        uabalt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      8     1.00                        uabalt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      8     1.00                        uabalt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        uabd	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  1      3     0.50                        uabd	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  1      3     0.50                        uabd	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        uabd	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  1      3     0.50                        uabdlb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      3     0.50                        uabdlb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        uabdlb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      3     0.50                        uabdlt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      3     0.50                        uabdlt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        uabdlt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      7     1.00                        uadalp	z0.h, p0/m, z1.b
+# CHECK-NEXT:  1      7     1.00                        uadalp	z29.s, p0/m, z30.h
+# CHECK-NEXT:  1      7     1.00                        uadalp	z30.d, p7/m, z31.s
+# CHECK-NEXT:  1      3     0.50                        uaddlb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      3     0.50                        uaddlb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        uaddlb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      3     0.50                        uaddlt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      3     0.50                        uaddlt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        uaddlt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      4     1.00                        uaddv	d0, p7, z31.b
+# CHECK-NEXT:  1      4     1.00                        uaddv	d0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        uaddv	d0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        uaddv	d0, p7, z31.s
+# CHECK-NEXT:  1      3     0.50                        uaddwb	z0.h, z1.h, z2.b
+# CHECK-NEXT:  1      3     0.50                        uaddwb	z29.s, z30.s, z31.h
+# CHECK-NEXT:  1      3     0.50                        uaddwb	z31.d, z31.d, z31.s
+# CHECK-NEXT:  1      3     0.50                        uaddwt	z0.h, z1.h, z2.b
+# CHECK-NEXT:  1      3     0.50                        uaddwt	z29.s, z30.s, z31.h
+# CHECK-NEXT:  1      3     0.50                        uaddwt	z31.d, z31.d, z31.s
+# CHECK-NEXT:  1      4     0.50                        ucvtf	z0.d, p0/m, z0.d
+# CHECK-NEXT:  1      4     0.50                        ucvtf	z0.d, p0/m, z0.s
+# CHECK-NEXT:  1      4     0.50                        ucvtf	z0.h, p0/m, z0.d
+# CHECK-NEXT:  1      4     0.50                        ucvtf	z0.h, p0/m, z0.h
+# CHECK-NEXT:  1      4     0.50                        ucvtf	z0.h, p0/m, z0.s
+# CHECK-NEXT:  1      4     0.50                        ucvtf	z0.s, p0/m, z0.d
+# CHECK-NEXT:  1      4     0.50                        ucvtf	z0.s, p0/m, z0.s
+# CHECK-NEXT:  1      26    23.00                       udiv	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      15    12.00                       udiv	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      26    23.00                       udivr	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      15    12.00                       udivr	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        udot	z0.d, z1.h, z15.h[1]
+# CHECK-NEXT:  1      4     0.50                        udot	z0.d, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        udot	z0.s, z1.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        udot	z0.s, z1.b, z7.b[3]
+# CHECK-NEXT:  1      3     0.50                        uhadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      3     0.50                        uhadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      3     0.50                        uhadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      3     0.50                        uhadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      3     0.50                        uhsub	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      3     0.50                        uhsub	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      3     0.50                        uhsub	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      3     0.50                        uhsub	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      3     0.50                        uhsubr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      3     0.50                        uhsubr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      3     0.50                        uhsubr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      3     0.50                        uhsubr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      3     0.50                        umax	z0.b, z0.b, #0
+# CHECK-NEXT:  1      3     0.50                        umax	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  1      3     0.50                        umax	z31.b, z31.b, #255
+# CHECK-NEXT:  1      3     0.50                        umax	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  1      3     0.50                        umax	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        umax	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  1      3     0.50                        umaxp	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      3     0.50                        umaxp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      3     0.50                        umaxp	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      3     0.50                        umaxp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        umaxv	b0, p7, z31.b
+# CHECK-NEXT:  1      4     1.00                        umaxv	d0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        umaxv	h0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        umaxv	s0, p7, z31.s
+# CHECK-NEXT:  1      3     0.50                        umin	z0.b, z0.b, #0
+# CHECK-NEXT:  1      3     0.50                        umin	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  1      3     0.50                        umin	z31.b, z31.b, #255
+# CHECK-NEXT:  1      3     0.50                        umin	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  1      3     0.50                        umin	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        umin	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  1      3     0.50                        uminp	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      3     0.50                        uminp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      3     0.50                        uminp	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      3     0.50                        uminp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     1.00                        uminv	b0, p7, z31.b
+# CHECK-NEXT:  1      4     1.00                        uminv	d0, p7, z31.d
+# CHECK-NEXT:  1      4     1.00                        uminv	h0, p7, z31.h
+# CHECK-NEXT:  1      4     1.00                        uminv	s0, p7, z31.s
+# CHECK-NEXT:  1      4     0.50                        umlalb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     0.50                        umlalb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        umlalb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        umlalb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        umlalb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        umlalt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     0.50                        umlalt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        umlalt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        umlalt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        umlalt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        umlslb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     0.50                        umlslb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        umlslb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        umlslb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        umlslb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        umlslt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     0.50                        umlslt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        umlslt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        umlslt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        umlslt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        ummla	z0.s, z1.b, z2.b
+# CHECK-NEXT:  1      4     0.50                        umulh	z0.b, p7/m, z0.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        umulh	z0.b, z1.b, z2.b
+# CHECK-NEXT:  1      4     0.50                        umulh	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        umulh	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        umulh	z0.h, z1.h, z2.h
+# CHECK-NEXT:  1      4     0.50                        umulh	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        umulh	z29.s, z30.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        umulh	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        umullb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     0.50                        umullb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      4     0.50                        umullb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        umullb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        umullb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        umullt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  1      4     0.50                        umullt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      4     0.50                        umullt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  1      4     0.50                        umullt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        umullt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      4     0.50                        uqadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     0.50                        uqadd	z0.b, z0.b, #0
+# CHECK-NEXT:  1      4     0.50                        uqadd	z0.b, z0.b, z0.b
+# CHECK-NEXT:  1      4     0.50                        uqadd	z0.d, z0.d, #0
+# CHECK-NEXT:  1      4     0.50                        uqadd	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  1      4     0.50                        uqadd	z0.d, z0.d, z0.d
+# CHECK-NEXT:  1      4     0.50                        uqadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        uqadd	z0.h, z0.h, #0
+# CHECK-NEXT:  1      4     0.50                        uqadd	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  1      4     0.50                        uqadd	z0.h, z0.h, z0.h
+# CHECK-NEXT:  1      4     0.50                        uqadd	z0.s, z0.s, #0
+# CHECK-NEXT:  1      4     0.50                        uqadd	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  1      4     0.50                        uqadd	z0.s, z0.s, z0.s
+# CHECK-NEXT:  1      4     0.50                        uqadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        uqadd	z31.b, z31.b, #255
+# CHECK-NEXT:  1      4     0.50                        uqadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     0.50                        uqadd	z31.d, z31.d, #65280
+# CHECK-NEXT:  1      4     0.50                        uqadd	z31.h, z31.h, #65280
+# CHECK-NEXT:  1      4     0.50                        uqadd	z31.s, z31.s, #65280
+# CHECK-NEXT:  1      1     0.33                        uqdecb	w0
+# CHECK-NEXT:  1      1     0.33                        uqdecb	w0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqdecb	w0, pow2
+# CHECK-NEXT:  1      1     0.33                        uqdecb	w0, pow2, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqdecb	x0
+# CHECK-NEXT:  1      1     0.33                        uqdecb	x0, #14
+# CHECK-NEXT:  1      1     0.33                        uqdecb	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqdecb	x0, pow2
+# CHECK-NEXT:  1      1     0.33                        uqdecb	x0, vl1
+# CHECK-NEXT:  1      1     0.33                        uqdecd	w0
+# CHECK-NEXT:  1      1     0.33                        uqdecd	w0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqdecd	w0, pow2
+# CHECK-NEXT:  1      1     0.33                        uqdecd	w0, pow2, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqdecd	x0
+# CHECK-NEXT:  1      1     0.33                        uqdecd	x0, #14
+# CHECK-NEXT:  1      1     0.33                        uqdecd	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqdecd	x0, pow2
+# CHECK-NEXT:  1      1     0.33                        uqdecd	x0, vl1
+# CHECK-NEXT:  1      4     0.50                        uqdecd	z0.d
+# CHECK-NEXT:  1      4     0.50                        uqdecd	z0.d, all, mul #16
+# CHECK-NEXT:  1      4     0.50                        uqdecd	z0.d, pow2
+# CHECK-NEXT:  1      4     0.50                        uqdecd	z0.d, pow2, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqdech	w0
+# CHECK-NEXT:  1      1     0.33                        uqdech	w0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqdech	w0, pow2
+# CHECK-NEXT:  1      1     0.33                        uqdech	w0, pow2, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqdech	x0
+# CHECK-NEXT:  1      1     0.33                        uqdech	x0, #14
+# CHECK-NEXT:  1      1     0.33                        uqdech	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqdech	x0, pow2
+# CHECK-NEXT:  1      1     0.33                        uqdech	x0, vl1
+# CHECK-NEXT:  1      4     0.50                        uqdech	z0.h
+# CHECK-NEXT:  1      4     0.50                        uqdech	z0.h, all, mul #16
+# CHECK-NEXT:  1      4     0.50                        uqdech	z0.h, pow2
+# CHECK-NEXT:  1      4     0.50                        uqdech	z0.h, pow2, mul #16
+# CHECK-NEXT:  1      8     1.00                        uqdecp	wzr, p15.b
+# CHECK-NEXT:  1      8     1.00                        uqdecp	wzr, p15.d
+# CHECK-NEXT:  1      8     1.00                        uqdecp	wzr, p15.h
+# CHECK-NEXT:  1      8     1.00                        uqdecp	wzr, p15.s
+# CHECK-NEXT:  1      8     1.00                        uqdecp	x0, p0.b
+# CHECK-NEXT:  1      8     1.00                        uqdecp	x0, p0.d
+# CHECK-NEXT:  1      8     1.00                        uqdecp	x0, p0.h
+# CHECK-NEXT:  1      8     1.00                        uqdecp	x0, p0.s
+# CHECK-NEXT:  1      4     0.50                        uqdecp	z0.d, p0.d
+# CHECK-NEXT:  1      4     0.50                        uqdecp	z0.h, p0.h
+# CHECK-NEXT:  1      4     0.50                        uqdecp	z0.s, p0.s
+# CHECK-NEXT:  1      1     0.33                        uqdecw	w0
+# CHECK-NEXT:  1      1     0.33                        uqdecw	w0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqdecw	w0, pow2
+# CHECK-NEXT:  1      1     0.33                        uqdecw	w0, pow2, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqdecw	x0
+# CHECK-NEXT:  1      1     0.33                        uqdecw	x0, #14
+# CHECK-NEXT:  1      1     0.33                        uqdecw	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqdecw	x0, pow2
+# CHECK-NEXT:  1      1     0.33                        uqdecw	x0, vl1
+# CHECK-NEXT:  1      4     0.50                        uqdecw	z0.s
+# CHECK-NEXT:  1      4     0.50                        uqdecw	z0.s, all, mul #16
+# CHECK-NEXT:  1      4     0.50                        uqdecw	z0.s, pow2
+# CHECK-NEXT:  1      4     0.50                        uqdecw	z0.s, pow2, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqincb	w0
+# CHECK-NEXT:  1      1     0.33                        uqincb	w0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqincb	w0, pow2
+# CHECK-NEXT:  1      1     0.33                        uqincb	w0, pow2, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqincb	x0
+# CHECK-NEXT:  1      1     0.33                        uqincb	x0, #14
+# CHECK-NEXT:  1      1     0.33                        uqincb	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqincb	x0, pow2
+# CHECK-NEXT:  1      1     0.33                        uqincb	x0, vl1
+# CHECK-NEXT:  1      1     0.33                        uqincd	w0
+# CHECK-NEXT:  1      1     0.33                        uqincd	w0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqincd	w0, pow2
+# CHECK-NEXT:  1      1     0.33                        uqincd	w0, pow2, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqincd	x0
+# CHECK-NEXT:  1      1     0.33                        uqincd	x0, #14
+# CHECK-NEXT:  1      1     0.33                        uqincd	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqincd	x0, pow2
+# CHECK-NEXT:  1      1     0.33                        uqincd	x0, vl1
+# CHECK-NEXT:  1      4     0.50                        uqincd	z0.d
+# CHECK-NEXT:  1      4     0.50                        uqincd	z0.d, all, mul #16
+# CHECK-NEXT:  1      4     0.50                        uqincd	z0.d, pow2
+# CHECK-NEXT:  1      4     0.50                        uqincd	z0.d, pow2, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqinch	w0
+# CHECK-NEXT:  1      1     0.33                        uqinch	w0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqinch	w0, pow2
+# CHECK-NEXT:  1      1     0.33                        uqinch	w0, pow2, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqinch	x0
+# CHECK-NEXT:  1      1     0.33                        uqinch	x0, #14
+# CHECK-NEXT:  1      1     0.33                        uqinch	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqinch	x0, pow2
+# CHECK-NEXT:  1      1     0.33                        uqinch	x0, vl1
+# CHECK-NEXT:  1      4     0.50                        uqinch	z0.h
+# CHECK-NEXT:  1      4     0.50                        uqinch	z0.h, all, mul #16
+# CHECK-NEXT:  1      4     0.50                        uqinch	z0.h, pow2
+# CHECK-NEXT:  1      4     0.50                        uqinch	z0.h, pow2, mul #16
+# CHECK-NEXT:  1      8     1.00                        uqincp	wzr, p15.b
+# CHECK-NEXT:  1      8     1.00                        uqincp	wzr, p15.d
+# CHECK-NEXT:  1      8     1.00                        uqincp	wzr, p15.h
+# CHECK-NEXT:  1      8     1.00                        uqincp	wzr, p15.s
+# CHECK-NEXT:  1      8     1.00                        uqincp	x0, p0.b
+# CHECK-NEXT:  1      8     1.00                        uqincp	x0, p0.d
+# CHECK-NEXT:  1      8     1.00                        uqincp	x0, p0.h
+# CHECK-NEXT:  1      8     1.00                        uqincp	x0, p0.s
+# CHECK-NEXT:  1      4     0.50                        uqincp	z0.d, p0.d
+# CHECK-NEXT:  1      4     0.50                        uqincp	z0.h, p0.h
+# CHECK-NEXT:  1      4     0.50                        uqincp	z0.s, p0.s
+# CHECK-NEXT:  1      1     0.33                        uqincw	w0
+# CHECK-NEXT:  1      1     0.33                        uqincw	w0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqincw	w0, pow2
+# CHECK-NEXT:  1      1     0.33                        uqincw	w0, pow2, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqincw	x0
+# CHECK-NEXT:  1      1     0.33                        uqincw	x0, #14
+# CHECK-NEXT:  1      1     0.33                        uqincw	x0, all, mul #16
+# CHECK-NEXT:  1      1     0.33                        uqincw	x0, pow2
+# CHECK-NEXT:  1      1     0.33                        uqincw	x0, vl1
+# CHECK-NEXT:  1      4     0.50                        uqincw	z0.s
+# CHECK-NEXT:  1      4     0.50                        uqincw	z0.s, all, mul #16
+# CHECK-NEXT:  1      4     0.50                        uqincw	z0.s, pow2
+# CHECK-NEXT:  1      4     0.50                        uqincw	z0.s, pow2, mul #16
+# CHECK-NEXT:  1      4     0.50                        uqrshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     0.50                        uqrshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        uqrshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        uqrshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     0.50                        uqrshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     0.50                        uqrshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        uqrshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        uqrshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     0.50                        uqrshrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     0.50                        uqrshrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     0.50                        uqrshrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     0.50                        uqrshrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     0.50                        uqrshrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     0.50                        uqrshrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     0.50                        uqrshrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     0.50                        uqrshrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     0.50                        uqrshrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     0.50                        uqrshrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     0.50                        uqrshrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     0.50                        uqrshrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     0.50                        uqshl	z0.b, p0/m, z0.b, #0
+# CHECK-NEXT:  1      4     0.50                        uqshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     0.50                        uqshl	z0.d, p0/m, z0.d, #0
+# CHECK-NEXT:  1      4     0.50                        uqshl	z0.h, p0/m, z0.h, #0
+# CHECK-NEXT:  1      4     0.50                        uqshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        uqshl	z0.s, p0/m, z0.s, #0
+# CHECK-NEXT:  1      4     0.50                        uqshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        uqshl	z31.b, p0/m, z31.b, #7
+# CHECK-NEXT:  1      4     0.50                        uqshl	z31.d, p0/m, z31.d, #63
+# CHECK-NEXT:  1      4     0.50                        uqshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     0.50                        uqshl	z31.h, p0/m, z31.h, #15
+# CHECK-NEXT:  1      4     0.50                        uqshl	z31.s, p0/m, z31.s, #31
+# CHECK-NEXT:  1      4     0.50                        uqshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     0.50                        uqshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        uqshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        uqshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     0.50                        uqshrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     0.50                        uqshrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     0.50                        uqshrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     0.50                        uqshrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     0.50                        uqshrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     0.50                        uqshrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     0.50                        uqshrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  1      4     0.50                        uqshrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  1      4     0.50                        uqshrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  1      4     0.50                        uqshrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  1      4     0.50                        uqshrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  1      4     0.50                        uqshrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  1      4     0.50                        uqsub	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     0.50                        uqsub	z0.b, z0.b, #0
+# CHECK-NEXT:  1      4     0.50                        uqsub	z0.b, z0.b, z0.b
+# CHECK-NEXT:  1      4     0.50                        uqsub	z0.d, z0.d, #0
+# CHECK-NEXT:  1      4     0.50                        uqsub	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  1      4     0.50                        uqsub	z0.d, z0.d, z0.d
+# CHECK-NEXT:  1      4     0.50                        uqsub	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        uqsub	z0.h, z0.h, #0
+# CHECK-NEXT:  1      4     0.50                        uqsub	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  1      4     0.50                        uqsub	z0.h, z0.h, z0.h
+# CHECK-NEXT:  1      4     0.50                        uqsub	z0.s, z0.s, #0
+# CHECK-NEXT:  1      4     0.50                        uqsub	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  1      4     0.50                        uqsub	z0.s, z0.s, z0.s
+# CHECK-NEXT:  1      4     0.50                        uqsub	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        uqsub	z31.b, z31.b, #255
+# CHECK-NEXT:  1      4     0.50                        uqsub	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     0.50                        uqsub	z31.d, z31.d, #65280
+# CHECK-NEXT:  1      4     0.50                        uqsub	z31.h, z31.h, #65280
+# CHECK-NEXT:  1      4     0.50                        uqsub	z31.s, z31.s, #65280
+# CHECK-NEXT:  1      4     0.50                        uqsubr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     0.50                        uqsubr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        uqsubr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        uqsubr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     0.50                        uqxtnb	z0.b, z31.h
+# CHECK-NEXT:  1      4     0.50                        uqxtnb	z0.h, z31.s
+# CHECK-NEXT:  1      4     0.50                        uqxtnb	z0.s, z31.d
+# CHECK-NEXT:  1      4     0.50                        uqxtnt	z0.b, z31.h
+# CHECK-NEXT:  1      4     0.50                        uqxtnt	z0.h, z31.s
+# CHECK-NEXT:  1      4     0.50                        uqxtnt	z0.s, z31.d
+# CHECK-NEXT:  1      4     0.50                        urecpe	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      4     0.50                        urhadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     0.50                        urhadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        urhadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        urhadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     0.50                        urshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     0.50                        urshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        urshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        urshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     0.50                        urshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     0.50                        urshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        urshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        urshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     0.50                        urshr	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  1      4     0.50                        urshr	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  1      4     0.50                        urshr	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  1      4     0.50                        urshr	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  1      4     0.50                        urshr	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  1      4     0.50                        urshr	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  1      4     0.50                        urshr	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  1      4     0.50                        urshr	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  1      4     0.50                        ursqrte	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      7     1.00                        ursra	z0.b, z0.b, #1
+# CHECK-NEXT:  1      7     1.00                        ursra	z0.d, z0.d, #1
+# CHECK-NEXT:  1      7     1.00                        ursra	z0.h, z0.h, #1
+# CHECK-NEXT:  1      7     1.00                        ursra	z0.s, z0.s, #1
+# CHECK-NEXT:  1      7     1.00                        ursra	z31.b, z31.b, #8
+# CHECK-NEXT:  1      7     1.00                        ursra	z31.d, z31.d, #64
+# CHECK-NEXT:  1      7     1.00                        ursra	z31.h, z31.h, #16
+# CHECK-NEXT:  1      7     1.00                        ursra	z31.s, z31.s, #32
+# CHECK-NEXT:  1      3     0.50                        ushllb	z0.d, z0.s, #0
+# CHECK-NEXT:  1      3     0.50                        ushllb	z0.h, z0.b, #0
+# CHECK-NEXT:  1      3     0.50                        ushllb	z0.s, z0.h, #0
+# CHECK-NEXT:  1      3     0.50                        ushllb	z31.d, z31.s, #31
+# CHECK-NEXT:  1      3     0.50                        ushllb	z31.h, z31.b, #7
+# CHECK-NEXT:  1      3     0.50                        ushllb	z31.s, z31.h, #15
+# CHECK-NEXT:  1      3     0.50                        ushllt	z0.d, z0.s, #0
+# CHECK-NEXT:  1      3     0.50                        ushllt	z0.h, z0.b, #0
+# CHECK-NEXT:  1      3     0.50                        ushllt	z0.s, z0.h, #0
+# CHECK-NEXT:  1      3     0.50                        ushllt	z31.d, z31.s, #31
+# CHECK-NEXT:  1      3     0.50                        ushllt	z31.h, z31.b, #7
+# CHECK-NEXT:  1      3     0.50                        ushllt	z31.s, z31.h, #15
+# CHECK-NEXT:  1      4     0.50                        usmmla	z0.s, z1.b, z2.b
+# CHECK-NEXT:  1      4     0.50                        usqadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  1      4     0.50                        usqadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  1      4     0.50                        usqadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  1      4     0.50                        usqadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  1      4     0.50                        usra	z0.b, z0.b, #1
+# CHECK-NEXT:  1      4     0.50                        usra	z0.d, z0.d, #1
+# CHECK-NEXT:  1      4     0.50                        usra	z0.h, z0.h, #1
+# CHECK-NEXT:  1      4     0.50                        usra	z0.s, z0.s, #1
+# CHECK-NEXT:  1      4     0.50                        usra	z31.b, z31.b, #8
+# CHECK-NEXT:  1      4     0.50                        usra	z31.d, z31.d, #64
+# CHECK-NEXT:  1      4     0.50                        usra	z31.h, z31.h, #16
+# CHECK-NEXT:  1      4     0.50                        usra	z31.s, z31.s, #32
+# CHECK-NEXT:  1      3     0.50                        usublb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      3     0.50                        usublb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        usublb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      3     0.50                        usublt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  1      3     0.50                        usublt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  1      3     0.50                        usublt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  1      3     0.50                        usubwb	z0.h, z1.h, z2.b
+# CHECK-NEXT:  1      3     0.50                        usubwb	z29.s, z30.s, z31.h
+# CHECK-NEXT:  1      3     0.50                        usubwb	z31.d, z31.d, z31.s
+# CHECK-NEXT:  1      3     0.50                        usubwt	z0.h, z1.h, z2.b
+# CHECK-NEXT:  1      3     0.50                        usubwt	z29.s, z30.s, z31.h
+# CHECK-NEXT:  1      3     0.50                        usubwt	z31.d, z31.d, z31.s
+# CHECK-NEXT:  1      4     0.50                        uunpkhi	z31.d, z31.s
+# CHECK-NEXT:  1      4     0.50                        uunpkhi	z31.h, z31.b
+# CHECK-NEXT:  1      4     0.50                        uunpkhi	z31.s, z31.h
+# CHECK-NEXT:  1      4     0.50                        uunpklo	z31.d, z31.s
+# CHECK-NEXT:  1      4     0.50                        uunpklo	z31.h, z31.b
+# CHECK-NEXT:  1      4     0.50                        uunpklo	z31.s, z31.h
+# CHECK-NEXT:  1      3     0.50                        uxtb	z0.d, p0/m, z0.d
+# CHECK-NEXT:  1      3     0.50                        uxtb	z0.h, p0/m, z0.h
+# CHECK-NEXT:  1      3     0.50                        uxtb	z0.s, p0/m, z0.s
+# CHECK-NEXT:  1      3     0.50                        uxtb	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      3     0.50                        uxtb	z31.h, p7/m, z31.h
+# CHECK-NEXT:  1      3     0.50                        uxtb	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      3     0.50                        uxth	z0.d, p0/m, z0.d
+# CHECK-NEXT:  1      3     0.50                        uxth	z0.s, p0/m, z0.s
+# CHECK-NEXT:  1      3     0.50                        uxth	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      3     0.50                        uxth	z31.s, p7/m, z31.s
+# CHECK-NEXT:  1      3     0.50                        uxtw	z0.d, p0/m, z0.d
+# CHECK-NEXT:  1      3     0.50                        uxtw	z31.d, p7/m, z31.d
+# CHECK-NEXT:  1      6     1.00                        uzp1	p15.b, p15.b, p15.b
+# CHECK-NEXT:  1      6     1.00                        uzp1	p15.d, p15.d, p15.d
+# CHECK-NEXT:  1      6     1.00                        uzp1	p15.h, p15.h, p15.h
+# CHECK-NEXT:  1      6     1.00                        uzp1	p15.s, p15.s, p15.s
+# CHECK-NEXT:  1      4     0.50                        uzp1	z31.b, z31.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        uzp1	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        uzp1	z31.h, z31.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        uzp1	z31.s, z31.s, z31.s
+# CHECK-NEXT:  1      6     1.00                        uzp2	p15.b, p15.b, p15.b
+# CHECK-NEXT:  1      6     1.00                        uzp2	p15.d, p15.d, p15.d
+# CHECK-NEXT:  1      6     1.00                        uzp2	p15.h, p15.h, p15.h
+# CHECK-NEXT:  1      6     1.00                        uzp2	p15.s, p15.s, p15.s
+# CHECK-NEXT:  1      4     0.50                        uzp2	z31.b, z31.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        uzp2	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        uzp2	z31.h, z31.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        uzp2	z31.s, z31.s, z31.s
+# CHECK-NEXT:  1      6     1.00                        whilege	p15.b, w0, wzr
+# CHECK-NEXT:  1      6     1.00                        whilege	p15.b, wzr, w0
+# CHECK-NEXT:  1      6     1.00                        whilege	p15.b, x0, xzr
+# CHECK-NEXT:  1      6     1.00                        whilege	p15.b, xzr, x0
+# CHECK-NEXT:  1      6     1.00                        whilege	p15.d, w0, wzr
+# CHECK-NEXT:  1      6     1.00                        whilege	p15.d, x0, xzr
+# CHECK-NEXT:  1      6     1.00                        whilege	p15.h, w0, wzr
+# CHECK-NEXT:  1      6     1.00                        whilege	p15.h, x0, xzr
+# CHECK-NEXT:  1      6     1.00                        whilege	p15.s, w0, wzr
+# CHECK-NEXT:  1      6     1.00                        whilege	p15.s, x0, xzr
+# CHECK-NEXT:  1      6     1.00                        whilerw	p15.b, x30, x30
+# CHECK-NEXT:  1      6     1.00                        whilerw	p15.d, x30, x30
+# CHECK-NEXT:  1      6     1.00                        whilerw	p15.h, x30, x30
+# CHECK-NEXT:  1      6     1.00                        whilerw	p15.s, x30, x30
+# CHECK-NEXT:  1      6     1.00                        whilewr	p15.b, x30, x30
+# CHECK-NEXT:  1      6     1.00                        whilewr	p15.d, x30, x30
+# CHECK-NEXT:  1      6     1.00                        whilewr	p15.h, x30, x30
+# CHECK-NEXT:  1      6     1.00                        whilewr	p15.s, x30, x30
+# CHECK-NEXT:  1      1     0.33           *      U     wrffr	p0.b
+# CHECK-NEXT:  1      1     0.33           *      U     wrffr	p15.b
+# CHECK-NEXT:  1      3     0.50                        xar	z0.b, z0.b, z1.b, #1
+# CHECK-NEXT:  1      3     0.50                        xar	z0.d, z0.d, z1.d, #1
+# CHECK-NEXT:  1      3     0.50                        xar	z0.h, z0.h, z1.h, #1
+# CHECK-NEXT:  1      3     0.50                        xar	z0.s, z0.s, z1.s, #1
+# CHECK-NEXT:  1      3     0.50                        xar	z31.b, z31.b, z30.b, #8
+# CHECK-NEXT:  1      3     0.50                        xar	z31.d, z31.d, z30.d, #64
+# CHECK-NEXT:  1      3     0.50                        xar	z31.h, z31.h, z30.h, #16
+# CHECK-NEXT:  1      3     0.50                        xar	z31.s, z31.s, z30.s, #32
+# CHECK-NEXT:  1      6     1.00                        zip1	p0.b, p0.b, p0.b
+# CHECK-NEXT:  1      6     1.00                        zip1	p0.d, p0.d, p0.d
+# CHECK-NEXT:  1      6     1.00                        zip1	p0.h, p0.h, p0.h
+# CHECK-NEXT:  1      6     1.00                        zip1	p0.s, p0.s, p0.s
+# CHECK-NEXT:  1      6     1.00                        zip1	p15.b, p15.b, p15.b
+# CHECK-NEXT:  1      6     1.00                        zip1	p15.d, p15.d, p15.d
+# CHECK-NEXT:  1      6     1.00                        zip1	p15.h, p15.h, p15.h
+# CHECK-NEXT:  1      6     1.00                        zip1	p15.s, p15.s, p15.s
+# CHECK-NEXT:  1      4     0.50                        zip1	z0.b, z0.b, z0.b
+# CHECK-NEXT:  1      4     0.50                        zip1	z0.d, z0.d, z0.d
+# CHECK-NEXT:  1      4     0.50                        zip1	z0.h, z0.h, z0.h
+# CHECK-NEXT:  1      4     0.50                        zip1	z0.s, z0.s, z0.s
+# CHECK-NEXT:  1      4     0.50                        zip1	z31.b, z31.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        zip1	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        zip1	z31.h, z31.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        zip1	z31.s, z31.s, z31.s
+# CHECK-NEXT:  1      6     1.00                        zip2	p0.b, p0.b, p0.b
+# CHECK-NEXT:  1      6     1.00                        zip2	p0.d, p0.d, p0.d
+# CHECK-NEXT:  1      6     1.00                        zip2	p0.h, p0.h, p0.h
+# CHECK-NEXT:  1      6     1.00                        zip2	p0.s, p0.s, p0.s
+# CHECK-NEXT:  1      6     1.00                        zip2	p15.b, p15.b, p15.b
+# CHECK-NEXT:  1      6     1.00                        zip2	p15.d, p15.d, p15.d
+# CHECK-NEXT:  1      6     1.00                        zip2	p15.h, p15.h, p15.h
+# CHECK-NEXT:  1      6     1.00                        zip2	p15.s, p15.s, p15.s
+# CHECK-NEXT:  1      4     0.50                        zip2	z0.b, z0.b, z0.b
+# CHECK-NEXT:  1      4     0.50                        zip2	z0.d, z0.d, z0.d
+# CHECK-NEXT:  1      4     0.50                        zip2	z0.h, z0.h, z0.h
+# CHECK-NEXT:  1      4     0.50                        zip2	z0.s, z0.s, z0.s
+# CHECK-NEXT:  1      4     0.50                        zip2	z31.b, z31.b, z31.b
+# CHECK-NEXT:  1      4     0.50                        zip2	z31.d, z31.d, z31.d
+# CHECK-NEXT:  1      4     0.50                        zip2	z31.h, z31.h, z31.h
+# CHECK-NEXT:  1      4     0.50                        zip2	z31.s, z31.s, z31.s
+
+# CHECK:      Resources:
+# CHECK-NEXT: [0]   - CortexA510UnitALU0
+# CHECK-NEXT: [1.0] - CortexA510UnitALU12
+# CHECK-NEXT: [1.1] - CortexA510UnitALU12
+# CHECK-NEXT: [2]   - CortexA510UnitB
+# CHECK-NEXT: [3]   - CortexA510UnitDiv
+# CHECK-NEXT: [4]   - CortexA510UnitLd1
+# CHECK-NEXT: [5]   - CortexA510UnitLdSt
+# CHECK-NEXT: [6]   - CortexA510UnitMAC
+# CHECK-NEXT: [7]   - CortexA510UnitPAC
+# CHECK-NEXT: [8]   - CortexA510UnitVALU0
+# CHECK-NEXT: [9]   - CortexA510UnitVALU1
+# CHECK-NEXT: [10.0] - CortexA510UnitVMAC
+# CHECK-NEXT: [10.1] - CortexA510UnitVMAC
+# CHECK-NEXT: [11]  - CortexA510UnitVMC
+
+# CHECK:      Resource pressure per iteration:
+# CHECK-NEXT: [0]    [1.0]  [1.1]  [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10.0] [10.1] [11]
+# CHECK-NEXT: 79.00  75.00  75.00  9.00    -     240.00 3698.00  -     -     1290.00 924.00 199.50 199.50 670.00
+
+# CHECK:      Resource pressure by instruction:
+# CHECK-NEXT: [0]    [1.0]  [1.1]  [2]    [3]    [4]    [5]    [6]    [7]    [8]    [9]    [10.0] [10.1] [11]   Instructions:
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     abs	z0.b, p0/m, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     abs	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     abs	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     abs	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     abs	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     abs	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     abs	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     abs	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     adclb	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     adclb	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     adclt	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     adclt	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z0.s, z1.s, z2.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z21.b, p5/m, z21.b, z10.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z21.b, z10.b, z21.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z21.d, p5/m, z21.d, z10.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z21.d, z10.d, z21.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z21.h, p5/m, z21.h, z10.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z21.h, z10.h, z21.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z21.s, p5/m, z21.s, z10.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z21.s, z10.s, z21.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z23.b, p3/m, z23.b, z13.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z23.b, z13.b, z8.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z23.d, p3/m, z23.d, z13.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z23.d, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z23.h, p3/m, z23.h, z13.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z23.h, z13.h, z8.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z23.s, p3/m, z23.s, z13.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z23.s, z13.s, z8.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z31.s, z31.s, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     add	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     addhnb	z0.b, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     addhnb	z0.h, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     addhnb	z0.s, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     addhnt	z0.b, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     addhnt	z0.h, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     addhnt	z0.s, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     addp	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     addp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     addp	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     addp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     addpl	sp, sp, #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     addpl	x0, x0, #-32
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     addpl	x21, x21, #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     addpl	x23, x8, #-1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     addvl	sp, sp, #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     addvl	x0, x0, #-32
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     addvl	x21, x21, #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     addvl	x23, x8, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     adr	z0.d, [z0.d, z0.d, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     adr	z0.d, [z0.d, z0.d, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     adr	z0.d, [z0.d, z0.d, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     adr	z0.d, [z0.d, z0.d, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     adr	z0.d, [z0.d, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     adr	z0.d, [z0.d, z0.d, sxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     adr	z0.d, [z0.d, z0.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     adr	z0.d, [z0.d, z0.d, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     adr	z0.d, [z0.d, z0.d, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     adr	z0.d, [z0.d, z0.d, uxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     adr	z0.d, [z0.d, z0.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     adr	z0.d, [z0.d, z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     adr	z0.s, [z0.s, z0.s, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     adr	z0.s, [z0.s, z0.s, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     adr	z0.s, [z0.s, z0.s, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     adr	z0.s, [z0.s, z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     aesd	z0.b, z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     aese	z0.b, z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     aesimc	z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     aesimc	z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     aesmc	z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     aesmc	z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     and	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     and	z0.d, z0.d, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     and	z0.d, z0.d, #0xfffffffffffffff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     and	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     and	z0.s, z0.s, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     and	z0.s, z0.s, #0xfffffff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     and	z23.d, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     and	z23.h, z23.h, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     and	z23.h, z23.h, #0xfff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     and	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     and	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     and	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     and	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     and	z5.b, z5.b, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     and	z5.b, z5.b, #0xf9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ands	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     andv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     andv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     andv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     andv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asr	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asr	z0.b, p0/m, z0.b, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asr	z0.b, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asr	z0.b, z1.b, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asr	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asr	z0.d, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asr	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asr	z0.h, p0/m, z0.h, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asr	z0.h, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asr	z0.h, z1.h, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asr	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asr	z0.s, p0/m, z0.s, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asr	z0.s, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asr	z0.s, z1.s, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asr	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asr	z31.b, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asr	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asr	z31.d, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asr	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asr	z31.h, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asr	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asr	z31.s, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asrd	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asrd	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asrd	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asrd	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asrd	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asrd	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asrd	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asrd	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asrr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asrr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asrr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     asrr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     bcax	z29.d, z29.d, z30.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     13.00  bdep	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     69.00  bdep	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     21.00  bdep	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     37.00  bdep	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     13.00  bext	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     69.00  bext	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     21.00  bext	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     37.00  bext	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     bfcvt	z0.h, p0/m, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     bfcvtnt	z0.h, p0/m, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -     bfdot	z0.s, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -     bfdot	z0.s, z1.h, z2.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -     bfdot	z0.s, z1.h, z2.h[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     bfmlalb	z0.s, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     bfmlalb	z0.s, z1.h, z2.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     bfmlalb	z0.s, z1.h, z2.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     bfmlalb	z10.s, z21.h, z14.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     bfmlalb	z21.s, z14.h, z3.h[2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     bfmlalt	z0.s, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     bfmlalt	z0.s, z1.h, z2.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     bfmlalt	z0.s, z1.h, z2.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     bfmlalt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     bfmlalt	z14.s, z10.h, z21.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50   0.50   0.50    -     bfmmla	z0.s, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     13.00  bgrp	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     69.00  bgrp	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     21.00  bgrp	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     37.00  bgrp	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     bic	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     bic	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     bic	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     bic	z23.d, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     bic	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     bic	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     bic	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     bic	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     bics	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     bics	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     brka	p0.b, p15/m, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     brka	p0.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     brkas	p0.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     brkb	p0.b, p15/m, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     brkb	p0.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     brkbs	p0.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     brkn	p0.b, p15/z, p1.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     brkn	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     brkns	p0.b, p15/z, p1.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     brkns	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     brkpa	p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     brkpa	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     brkpas	p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     brkpas	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     brkpb	p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     brkpb	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     brkpbs	p0.b, p15/z, p1.b, p2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     brkpbs	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     bsl	z0.d, z0.d, z1.d, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     bsl1n	z0.d, z0.d, z1.d, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     bsl2n	z0.d, z0.d, z1.d, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cadd	z0.b, z0.b, z0.b, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cadd	z0.d, z0.d, z0.d, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cadd	z0.h, z0.h, z0.h, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cadd	z0.s, z0.s, z0.s, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cadd	z31.b, z31.b, z31.b, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cadd	z31.d, z31.d, z31.d, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cadd	z31.h, z31.h, z31.h, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cadd	z31.s, z31.s, z31.s, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cdot	z0.d, z1.h, z15.h[1], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cdot	z0.d, z1.h, z31.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cdot	z0.d, z1.h, z31.h, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cdot	z0.d, z1.h, z31.h, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cdot	z0.d, z1.h, z31.h, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cdot	z0.s, z1.b, z31.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cdot	z0.s, z1.b, z7.b[3], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cdot	z29.d, z30.h, z0.h[0], #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cdot	z31.d, z30.h, z7.h[1], #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cdot	z5.d, z6.h, z3.h[0], #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     clasta	b0, p7, b0, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     clasta	d0, p7, d0, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     clasta	h0, p7, h0, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     clasta	s0, p7, s0, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     clasta	w0, p7, w0, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     clasta	w0, p7, w0, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     clasta	w0, p7, w0, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     clasta	x0, p7, x0, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     clasta	z0.b, p7, z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     clasta	z0.d, p7, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     clasta	z0.h, p7, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     clasta	z0.s, p7, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     clastb	b0, p7, b0, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     clastb	d0, p7, d0, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     clastb	h0, p7, h0, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     clastb	s0, p7, s0, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     clastb	w0, p7, w0, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     clastb	w0, p7, w0, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     clastb	w0, p7, w0, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     clastb	x0, p7, x0, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     clastb	z0.b, p7, z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     clastb	z0.d, p7, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     clastb	z0.h, p7, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     clastb	z0.s, p7, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cls	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cls	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cls	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cls	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     clz	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     clz	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     clz	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     clz	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cmla	z0.b, z1.b, z2.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cmla	z0.d, z1.d, z2.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cmla	z0.h, z1.h, z2.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cmla	z0.h, z1.h, z2.h[0], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cmla	z0.s, z1.s, z2.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cmla	z0.s, z1.s, z2.s[0], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cmla	z15.b, z16.b, z17.b, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cmla	z15.d, z16.d, z17.d, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cmla	z15.h, z16.h, z17.h, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cmla	z15.s, z16.s, z17.s, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cmla	z29.b, z30.b, z31.b, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cmla	z29.d, z30.d, z31.d, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cmla	z29.h, z30.h, z31.h, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cmla	z29.s, z30.s, z31.s, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cmla	z31.b, z31.b, z31.b, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cmla	z31.d, z31.d, z31.d, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cmla	z31.h, z30.h, z7.h[0], #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cmla	z31.h, z31.h, z31.h, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cmla	z31.s, z30.s, z7.s[0], #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     cmla	z31.s, z31.s, z31.s, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpeq	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpeq	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpeq	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpeq	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpeq	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpeq	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpeq	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpeq	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpeq	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpeq	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpeq	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpeq	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpeq	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpeq	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpeq	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpge	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpge	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpge	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpge	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpge	p0.b, p0/z, z1.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpge	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpge	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpge	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpge	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpge	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpge	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpge	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpge	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpge	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpge	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpge	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpge	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpge	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpge	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpgt	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpgt	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpgt	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpgt	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpgt	p0.b, p0/z, z1.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpgt	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpgt	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpgt	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpgt	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpgt	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpgt	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpgt	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpgt	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpgt	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpgt	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpgt	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpgt	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpgt	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpgt	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphi	p0.b, p0/z, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphi	p0.b, p0/z, z0.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphi	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphi	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphi	p0.b, p0/z, z1.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphi	p0.d, p0/z, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphi	p0.d, p0/z, z0.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphi	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphi	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphi	p0.h, p0/z, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphi	p0.h, p0/z, z0.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphi	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphi	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphi	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphi	p0.s, p0/z, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphi	p0.s, p0/z, z0.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphi	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphi	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphi	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphs	p0.b, p0/z, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphs	p0.b, p0/z, z0.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphs	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphs	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphs	p0.b, p0/z, z1.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphs	p0.d, p0/z, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphs	p0.d, p0/z, z0.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphs	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphs	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphs	p0.h, p0/z, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphs	p0.h, p0/z, z0.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphs	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphs	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphs	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphs	p0.s, p0/z, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphs	p0.s, p0/z, z0.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphs	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphs	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmphs	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmple	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmple	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmple	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmple	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmple	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmple	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmple	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmple	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmple	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmple	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmple	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmplo	p0.b, p0/z, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmplo	p0.b, p0/z, z0.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmplo	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmplo	p0.d, p0/z, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmplo	p0.d, p0/z, z0.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmplo	p0.h, p0/z, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmplo	p0.h, p0/z, z0.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmplo	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmplo	p0.s, p0/z, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmplo	p0.s, p0/z, z0.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmplo	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpls	p0.b, p0/z, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpls	p0.b, p0/z, z0.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpls	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpls	p0.d, p0/z, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpls	p0.d, p0/z, z0.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpls	p0.h, p0/z, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpls	p0.h, p0/z, z0.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpls	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpls	p0.s, p0/z, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpls	p0.s, p0/z, z0.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpls	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmplt	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmplt	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmplt	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmplt	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmplt	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmplt	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmplt	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmplt	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmplt	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmplt	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmplt	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpne	p0.b, p0/z, z0.b, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpne	p0.b, p0/z, z0.b, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpne	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpne	p0.b, p0/z, z0.b, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpne	p0.d, p0/z, z0.d, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpne	p0.d, p0/z, z0.d, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpne	p0.d, p0/z, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpne	p0.h, p0/z, z0.h, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpne	p0.h, p0/z, z0.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpne	p0.h, p0/z, z0.h, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpne	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpne	p0.s, p0/z, z0.s, #-16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpne	p0.s, p0/z, z0.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpne	p0.s, p0/z, z0.s, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cmpne	p0.s, p0/z, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cnot	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cnot	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cnot	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cnot	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cnt	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cnt	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cnt	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     cnt	z31.s, p7/m, z31.s
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cntb	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cntb	x0, #28
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cntb	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cntb	x0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cntd	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cntd	x0, #28
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cntd	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cntd	x0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cnth	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cnth	x0, #28
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cnth	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cnth	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     cntp	x0, p15, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     cntp	x0, p15, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     cntp	x0, p15, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     cntp	x0, p15, p0.s
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cntw	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cntw	x0, #28
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cntw	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     cntw	x0, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     compact	z31.d, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     compact	z31.s, p7, z31.s
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ctermeq	w30, wzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ctermeq	wzr, w30
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ctermeq	x30, xzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ctermeq	xzr, x30
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ctermne	w30, wzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ctermne	wzr, w30
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ctermne	x30, xzr
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     ctermne	xzr, x30
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     decb	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     decb	x0, #14
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     decb	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     decb	x0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     decb	x0, vl1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     decd	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     decd	x0, #14
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     decd	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     decd	x0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     decd	x0, vl1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     dech	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     dech	x0, #14
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     dech	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     dech	x0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     dech	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     decp	x0, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     decp	x0, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     decp	x0, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     decp	x0, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     decp	xzr, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     decp	xzr, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     decp	xzr, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     decp	xzr, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     decp	z31.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     decp	z31.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     decp	z31.s, p15.s
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     decw	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     decw	x0, #14
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     decw	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     decw	x0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     decw	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     dupm	z0.d, #0xfffffffffffffff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     dupm	z0.s, #0xfffffff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     dupm	z23.h, #0xfff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     dupm	z5.b, #0xf9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     eor	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     eor	z0.d, z0.d, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     eor	z0.d, z0.d, #0xfffffffffffffff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     eor	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     eor	z0.s, z0.s, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     eor	z0.s, z0.s, #0xfffffff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     eor	z23.d, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     eor	z23.h, z23.h, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     eor	z23.h, z23.h, #0xfff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     eor	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     eor	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     eor	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     eor	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     eor	z5.b, z5.b, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     eor	z5.b, z5.b, #0xf9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     eor3	z29.d, z29.d, z30.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     eorbt	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     eorbt	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     eorbt	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     eorbt	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     eors	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     eortb	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     eortb	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     eortb	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     eortb	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     eorv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     eorv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     eorv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     eorv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ext	z0.b, { z1.b, z2.b }, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ext	z31.b, z31.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ext	z31.b, z31.b, z0.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ext	z31.b, { z30.b, z31.b }, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fabd	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fabd	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fabd	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fabs	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fabs	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fabs	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facge	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facge	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facge	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facge	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facge	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facge	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facgt	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facgt	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facgt	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facgt	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facgt	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     facgt	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fadd	z0.d, p0/m, z0.d, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fadd	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fadd	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fadd	z0.h, p0/m, z0.h, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fadd	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fadd	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fadd	z0.s, p0/m, z0.s, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fadd	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fadd	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fadd	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fadd	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fadd	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.50   2.50    -      -      -     fadda	d0, p7, d0, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     14.50  14.50   -      -      -     fadda	h0, p7, h0, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     6.50   6.50    -      -      -     fadda	s0, p7, s0, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     faddp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     faddp	z29.s, p3/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     faddp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     faddv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     11.00   -      -      -      -     faddv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     5.00    -      -      -      -     faddv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcadd	z0.d, p0/m, z0.d, z0.d, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcadd	z0.h, p0/m, z0.h, z0.h, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcadd	z0.s, p0/m, z0.s, z0.s, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcadd	z31.d, p7/m, z31.d, z31.d, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcadd	z31.h, p7/m, z31.h, z31.h, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcadd	z31.s, p7/m, z31.s, z31.s, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmeq	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmeq	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmeq	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmeq	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmeq	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmeq	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmge	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmge	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmge	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmge	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmge	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmge	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmge	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmge	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmge	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmgt	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmgt	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmgt	p0.d, p0/z, z1.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmgt	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmgt	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmgt	p0.h, p0/z, z1.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmgt	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmgt	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmgt	p0.s, p0/z, z1.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fcmla	z0.d, p0/m, z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fcmla	z0.d, p0/m, z1.d, z2.d, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fcmla	z0.h, p0/m, z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fcmla	z0.h, p0/m, z1.h, z2.h, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fcmla	z0.h, z0.h, z0.h[0], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fcmla	z0.s, p0/m, z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fcmla	z0.s, p0/m, z1.s, z2.s, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fcmla	z21.s, z10.s, z5.s[1], #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fcmla	z23.s, z13.s, z8.s[0], #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fcmla	z29.d, p7/m, z30.d, z31.d, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fcmla	z29.h, p7/m, z30.h, z31.h, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fcmla	z29.s, p7/m, z30.s, z31.s, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fcmla	z31.d, p7/m, z31.d, z31.d, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fcmla	z31.h, p7/m, z31.h, z31.h, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fcmla	z31.h, z31.h, z7.h[3], #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fcmla	z31.s, p7/m, z31.s, z31.s, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmle	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmle	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmle	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmlt	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmlt	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmlt	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmne	p0.d, p0/z, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmne	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmne	p0.h, p0/z, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmne	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmne	p0.s, p0/z, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmne	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmuo	p0.d, p0/z, z0.d, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmuo	p0.h, p0/z, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcmuo	p0.s, p0/z, z0.s, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvt	z0.d, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvt	z0.d, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvt	z0.h, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvt	z0.h, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvt	z0.s, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvt	z0.s, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtlt	z0.s, p0/m, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtlt	z30.d, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtnt	z0.h, p0/m, z1.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtnt	z30.s, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtx	z0.s, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtx	z30.s, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtxnt	z0.s, p0/m, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtxnt	z30.s, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	z0.d, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	z0.d, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	z0.s, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	z0.s, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzs	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	z0.d, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	z0.d, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	z0.s, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	z0.s, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fcvtzu	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     19.00  fdiv	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     5.00   fdiv	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     10.00  fdiv	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     19.00  fdivr	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     5.00   fdivr	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     10.00  fdivr	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fexpa	z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fexpa	z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fexpa	z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     flogb	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     flogb	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     flogb	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmad	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmad	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmad	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmax	z0.d, p0/m, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmax	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmax	z0.h, p0/m, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmax	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmax	z0.s, p0/m, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmax	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmax	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmax	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmax	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmaxnm	z0.d, p0/m, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmaxnm	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmaxnm	z0.h, p0/m, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmaxnm	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmaxnm	z0.s, p0/m, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmaxnm	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmaxnm	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmaxnm	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmaxnm	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmaxnmp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmaxnmp	z29.s, p3/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmaxnmp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     fmaxnmv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     fmaxnmv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     fmaxnmv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmaxp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmaxp	z29.s, p3/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmaxp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     fmaxv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     fmaxv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     fmaxv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmin	z0.d, p0/m, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmin	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmin	z0.h, p0/m, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmin	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmin	z0.s, p0/m, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmin	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmin	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmin	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fmin	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fminnm	z0.d, p0/m, z0.d, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fminnm	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fminnm	z0.h, p0/m, z0.h, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fminnm	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fminnm	z0.s, p0/m, z0.s, #0.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fminnm	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fminnm	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fminnm	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fminnm	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fminnmp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fminnmp	z29.s, p3/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fminnmp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     fminnmv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     fminnmv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     fminnmv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fminp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fminp	z29.s, p3/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fminp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     fminv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     fminv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     fminv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmla	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmla	z0.d, z1.d, z7.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmla	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmla	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmla	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmla	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmlalb	z0.s, z1.h, z7.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmlalb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmlalb	z30.s, z31.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmlalt	z0.s, z1.h, z7.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmlalt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmlalt	z30.s, z31.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmls	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmls	z0.d, z1.d, z7.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmls	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmls	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmls	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmls	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmlslb	z0.s, z1.h, z7.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmlslb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmlslb	z30.s, z31.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmlslt	z0.s, z1.h, z7.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmlslt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmlslt	z30.s, z31.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     fmov	z0.d, #-10.00000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     fmov	z0.d, #0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     fmov	z0.d, p0/m, #-10.00000000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     fmov	z0.d, p0/m, #0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     fmov	z0.h, #-0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     fmov	z0.h, p0/m, #-0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     fmov	z0.s, #-0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     fmov	z0.s, p0/m, #-0.12500000
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmsb	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmsb	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmsb	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmul	z0.d, p0/m, z0.d, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmul	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmul	z0.d, z0.d, z0.d[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmul	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmul	z0.h, p0/m, z0.h, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmul	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmul	z0.h, z0.h, z0.h[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmul	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmul	z0.s, p0/m, z0.s, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmul	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmul	z0.s, z0.s, z0.s[0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmul	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmul	z31.d, p7/m, z31.d, #2.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmul	z31.d, z31.d, z15.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmul	z31.h, p7/m, z31.h, #2.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmul	z31.h, z31.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmul	z31.s, p7/m, z31.s, #2.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmul	z31.s, z31.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmulx	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmulx	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fmulx	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fneg	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fneg	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fneg	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fnmad	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fnmad	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fnmad	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fnmla	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fnmla	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fnmla	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fnmls	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fnmls	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fnmls	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fnmsb	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fnmsb	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fnmsb	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     frecpe	z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     frecpe	z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     frecpe	z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     frecps	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     frecps	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     frecps	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     frecpx	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     frecpx	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     frecpx	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frinta	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frinta	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frinta	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frinti	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frinti	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frinti	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintm	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintm	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintm	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintn	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintn	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintn	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintp	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintp	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintp	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintx	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintx	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintx	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintz	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintz	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     frintz	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     frsqrte	z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     frsqrte	z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     frsqrte	z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     frsqrts	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     frsqrts	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     frsqrts	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fscale	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fscale	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     fscale	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     19.00  fsqrt	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     5.00   fsqrt	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     9.00   fsqrt	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fsub	z0.d, p0/m, z0.d, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fsub	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fsub	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fsub	z0.h, p0/m, z0.h, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fsub	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fsub	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fsub	z0.s, p0/m, z0.s, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fsub	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fsub	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fsub	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fsub	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fsub	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fsubr	z0.d, p0/m, z0.d, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fsubr	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fsubr	z0.h, p0/m, z0.h, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fsubr	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fsubr	z0.s, p0/m, z0.s, #0.5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fsubr	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fsubr	z31.d, p7/m, z31.d, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fsubr	z31.h, p7/m, z31.h, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     fsubr	z31.s, p7/m, z31.s, #1.0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     ftmad	z0.d, z0.d, z31.d, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     ftmad	z0.h, z0.h, z31.h, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     ftmad	z0.s, z0.s, z31.s, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     ftsmul	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     ftsmul	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     ftsmul	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ftssel	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ftssel	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ftssel	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -      -     histcnt	z0.s, p0/z, z1.s, z2.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -      -     histcnt	z29.d, p7/z, z30.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -      -     histseg	z0.b, z1.b, z31.b
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     incb	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     incb	x0, #14
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     incb	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     incb	x0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     incb	x0, vl1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     incd	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     incd	x0, #14
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     incd	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     incd	x0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     incd	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     incd	z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     incd	z0.d, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     inch	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     inch	x0, #14
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     inch	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     inch	x0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     inch	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     inch	z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     inch	z0.h, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     incp	x0, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     incp	x0, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     incp	x0, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     incp	x0, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     incp	xzr, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     incp	xzr, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     incp	xzr, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     incp	xzr, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     incp	z31.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     incp	z31.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     incp	z31.s, p15.s
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     incw	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     incw	x0, #14
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     incw	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     incw	x0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     incw	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     incw	z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     incw	z0.s, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z0.b, #0, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z0.d, #0, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z0.h, #0, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z0.h, w0, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z0.s, #0, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z21.b, w10, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z21.d, x10, x21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z21.s, w10, w21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z23.b, #13, w8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z23.b, w13, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z23.d, #13, x8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z23.d, x13, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z23.h, #13, w8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z23.h, w13, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z23.s, #13, w8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z23.s, w13, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z31.b, #-1, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z31.b, #-1, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z31.b, wzr, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z31.b, wzr, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z31.d, #-1, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z31.d, #-1, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z31.d, xzr, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z31.d, xzr, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z31.h, #-1, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z31.h, #-1, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z31.h, wzr, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z31.h, wzr, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z31.s, #-1, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z31.s, #-1, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z31.s, wzr, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     index	z31.s, wzr, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -      -     insr	z0.b, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -      -     insr	z0.d, x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -      -     insr	z0.h, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -      -     insr	z0.s, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     insr	z31.b, b31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -      -     insr	z31.b, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     insr	z31.d, d31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -      -     insr	z31.d, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     insr	z31.h, h31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -      -     insr	z31.h, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     insr	z31.s, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -      -     insr	z31.s, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lasta	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lasta	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lasta	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lasta	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -      -     lasta	w0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -      -     lasta	w0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -      -     lasta	w0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -      -     lasta	x0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lastb	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lastb	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lastb	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lastb	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -      -     lastb	w0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -      -     lastb	w0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -      -     lastb	w0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     2.00    -      -      -      -     lastb	x0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1b	{ z0.b }, p0/z, [sp, x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1b	{ z0.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1b	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1b	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1b	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1b	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ld1b	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ld1b	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1b	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ld1b	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1b	{ z21.b }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1b	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1b	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1b	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1b	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1b	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1b	{ z21.s }, p5/z, [x10, x21]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1b	{ z23.d }, p3/z, [x13, x8]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1b	{ z31.b }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1b	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1b	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1b	{ z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1b	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1b	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ld1b	{ z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1b	{ z5.h }, p3/z, [x17, x16]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1d	{ z0.d }, p0/z, [x0, z0.d, sxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1d	{ z0.d }, p0/z, [x0, z0.d, uxtw #3]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1d	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1d	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1d	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1d	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1d	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1d	{ z23.d }, p3/z, [sp, x8, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1d	{ z23.d }, p3/z, [x13, x8, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1d	{ z23.d }, p3/z, [x13, z8.d, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1d	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1d	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1d	{ z31.d }, p7/z, [z31.d, #248]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1h	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1h	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1h	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1h	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1h	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ld1h	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ld1h	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1h	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ld1h	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1h	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1h	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1h	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1h	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1h	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1h	{ z21.s }, p5/z, [x10, x21, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1h	{ z23.d }, p3/z, [x13, x8, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1h	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1h	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1h	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1h	{ z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1h	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1h	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ld1h	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ld1h	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ld1h	{ z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1h	{ z5.h }, p3/z, [sp, x16, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1h	{ z5.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rb	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rb	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rb	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rb	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rb	{ z31.b }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rb	{ z31.d }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rb	{ z31.h }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rb	{ z31.s }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rd	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rd	{ z31.d }, p7/z, [sp, #504]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rh	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rh	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rh	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rh	{ z31.d }, p7/z, [sp, #126]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rh	{ z31.h }, p7/z, [sp, #126]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rh	{ z31.s }, p7/z, [sp, #126]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ld1rqb	{ z0.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rqb	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rqb	{ z21.b }, p5/z, [x10, #112]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rqb	{ z23.b }, p3/z, [x13, #-128]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rqb	{ z31.b }, p7/z, [sp, #-16]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ld1rqd	{ z0.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rqd	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rqd	{ z23.d }, p3/z, [x13, #-128]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rqd	{ z23.d }, p3/z, [x13, #112]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rqd	{ z31.d }, p7/z, [sp, #-16]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ld1rqh	{ z0.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rqh	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rqh	{ z23.h }, p3/z, [x13, #-128]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rqh	{ z23.h }, p3/z, [x13, #112]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rqh	{ z31.h }, p7/z, [sp, #-16]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ld1rqw	{ z0.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rqw	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rqw	{ z23.s }, p3/z, [x13, #-128]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rqw	{ z23.s }, p3/z, [x13, #112]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rqw	{ z31.s }, p7/z, [sp, #-16]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rsb	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rsb	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rsb	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rsb	{ z31.d }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rsb	{ z31.h }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rsb	{ z31.s }, p7/z, [sp, #63]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rsh	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rsh	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rsh	{ z31.d }, p7/z, [sp, #126]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rsh	{ z31.s }, p7/z, [sp, #126]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rsw	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rsw	{ z31.d }, p7/z, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rw	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rw	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rw	{ z31.d }, p7/z, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1rw	{ z31.s }, p7/z, [sp, #252]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sb	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1sb	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sb	{ z0.h }, p0/z, [sp, x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sb	{ z0.h }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sb	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ld1sb	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sb	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ld1sb	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sb	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1sb	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1sb	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sb	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sb	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sb	{ z21.s }, p5/z, [x10, x21]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sb	{ z23.d }, p3/z, [x13, x8]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sb	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1sb	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1sb	{ z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sb	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sb	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ld1sb	{ z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1sh	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1sh	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sh	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1sh	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ld1sh	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ld1sh	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sh	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ld1sh	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sh	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1sh	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1sh	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sh	{ z21.s }, p5/z, [sp, x21, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sh	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sh	{ z21.s }, p5/z, [x10, x21, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sh	{ z23.d }, p3/z, [x13, x8, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1sh	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sh	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1sh	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1sh	{ z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sh	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ld1sh	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ld1sh	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ld1sh	{ z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1sw	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1sw	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sw	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1sw	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sw	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1sw	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1sw	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sw	{ z23.d }, p3/z, [sp, x8, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sw	{ z23.d }, p3/z, [x13, x8, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1sw	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1sw	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1sw	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1sw	{ z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1w	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1w	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1w	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1w	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ld1w	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ld1w	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1w	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ld1w	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1w	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1w	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1w	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1w	{ z21.s }, p5/z, [sp, x21, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1w	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1w	{ z21.s }, p5/z, [x10, x21, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1w	{ z23.d }, p3/z, [x13, x8, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1w	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1w	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1w	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ld1w	{ z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ld1w	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ld1w	{ z31.s }, p7/z, [sp, z31.s, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ld1w	{ z31.s }, p7/z, [sp, z31.s, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ld1w	{ z31.s }, p7/z, [z31.s, #124]
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ld2b	{ z0.b, z1.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ld2b	{ z0.b, z1.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ld2b	{ z21.b, z22.b }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ld2b	{ z23.b, z24.b }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ld2b	{ z5.b, z6.b }, p3/z, [x17, x16]
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ld2d	{ z0.d, z1.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ld2d	{ z0.d, z1.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ld2d	{ z21.d, z22.d }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ld2d	{ z23.d, z24.d }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ld2d	{ z5.d, z6.d }, p3/z, [x17, x16, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ld2h	{ z0.h, z1.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ld2h	{ z0.h, z1.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ld2h	{ z21.h, z22.h }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ld2h	{ z23.h, z24.h }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ld2h	{ z5.h, z6.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ld2w	{ z0.s, z1.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ld2w	{ z0.s, z1.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ld2w	{ z21.s, z22.s }, p5/z, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ld2w	{ z23.s, z24.s }, p3/z, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     2.00    -      -      -      -      -      -      -     ld2w	{ z5.s, z6.s }, p3/z, [x17, x16, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld3b	{ z0.b - z2.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld3b	{ z0.b - z2.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld3b	{ z21.b - z23.b }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld3b	{ z23.b - z25.b }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld3b	{ z5.b - z7.b }, p3/z, [x17, x16]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld3d	{ z0.d - z2.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld3d	{ z0.d - z2.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld3d	{ z21.d - z23.d }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld3d	{ z23.d - z25.d }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld3d	{ z5.d - z7.d }, p3/z, [x17, x16, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld3h	{ z0.h - z2.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld3h	{ z0.h - z2.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld3h	{ z21.h - z23.h }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld3h	{ z23.h - z25.h }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld3h	{ z5.h - z7.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld3w	{ z0.s - z2.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld3w	{ z0.s - z2.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld3w	{ z21.s - z23.s }, p5/z, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld3w	{ z23.s - z25.s }, p3/z, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld3w	{ z5.s - z7.s }, p3/z, [x17, x16, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld4b	{ z0.b - z3.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld4b	{ z0.b - z3.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld4b	{ z21.b - z24.b }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld4b	{ z23.b - z26.b }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld4b	{ z5.b - z8.b }, p3/z, [x17, x16]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld4d	{ z0.d - z3.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld4d	{ z0.d - z3.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld4d	{ z21.d - z24.d }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld4d	{ z23.d - z26.d }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld4d	{ z5.d - z8.d }, p3/z, [x17, x16, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld4h	{ z0.h - z3.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld4h	{ z0.h - z3.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld4h	{ z21.h - z24.h }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld4h	{ z23.h - z26.h }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld4h	{ z5.h - z8.h }, p3/z, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld4w	{ z0.s - z3.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld4w	{ z0.s - z3.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld4w	{ z21.s - z24.s }, p5/z, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld4w	{ z23.s - z26.s }, p3/z, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     3.00    -      -      -      -      -      -      -     ld4w	{ z5.s - z8.s }, p3/z, [x17, x16, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1b	{ z0.d }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1b	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1b	{ z0.h }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1b	{ z0.s }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ldff1b	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ldff1b	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ldff1b	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1b	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1b	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1b	{ z31.b }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1b	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1b	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1b	{ z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1b	{ z31.h }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1b	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ldff1b	{ z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1d	{ z0.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1d	{ z0.d }, p0/z, [x0, z0.d, sxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1d	{ z0.d }, p0/z, [x0, z0.d, uxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1d	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1d	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1d	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1d	{ z23.d }, p3/z, [x13, z8.d, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1d	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1d	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1d	{ z31.d }, p7/z, [z31.d, #248]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1h	{ z0.d }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1h	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1h	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1h	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1h	{ z0.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1h	{ z0.s }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ldff1h	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ldff1h	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ldff1h	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1h	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1h	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1h	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1h	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1h	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1h	{ z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1h	{ z31.h }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ldff1h	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ldff1h	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1h	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ldff1h	{ z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1sb	{ z0.d }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1sb	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1sb	{ z0.h }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1sb	{ z0.s }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ldff1sb	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ldff1sb	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ldff1sb	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1sb	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1sb	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1sb	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1sb	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1sb	{ z31.d }, p7/z, [z31.d, #31]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1sb	{ z31.h }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1sb	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ldff1sb	{ z31.s }, p7/z, [z31.s, #31]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1sh	{ z0.d }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1sh	{ z0.d }, p0/z, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1sh	{ z0.d }, p0/z, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1sh	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1sh	{ z0.s }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ldff1sh	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ldff1sh	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ldff1sh	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1sh	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1sh	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1sh	{ z23.d }, p3/z, [x13, z8.d, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1sh	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1sh	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1sh	{ z31.d }, p7/z, [z31.d, #62]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ldff1sh	{ z31.s }, p7/z, [sp, z31.s, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ldff1sh	{ z31.s }, p7/z, [sp, z31.s, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1sh	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ldff1sh	{ z31.s }, p7/z, [z31.s, #62]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1sw	{ z0.d }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1sw	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1sw	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1sw	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1sw	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1sw	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1sw	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1sw	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1sw	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1sw	{ z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1w	{ z0.d }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1w	{ z0.d }, p0/z, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1w	{ z0.d }, p0/z, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1w	{ z0.d }, p0/z, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1w	{ z0.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ldff1w	{ z0.s }, p0/z, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ldff1w	{ z0.s }, p0/z, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ldff1w	{ z0.s }, p0/z, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1w	{ z21.d }, p5/z, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1w	{ z21.d }, p5/z, [x10, z21.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1w	{ z23.d }, p3/z, [x13, z8.d, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1w	{ z31.d }, p7/z, [sp, z31.d]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1w	{ z31.d }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldff1w	{ z31.d }, p7/z, [z31.d, #124]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ldff1w	{ z31.s }, p7/z, [sp, z31.s, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -     4.50   4.50    -      -      -      -      -      -      -     ldff1w	{ z31.s }, p7/z, [sp, z31.s, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldff1w	{ z31.s }, p7/z, [sp]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ldff1w	{ z31.s }, p7/z, [z31.s, #124]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1b	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1b	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1b	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1b	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1b	{ z21.b }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1b	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1b	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1b	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1b	{ z31.b }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1b	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1b	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1b	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1d	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1d	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1d	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1h	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1h	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1h	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1h	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1h	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1h	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1h	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1h	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1h	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1sb	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1sb	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1sb	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1sb	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1sb	{ z21.h }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1sb	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1sb	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1sb	{ z31.h }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1sb	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1sh	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1sh	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1sh	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1sh	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1sh	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1sh	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1sw	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1sw	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1sw	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1w	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1w	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1w	{ z21.d }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1w	{ z21.s }, p5/z, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1w	{ z31.d }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldnf1w	{ z31.s }, p7/z, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ldnt1b	{ z0.b }, p0/z, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ldnt1b	{ z0.b }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldnt1b	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ldnt1b	{ z0.s }, p0/z, [z1.s]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ldnt1b	{ z21.b }, p5/z, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ldnt1b	{ z23.b }, p3/z, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldnt1b	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldnt1b	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ldnt1b	{ z31.s }, p7/z, [z31.s, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ldnt1b	{ z31.s }, p7/z, [z31.s]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ldnt1d	{ z0.d }, p0/z, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ldnt1d	{ z0.d }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldnt1d	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ldnt1d	{ z21.d }, p5/z, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ldnt1d	{ z23.d }, p3/z, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldnt1d	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldnt1d	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldnt1h	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ldnt1h	{ z0.h }, p0/z, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ldnt1h	{ z0.h }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ldnt1h	{ z0.s }, p0/z, [z1.s]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ldnt1h	{ z21.h }, p5/z, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ldnt1h	{ z23.h }, p3/z, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldnt1h	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldnt1h	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ldnt1h	{ z31.s }, p7/z, [z31.s, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ldnt1h	{ z31.s }, p7/z, [z31.s]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldnt1sb	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ldnt1sb	{ z0.s }, p0/z, [z1.s]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldnt1sb	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldnt1sb	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ldnt1sb	{ z31.s }, p7/z, [z31.s, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ldnt1sb	{ z31.s }, p7/z, [z31.s]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldnt1sh	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ldnt1sh	{ z0.s }, p0/z, [z1.s]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldnt1sh	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldnt1sh	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ldnt1sh	{ z31.s }, p7/z, [z31.s, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ldnt1sh	{ z31.s }, p7/z, [z31.s]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldnt1sw	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldnt1sw	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldnt1sw	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldnt1w	{ z0.d }, p0/z, [z1.d]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ldnt1w	{ z0.s }, p0/z, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ldnt1w	{ z0.s }, p0/z, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ldnt1w	{ z0.s }, p0/z, [z1.s]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ldnt1w	{ z21.s }, p5/z, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ldnt1w	{ z23.s }, p3/z, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldnt1w	{ z31.d }, p7/z, [z31.d, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     ldnt1w	{ z31.d }, p7/z, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ldnt1w	{ z31.s }, p7/z, [z31.s, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     ldnt1w	{ z31.s }, p7/z, [z31.s]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ldr	p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ldr	p5, [x10, #255, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     ldr	p7, [x13, #-256, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	z0, [x0]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	z23, [x13, #255, mul vl]
+# CHECK-NEXT:  -      -      -      -      -     0.50   0.50    -      -      -      -      -      -      -     ldr	z31, [sp, #-256, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsl	z0.b, p0/m, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsl	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsl	z0.b, p0/m, z0.b, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsl	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsl	z0.b, z1.b, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsl	z0.d, p0/m, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsl	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsl	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsl	z0.h, p0/m, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsl	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsl	z0.h, p0/m, z0.h, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsl	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsl	z0.h, z1.h, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsl	z0.s, p0/m, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsl	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsl	z0.s, p0/m, z0.s, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsl	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsl	z0.s, z1.s, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsl	z31.b, p0/m, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsl	z31.b, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsl	z31.d, p0/m, z31.d, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsl	z31.d, z31.d, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsl	z31.h, p0/m, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsl	z31.h, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsl	z31.s, p0/m, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsl	z31.s, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lslr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lslr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lslr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lslr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsr	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsr	z0.b, p0/m, z0.b, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsr	z0.b, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsr	z0.b, z1.b, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsr	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsr	z0.d, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsr	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsr	z0.h, p0/m, z0.h, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsr	z0.h, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsr	z0.h, z1.h, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsr	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsr	z0.s, p0/m, z0.s, z1.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsr	z0.s, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsr	z0.s, z1.s, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsr	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsr	z31.b, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsr	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsr	z31.d, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsr	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsr	z31.h, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsr	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsr	z31.s, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsrr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsrr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsrr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     lsrr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mad	z0.b, p7/m, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mad	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mad	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mad	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     match	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     match	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     match	p15.b, p7/z, z30.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     match	p15.h, p7/z, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mla	z0.b, p7/m, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mla	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mla	z0.d, z1.d, z7.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mla	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mla	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mla	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mla	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mls	z0.b, p7/m, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mls	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mls	z0.d, z1.d, z7.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mls	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mls	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mls	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mls	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     mov	p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     mov	p0.b, p0/m, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     mov	p0.b, p0/z, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     mov	p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     mov	p15.b, p15/m, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     mov	p15.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.b, b0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.b, p0/m, b0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     mov	z0.b, p0/m, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.b, p0/z, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.b, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.d, #0xe0000000000003ff
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.d, #0xffffffffffff7fff
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.d, #32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.d, d0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.d, p0/m, d0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     mov	z0.d, p0/m, x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.d, x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.h, #-256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.h, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.h, #32512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.h, #32767
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.h, h0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.h, p0/m, h0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     mov	z0.h, p0/m, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.h, p0/z, #32512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.h, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.q, q0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.s, #0xffff7fff
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.s, #32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.s, p0/m, s0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     mov	z0.s, p0/m, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.s, s0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z0.s, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.d, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.d, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.d, #32512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.d, p0/z, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.d, p0/z, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.d, p0/z, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.d, p0/z, #32512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.d, p15/m, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.d, p15/m, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.h, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.h, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.h, #32512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.h, p0/z, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.h, p0/z, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.h, p0/z, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.h, p0/z, #32512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.h, p15/m, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.h, p15/m, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.s, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.s, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.s, #32512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.s, p0/z, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.s, p0/z, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.s, p0/z, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.s, p0/z, #32512
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.s, p15/m, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z21.s, p15/m, #-32768
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z31.b, p15/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z31.b, p7/m, b31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     movprfx	z31, z6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     mov	z31.b, p7/m, wsp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z31.b, wsp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z31.b, z31.b[63]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z31.d, p15/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z31.d, p7/m, d31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     movprfx	z31.d, p7/z, z6.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     mov	z31.d, p7/m, sp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z31.d, sp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z31.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z31.d, z31.d[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z31.h, p15/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z31.h, p7/m, h31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     mov	z31.h, p7/m, wsp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z31.h, wsp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z31.h, z31.h[31]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z31.s, p15/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z31.s, p7/m, s31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     mov	z31.s, p7/m, wsp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z31.s, wsp
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z31.s, z31.s[15]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z5.b, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z5.b, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z5.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z5.b, p0/z, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z5.b, p0/z, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z5.b, p0/z, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z5.b, p15/m, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z5.d, #-6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z5.h, #-6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z5.q, z17.q[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     mov	z5.s, #-6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     movs	p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     movs	p0.b, p0/z, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     movs	p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     movs	p15.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     mrs	x3, ID_AA64ZFR0_EL1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     mrs	x3, ZCR_EL1
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     mrs	x3, ZCR_EL12
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     mrs	x3, ZCR_EL2
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     mrs	x3, ZCR_EL3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     msb	z0.b, p7/m, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     msb	z0.d, p7/m, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     msb	z0.h, p7/m, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     msb	z0.s, p7/m, z1.s, z31.s
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     msr	ZCR_EL1, x3
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     msr	ZCR_EL12, x3
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     msr	ZCR_EL2, x3
+# CHECK-NEXT:  -      -      -     1.00    -      -      -      -      -      -      -      -      -      -     msr	ZCR_EL3, x3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mul	z0.b, p7/m, z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mul	z0.b, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mul	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mul	z0.d, z1.d, z15.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mul	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mul	z0.h, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mul	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mul	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mul	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mul	z29.s, z30.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mul	z31.b, z31.b, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mul	z31.b, z31.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mul	z31.d, z31.d, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mul	z31.d, z31.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mul	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mul	z31.h, z31.h, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mul	z31.h, z31.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mul	z31.s, z31.s, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     mul	z31.s, z31.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     nand	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     nand	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     nands	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     nands	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     nbsl	z0.d, z0.d, z1.d, z2.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     neg	z0.b, p0/m, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     neg	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     neg	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     neg	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     neg	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     neg	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     neg	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     neg	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     nmatch	p0.b, p0/z, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     nmatch	p0.h, p0/z, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     nmatch	p15.b, p7/z, z30.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     nmatch	p15.h, p7/z, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     nor	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     nor	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     nors	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     nors	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     not	p0.b, p0/z, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     not	p15.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     not	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     not	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     not	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     not	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     nots	p0.b, p0/z, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     nots	p15.b, p15/z, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     orn	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     orn	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     orns	p0.b, p0/z, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     orns	p15.b, p15/z, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     orr	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     orr	z0.d, z0.d, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     orr	z0.d, z0.d, #0xfffffffffffffff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     orr	z0.s, z0.s, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     orr	z0.s, z0.s, #0xfffffff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     orr	z23.d, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     orr	z23.h, z23.h, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     orr	z23.h, z23.h, #0xfff9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     orr	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     orr	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     orr	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     orr	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     orr	z5.b, z5.b, #0x6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     orr	z5.b, z5.b, #0xf9
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     orrs	p0.b, p0/z, p0.b, p1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     orv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     orv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     orv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     orv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     pfalse	p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     pfirst	p0.b, p15, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     pfirst	p15.b, p15, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     pmul	z0.b, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     pmul	z29.b, z30.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   pmullb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   pmullb	z29.q, z30.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   pmullb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   pmullt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   pmullt	z29.q, z30.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   pmullt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     pnext	p0.b, p15, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     pnext	p0.d, p15, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     pnext	p0.h, p15, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     pnext	p0.s, p15, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     pnext	p15.b, p15, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	#14, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	#15, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	#6, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	#7, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	#7, p3, [z13.s, #31]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	#7, p3, [z13.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	pldl1keep, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	pldl1keep, p0, [x0, z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	pldl1keep, p0, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	pldl1keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	pldl1strm, p0, [x0, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	pldl1strm, p0, [x0, #31, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	pldl1strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	pldl2keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	pldl2strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	pldl3keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	pldl3strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	pldl3strm, p5, [x10, z21.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	pldl3strm, p5, [x10, z21.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	pldl3strm, p5, [z10.d, #31]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	pldl3strm, p5, [z10.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	pstl1keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	pstl1strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	pstl2keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	pstl2strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	pstl3keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfb	pstl3strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	#14, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	#15, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	#15, p7, [z31.d, #248]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	#15, p7, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	#15, p7, [z31.s, #248]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	#15, p7, [z31.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	#6, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	#7, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	pldl1keep, p0, [x0, z0.d, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	pldl1keep, p0, [x0, z0.d, sxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	pldl1keep, p0, [x0, z0.d, uxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	pldl1keep, p0, [x0, z0.s, sxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	pldl1keep, p0, [x0, z0.s, uxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	pldl1keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	pldl1strm, p0, [x0, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	pldl1strm, p0, [x0, #31, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	pldl1strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	pldl2keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	pldl2strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	pldl3keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	pldl3strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	pstl1keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	pstl1strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	pstl2keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	pstl2strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	pstl3keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfd	pstl3strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	#14, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	#15, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	#15, p7, [z31.d, #62]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	#15, p7, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	#15, p7, [z31.s, #62]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	#15, p7, [z31.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	#6, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	#7, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	pldl1keep, p0, [x0, z0.d, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	pldl1keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	pldl1strm, p0, [x0, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	pldl1strm, p0, [x0, #31, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	pldl1strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	pldl2keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	pldl2strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	pldl3keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	pldl3strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	pldl3strm, p5, [x10, z21.d, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	pldl3strm, p5, [x10, z21.d, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	pldl3strm, p5, [x10, z21.s, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	pldl3strm, p5, [x10, z21.s, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	pstl1keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	pstl1strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	pstl2keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	pstl2strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	pstl3keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfh	pstl3strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	#14, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	#15, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	#15, p7, [z31.d, #124]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	#15, p7, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	#15, p7, [z31.s, #124]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	#15, p7, [z31.s]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	#6, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	#7, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	#7, p3, [x13, z8.d, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	pldl1keep, p0, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	pldl1keep, p0, [x0, z0.s, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	pldl1keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	pldl1strm, p0, [x0, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	pldl1strm, p0, [x0, #31, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	pldl1strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	pldl2keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	pldl2strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	pldl3keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	pldl3strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	pldl3strm, p5, [x10, z21.d, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	pldl3strm, p5, [x10, z21.s, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	pstl1keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	pstl1strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	pstl2keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	pstl2strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	pstl3keep, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     prfw	pstl3strm, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptest	p15, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptest	p15, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p0.b, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p0.d, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p0.h, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p0.s, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, #17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, #18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, #19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, #21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, #22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, #23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, #24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, #25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, #26
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, #27
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, #28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, mul3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, mul4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, vl128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, vl16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, vl2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, vl256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, vl3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, vl32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, vl4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, vl5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, vl6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, vl64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, vl7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrue	p7.s, vl8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p0.b, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p0.d, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p0.h, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p0.s, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, #14
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, #17
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, #18
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, #19
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, #20
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, #21
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, #22
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, #23
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, #24
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, #25
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, #26
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, #27
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, #28
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, mul3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, mul4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, vl128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, vl16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, vl2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, vl256
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, vl3
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, vl32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, vl4
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, vl5
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, vl6
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, vl64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, vl7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     ptrues	p7.s, vl8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     punpkhi	p0.h, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     punpkhi	p15.h, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     punpklo	p0.h, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     punpklo	p15.h, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     raddhnb	z0.b, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     raddhnb	z0.h, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     raddhnb	z0.s, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     raddhnt	z0.b, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     raddhnt	z0.h, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     raddhnt	z0.s, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   rax1	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rbit	z0.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rbit	z0.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rbit	z0.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rbit	z0.s, p7/m, z31.s
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     rdffr	p0.b
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     rdffr	p0.b, p0/z
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     rdffr	p15.b
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     rdffr	p15.b, p15/z
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     rdffrs	p0.b, p0/z
+# CHECK-NEXT: 1.00    -      -      -      -      -      -      -      -      -      -      -      -      -     rdffrs	p15.b, p15/z
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     rdvl	x0, #0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     rdvl	x21, #-32
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     rdvl	x23, #31
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     rdvl	xzr, #-1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rev	z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rev	z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rev	z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rev	z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     revb	z0.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     revb	z0.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     revb	z0.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     revh	z0.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     revh	z0.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     revw	z0.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rshrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rshrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rshrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rshrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rshrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rshrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rshrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rshrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rshrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rshrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rshrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rshrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rsubhnb	z0.b, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rsubhnb	z0.h, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rsubhnb	z0.s, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rsubhnt	z0.b, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rsubhnt	z0.h, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     rsubhnt	z0.s, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     saba	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     saba	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     saba	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     saba	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     sabalb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     sabalb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     sabalb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     sabalt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     sabalt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     sabalt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sabd	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sabd	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sabd	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sabd	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sabdlb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sabdlb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sabdlb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sabdlt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sabdlt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sabdlt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     sadalp	z0.h, p0/m, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     sadalp	z29.s, p0/m, z30.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     sadalp	z30.d, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddlb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddlb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddlb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddlbt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddlbt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddlbt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddlt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddlt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddlt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     saddv	d0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     saddv	d0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     saddv	d0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddwb	z0.h, z1.h, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddwb	z29.s, z30.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddwb	z31.d, z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddwt	z0.h, z1.h, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddwt	z29.s, z30.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     saddwt	z31.d, z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sbclb	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sbclb	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sbclt	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sbclt	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	z0.d, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	z0.h, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	z0.h, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	z0.s, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     scvtf	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     23.00  sdiv	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     12.00  sdiv	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     23.00  sdivr	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     12.00  sdivr	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sdot	z0.d, z1.h, z15.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sdot	z0.d, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sdot	z0.s, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sdot	z0.s, z1.b, z7.b[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sel	z23.b, p11, z13.b, z8.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sel	z23.d, p11, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sel	z23.h, p11, z13.h, z8.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sel	z23.s, p11, z13.s, z8.s
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     setffr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shsub	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shsub	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shsub	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shsub	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shsubr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shsubr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shsubr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     shsubr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sli	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sli	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sli	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sli	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sli	z31.b, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sli	z31.d, z31.d, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sli	z31.h, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sli	z31.s, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   sm4e	z0.s, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     1.00   sm4ekey	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smax	z0.b, z0.b, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smax	z0.d, z0.d, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smax	z0.h, z0.h, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smax	z0.s, z0.s, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smax	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smax	z31.b, z31.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smax	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smax	z31.d, z31.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smax	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smax	z31.h, z31.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smax	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smax	z31.s, z31.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smaxp	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smaxp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smaxp	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smaxp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     smaxv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     smaxv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     smaxv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     smaxv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smin	z0.b, z0.b, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smin	z0.d, z0.d, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smin	z0.h, z0.h, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smin	z0.s, z0.s, #-128
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smin	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smin	z31.b, z31.b, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smin	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smin	z31.d, z31.d, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smin	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smin	z31.h, z31.h, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smin	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     smin	z31.s, z31.s, #127
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sminp	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sminp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sminp	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sminp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     sminv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     sminv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     sminv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     sminv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smlalb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smlalb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smlalb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smlalb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smlalb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smlalt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smlalt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smlalt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smlalt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smlalt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smlslb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smlslb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smlslb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smlslb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smlslb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smlslt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smlslt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smlslt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smlslt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smlslt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smmla	z0.s, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smulh	z0.b, p7/m, z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smulh	z0.b, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smulh	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smulh	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smulh	z0.h, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smulh	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smulh	z29.s, z30.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smulh	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smullb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smullb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smullb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smullb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smullb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smullt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smullt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smullt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smullt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     smullt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     splice	z29.b, p7, { z30.b, z31.b }
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     splice	z29.d, p7, { z30.d, z31.d }
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     splice	z29.h, p7, { z30.h, z31.h }
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     splice	z29.s, p7, { z30.s, z31.s }
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     splice	z31.b, p7, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     splice	z31.d, p7, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     splice	z31.h, p7, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     splice	z31.s, p7, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqabs	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqabs	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqabs	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqabs	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqadd	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqadd	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqadd	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqadd	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqadd	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqadd	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqadd	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqadd	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqadd	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqadd	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqadd	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqadd	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqadd	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqadd	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqadd	z31.s, z31.s, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqcadd	z0.b, z0.b, z0.b, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqcadd	z0.d, z0.d, z0.d, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqcadd	z0.h, z0.h, z0.h, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqcadd	z0.s, z0.s, z0.s, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqcadd	z31.b, z31.b, z31.b, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqcadd	z31.d, z31.d, z31.d, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqcadd	z31.h, z31.h, z31.h, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqcadd	z31.s, z31.s, z31.s, #270
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecb	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecb	x0, #14
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecb	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecb	x0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecb	x0, vl1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecb	x0, w0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecb	x0, w0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecb	x0, w0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecb	x0, w0, pow2, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecd	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecd	x0, #14
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecd	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecd	x0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecd	x0, vl1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecd	x0, w0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecd	x0, w0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecd	x0, w0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecd	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdecd	z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdecd	z0.d, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdecd	z0.d, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdecd	z0.d, pow2, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdech	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdech	x0, #14
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdech	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdech	x0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdech	x0, vl1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdech	x0, w0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdech	x0, w0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdech	x0, w0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdech	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdech	z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdech	z0.h, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdech	z0.h, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdech	z0.h, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     sqdecp	x0, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     sqdecp	x0, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     sqdecp	x0, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     sqdecp	x0, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     sqdecp	xzr, p15.b, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     sqdecp	xzr, p15.d, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     sqdecp	xzr, p15.h, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     sqdecp	xzr, p15.s, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdecp	z0.d, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdecp	z0.h, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdecp	z0.s, p0.s
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecw	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecw	x0, #14
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecw	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecw	x0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecw	x0, vl1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecw	x0, w0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecw	x0, w0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecw	x0, w0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqdecw	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdecw	z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdecw	z0.s, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdecw	z0.s, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqdecw	z0.s, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmlalb	z0.d, z1.s, z15.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmlalb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmlalb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmlalb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmlalb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmlalbt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmlalbt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmlalbt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmlalt	z0.d, z1.s, z15.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmlalt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmlalt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmlalt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmlalt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmlslb	z0.d, z1.s, z15.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmlslb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmlslb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmlslb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmlslb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmlslbt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmlslbt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmlslbt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmlslt	z0.d, z1.s, z15.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmlslt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmlslt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmlslt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmlslt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmulh	z0.b, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmulh	z0.d, z1.d, z15.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmulh	z0.h, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmulh	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmulh	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmulh	z29.s, z30.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmulh	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmullb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmullb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmullb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmullb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmullb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmullt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmullt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmullt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmullt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqdmullt	z31.d, z31.s, z31.s
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincb	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincb	x0, #14
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincb	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincb	x0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincb	x0, vl1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincb	x0, w0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincb	x0, w0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincb	x0, w0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincb	x0, w0, pow2, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincd	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincd	x0, #14
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincd	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincd	x0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincd	x0, vl1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincd	x0, w0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincd	x0, w0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincd	x0, w0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincd	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqincd	z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqincd	z0.d, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqincd	z0.d, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqincd	z0.d, pow2, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqinch	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqinch	x0, #14
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqinch	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqinch	x0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqinch	x0, vl1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqinch	x0, w0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqinch	x0, w0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqinch	x0, w0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqinch	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqinch	z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqinch	z0.h, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqinch	z0.h, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqinch	z0.h, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     sqincp	x0, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     sqincp	x0, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     sqincp	x0, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     sqincp	x0, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     sqincp	xzr, p15.b, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     sqincp	xzr, p15.d, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     sqincp	xzr, p15.h, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     sqincp	xzr, p15.s, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqincp	z0.d, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqincp	z0.h, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqincp	z0.s, p0.s
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincw	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincw	x0, #14
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincw	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincw	x0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincw	x0, vl1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincw	x0, w0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincw	x0, w0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincw	x0, w0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     sqincw	x0, w0, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqincw	z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqincw	z0.s, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqincw	z0.s, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqincw	z0.s, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqneg	z31.b, p7/m, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqneg	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqneg	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqneg	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdcmlah	z0.b, z1.b, z2.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdcmlah	z0.d, z1.d, z2.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdcmlah	z0.h, z1.h, z2.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdcmlah	z0.h, z1.h, z2.h[0], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdcmlah	z0.s, z1.s, z2.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdcmlah	z0.s, z1.s, z2.s[0], #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdcmlah	z15.b, z16.b, z17.b, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdcmlah	z15.d, z16.d, z17.d, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdcmlah	z15.h, z16.h, z17.h, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdcmlah	z15.s, z16.s, z17.s, #270
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdcmlah	z29.b, z30.b, z31.b, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdcmlah	z29.d, z30.d, z31.d, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdcmlah	z29.h, z30.h, z31.h, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdcmlah	z29.s, z30.s, z31.s, #90
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdcmlah	z31.b, z31.b, z31.b, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdcmlah	z31.d, z31.d, z31.d, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdcmlah	z31.h, z30.h, z7.h[0], #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdcmlah	z31.h, z31.h, z31.h, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdcmlah	z31.s, z30.s, z7.s[0], #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdcmlah	z31.s, z31.s, z31.s, #180
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdmlah	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdmlah	z0.d, z1.d, z15.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdmlah	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdmlah	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdmlah	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdmlah	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdmlah	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdmlsh	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdmlsh	z0.d, z1.d, z15.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdmlsh	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdmlsh	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdmlsh	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdmlsh	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdmlsh	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdmulh	z0.b, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdmulh	z0.d, z1.d, z15.d[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdmulh	z0.h, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdmulh	z0.h, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdmulh	z0.s, z1.s, z7.s[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdmulh	z29.s, z30.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     sqrdmulh	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrunb	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrunb	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrunb	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrunb	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrunb	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrunb	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrunt	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrunt	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrunt	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrunt	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrunt	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqrshrunt	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	z0.b, p0/m, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	z0.d, p0/m, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	z0.h, p0/m, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	z0.s, p0/m, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	z31.b, p0/m, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	z31.d, p0/m, z31.d, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	z31.h, p0/m, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshl	z31.s, p0/m, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshlu	z0.b, p0/m, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshlu	z0.d, p0/m, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshlu	z0.h, p0/m, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshlu	z0.s, p0/m, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshlu	z31.b, p0/m, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshlu	z31.d, p0/m, z31.d, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshlu	z31.h, p0/m, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshlu	z31.s, p0/m, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrunb	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrunb	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrunb	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrunb	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrunb	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrunb	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrunt	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrunt	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrunt	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrunt	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrunt	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqshrunt	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsub	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsub	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsub	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsub	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsub	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsub	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsub	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsub	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsub	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsub	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsub	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsub	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsub	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsub	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsub	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsub	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsub	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsub	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsub	z31.s, z31.s, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsubr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsubr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsubr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqsubr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtnb	z0.b, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtnb	z0.h, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtnb	z0.s, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtnt	z0.b, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtnt	z0.h, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtnt	z0.s, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtunb	z0.b, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtunb	z0.h, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtunb	z0.s, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtunt	z0.b, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtunt	z0.h, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sqxtunt	z0.s, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srhadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srhadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srhadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srhadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sri	z0.b, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sri	z0.d, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sri	z0.h, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sri	z0.s, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sri	z31.b, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sri	z31.d, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sri	z31.h, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sri	z31.s, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshr	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshr	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshr	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshr	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshr	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshr	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshr	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     srshr	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     srsra	z0.b, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     srsra	z0.d, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     srsra	z0.h, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     srsra	z0.s, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     srsra	z31.b, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     srsra	z31.d, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     srsra	z31.h, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     srsra	z31.s, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshllb	z0.d, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshllb	z0.h, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshllb	z0.s, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshllb	z31.d, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshllb	z31.h, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshllb	z31.s, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshllt	z0.d, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshllt	z0.h, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshllt	z0.s, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshllt	z31.d, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshllt	z31.h, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sshllt	z31.s, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssra	z0.b, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssra	z0.d, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssra	z0.h, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssra	z0.s, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssra	z31.b, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssra	z31.d, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssra	z31.h, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssra	z31.s, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssublb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssublb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssublb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssublbt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssublbt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssublbt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssublt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssublt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssublt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssubltb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssubltb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssubltb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssubwb	z0.h, z1.h, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssubwb	z29.s, z30.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssubwb	z31.d, z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssubwt	z0.h, z1.h, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssubwt	z29.s, z30.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ssubwt	z31.d, z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1b	{ z0.b }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1b	{ z0.b }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1b	{ z0.d }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1b	{ z0.d }, p0, [x0, z0.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1b	{ z0.d }, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1b	{ z0.d }, p0, [x0, z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1b	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     st1b	{ z0.d }, p7, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1b	{ z0.h }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1b	{ z0.h }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1b	{ z0.s }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1b	{ z0.s }, p0, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1b	{ z0.s }, p0, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1b	{ z0.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     st1b	{ z0.s }, p7, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1b	{ z21.b }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1b	{ z21.d }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1b	{ z21.h }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1b	{ z21.s }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1b	{ z31.b }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1b	{ z31.d }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     st1b	{ z31.d }, p7, [z31.d, #31]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1b	{ z31.h }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1b	{ z31.s }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     st1b	{ z31.s }, p7, [z31.s, #31]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1d	{ z0.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1d	{ z0.d }, p0, [x0, z0.d, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1d	{ z0.d }, p0, [x0, z0.d, sxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1d	{ z0.d }, p0, [x0, z0.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1d	{ z0.d }, p0, [x0, z0.d, uxtw #3]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1d	{ z0.d }, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1d	{ z0.d }, p0, [x0, z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1d	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     st1d	{ z0.d }, p7, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1d	{ z21.d }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1d	{ z31.d }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     st1d	{ z31.d }, p7, [z31.d, #248]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1h	{ z0.d }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1h	{ z0.d }, p0, [x0, z0.d, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1h	{ z0.d }, p0, [x0, z0.d, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1h	{ z0.d }, p0, [x0, z0.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1h	{ z0.d }, p0, [x0, z0.d, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1h	{ z0.d }, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1h	{ z0.d }, p0, [x0, z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1h	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     st1h	{ z0.d }, p7, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1h	{ z0.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1h	{ z0.h }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1h	{ z0.s }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1h	{ z0.s }, p0, [x0, z0.s, sxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1h	{ z0.s }, p0, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1h	{ z0.s }, p0, [x0, z0.s, uxtw #1]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1h	{ z0.s }, p0, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1h	{ z0.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     st1h	{ z0.s }, p7, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1h	{ z21.d }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1h	{ z21.h }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1h	{ z21.s }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1h	{ z31.d }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     st1h	{ z31.d }, p7, [z31.d, #62]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1h	{ z31.h }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1h	{ z31.s }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     st1h	{ z31.s }, p7, [z31.s, #62]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1w	{ z0.d }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1w	{ z0.d }, p0, [x0, z0.d, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1w	{ z0.d }, p0, [x0, z0.d, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1w	{ z0.d }, p0, [x0, z0.d, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1w	{ z0.d }, p0, [x0, z0.d, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1w	{ z0.d }, p0, [x0, z0.d, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1w	{ z0.d }, p0, [x0, z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1w	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     st1w	{ z0.d }, p7, [z0.d]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1w	{ z0.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1w	{ z0.s }, p0, [x0, z0.s, sxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1w	{ z0.s }, p0, [x0, z0.s, sxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1w	{ z0.s }, p0, [x0, z0.s, uxtw #2]
+# CHECK-NEXT:  -      -      -      -      -      -     8.00    -      -      -      -      -      -      -     st1w	{ z0.s }, p0, [x0, z0.s, uxtw]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1w	{ z0.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     st1w	{ z0.s }, p7, [z0.s]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1w	{ z21.d }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1w	{ z21.s }, p5, [x10, #5, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1w	{ z31.d }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     st1w	{ z31.d }, p7, [z31.d, #124]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     st1w	{ z31.s }, p7, [sp, #-1, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     st1w	{ z31.s }, p7, [z31.s, #124]
+# CHECK-NEXT:  -      -      -      -      -      -     11.00   -      -      -      -      -      -      -     st2b	{ z0.b, z1.b }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     11.00   -      -      -      -      -      -      -     st2b	{ z0.b, z1.b }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     11.00   -      -      -      -      -      -      -     st2b	{ z21.b, z22.b }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     11.00   -      -      -      -      -      -      -     st2b	{ z23.b, z24.b }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     11.00   -      -      -      -      -      -      -     st2b	{ z5.b, z6.b }, p3, [x17, x16]
+# CHECK-NEXT:  -      -      -      -      -      -     11.00   -      -      -      -      -      -      -     st2d	{ z0.d, z1.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -     11.00   -      -      -      -      -      -      -     st2d	{ z0.d, z1.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     11.00   -      -      -      -      -      -      -     st2d	{ z21.d, z22.d }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     11.00   -      -      -      -      -      -      -     st2d	{ z23.d, z24.d }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     11.00   -      -      -      -      -      -      -     st2d	{ z5.d, z6.d }, p3, [x17, x16, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -     11.00   -      -      -      -      -      -      -     st2h	{ z0.h, z1.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     11.00   -      -      -      -      -      -      -     st2h	{ z0.h, z1.h }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     11.00   -      -      -      -      -      -      -     st2h	{ z21.h, z22.h }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     11.00   -      -      -      -      -      -      -     st2h	{ z23.h, z24.h }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     11.00   -      -      -      -      -      -      -     st2h	{ z5.h, z6.h }, p3, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     11.00   -      -      -      -      -      -      -     st2w	{ z0.s, z1.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -     11.00   -      -      -      -      -      -      -     st2w	{ z0.s, z1.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     11.00   -      -      -      -      -      -      -     st2w	{ z21.s, z22.s }, p5, [x10, #10, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     11.00   -      -      -      -      -      -      -     st2w	{ z23.s, z24.s }, p3, [x13, #-16, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     11.00   -      -      -      -      -      -      -     st2w	{ z5.s, z6.s }, p3, [x17, x16, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -     25.00   -      -      -      -      -      -      -     st3b	{ z0.b - z2.b }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     25.00   -      -      -      -      -      -      -     st3b	{ z0.b - z2.b }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     25.00   -      -      -      -      -      -      -     st3b	{ z21.b - z23.b }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     25.00   -      -      -      -      -      -      -     st3b	{ z23.b - z25.b }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     25.00   -      -      -      -      -      -      -     st3b	{ z5.b - z7.b }, p3, [x17, x16]
+# CHECK-NEXT:  -      -      -      -      -      -     14.00   -      -      -      -      -      -      -     st3d	{ z0.d - z2.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -     14.00   -      -      -      -      -      -      -     st3d	{ z0.d - z2.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     14.00   -      -      -      -      -      -      -     st3d	{ z21.d - z23.d }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     14.00   -      -      -      -      -      -      -     st3d	{ z23.d - z25.d }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     14.00   -      -      -      -      -      -      -     st3d	{ z5.d - z7.d }, p3, [x17, x16, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -     25.00   -      -      -      -      -      -      -     st3h	{ z0.h - z2.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     25.00   -      -      -      -      -      -      -     st3h	{ z0.h - z2.h }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     25.00   -      -      -      -      -      -      -     st3h	{ z21.h - z23.h }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     25.00   -      -      -      -      -      -      -     st3h	{ z23.h - z25.h }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     25.00   -      -      -      -      -      -      -     st3h	{ z5.h - z7.h }, p3, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     25.00   -      -      -      -      -      -      -     st3w	{ z0.s - z2.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -     25.00   -      -      -      -      -      -      -     st3w	{ z0.s - z2.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     25.00   -      -      -      -      -      -      -     st3w	{ z21.s - z23.s }, p5, [x10, #15, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     25.00   -      -      -      -      -      -      -     st3w	{ z23.s - z25.s }, p3, [x13, #-24, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     25.00   -      -      -      -      -      -      -     st3w	{ z5.s - z7.s }, p3, [x17, x16, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -     50.00   -      -      -      -      -      -      -     st4b	{ z0.b - z3.b }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     50.00   -      -      -      -      -      -      -     st4b	{ z0.b - z3.b }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     50.00   -      -      -      -      -      -      -     st4b	{ z21.b - z24.b }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     50.00   -      -      -      -      -      -      -     st4b	{ z23.b - z26.b }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     50.00   -      -      -      -      -      -      -     st4b	{ z5.b - z8.b }, p3, [x17, x16]
+# CHECK-NEXT:  -      -      -      -      -      -     25.00   -      -      -      -      -      -      -     st4d	{ z0.d - z3.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -     25.00   -      -      -      -      -      -      -     st4d	{ z0.d - z3.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     25.00   -      -      -      -      -      -      -     st4d	{ z21.d - z24.d }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     25.00   -      -      -      -      -      -      -     st4d	{ z23.d - z26.d }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     25.00   -      -      -      -      -      -      -     st4d	{ z5.d - z8.d }, p3, [x17, x16, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -     50.00   -      -      -      -      -      -      -     st4h	{ z0.h - z3.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     50.00   -      -      -      -      -      -      -     st4h	{ z0.h - z3.h }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     50.00   -      -      -      -      -      -      -     st4h	{ z21.h - z24.h }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     50.00   -      -      -      -      -      -      -     st4h	{ z23.h - z26.h }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     50.00   -      -      -      -      -      -      -     st4h	{ z5.h - z8.h }, p3, [x17, x16, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     50.00   -      -      -      -      -      -      -     st4w	{ z0.s - z3.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -     50.00   -      -      -      -      -      -      -     st4w	{ z0.s - z3.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     50.00   -      -      -      -      -      -      -     st4w	{ z21.s - z24.s }, p5, [x10, #20, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     50.00   -      -      -      -      -      -      -     st4w	{ z23.s - z26.s }, p3, [x13, #-32, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     50.00   -      -      -      -      -      -      -     st4w	{ z5.s - z8.s }, p3, [x17, x16, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stnt1b	{ z0.b }, p0, [x0, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stnt1b	{ z0.b }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     stnt1b	{ z0.d }, p0, [z1.d]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     stnt1b	{ z0.s }, p0, [z1.s]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stnt1b	{ z21.b }, p5, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stnt1b	{ z23.b }, p3, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     stnt1b	{ z31.d }, p7, [z31.d, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     stnt1b	{ z31.d }, p7, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     stnt1b	{ z31.s }, p7, [z31.s, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     stnt1b	{ z31.s }, p7, [z31.s]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stnt1d	{ z0.d }, p0, [x0, x0, lsl #3]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stnt1d	{ z0.d }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     stnt1d	{ z0.d }, p0, [z1.d]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stnt1d	{ z21.d }, p5, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stnt1d	{ z23.d }, p3, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     stnt1d	{ z31.d }, p7, [z31.d, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     stnt1d	{ z31.d }, p7, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     stnt1h	{ z0.d }, p0, [z1.d]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stnt1h	{ z0.h }, p0, [x0, x0, lsl #1]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stnt1h	{ z0.h }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     stnt1h	{ z0.s }, p0, [z1.s]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stnt1h	{ z21.h }, p5, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stnt1h	{ z23.h }, p3, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     stnt1h	{ z31.d }, p7, [z31.d, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     stnt1h	{ z31.d }, p7, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     stnt1h	{ z31.s }, p7, [z31.s, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     stnt1h	{ z31.s }, p7, [z31.s]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     stnt1w	{ z0.d }, p0, [z1.d]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stnt1w	{ z0.s }, p0, [x0, x0, lsl #2]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stnt1w	{ z0.s }, p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     stnt1w	{ z0.s }, p0, [z1.s]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stnt1w	{ z21.s }, p5, [x10, #7, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     stnt1w	{ z23.s }, p3, [x13, #-8, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     stnt1w	{ z31.d }, p7, [z31.d, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     7.00    -      -      -      -      -      -      -     stnt1w	{ z31.d }, p7, [z31.d]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     stnt1w	{ z31.s }, p7, [z31.s, x0]
+# CHECK-NEXT:  -      -      -      -      -      -     9.00    -      -      -      -      -      -      -     stnt1w	{ z31.s }, p7, [z31.s]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	p0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	p15, [sp, #-256, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	p5, [x10, #255, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	z0, [x0]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	z21, [x10, #-256, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -     1.00    -      -      -      -      -      -      -     str	z31, [sp, #255, mul vl]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z21.b, p5/m, z21.b, z10.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z21.b, z10.b, z21.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z21.d, p5/m, z21.d, z10.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z21.d, z10.d, z21.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z21.h, p5/m, z21.h, z10.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z21.h, z10.h, z21.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z21.s, p5/m, z21.s, z10.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z21.s, z10.s, z21.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z23.b, p3/m, z23.b, z13.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z23.b, z13.b, z8.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z23.d, p3/m, z23.d, z13.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z23.d, z13.d, z8.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z23.h, p3/m, z23.h, z13.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z23.h, z13.h, z8.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z23.s, p3/m, z23.s, z13.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z23.s, z13.s, z8.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z31.s, z31.s, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sub	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     subhnb	z0.b, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     subhnb	z0.h, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     subhnb	z0.s, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     subhnt	z0.b, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     subhnt	z0.h, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     subhnt	z0.s, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     subr	z0.b, p0/m, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     subr	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     subr	z0.d, p0/m, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     subr	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     subr	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     subr	z0.h, p0/m, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     subr	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     subr	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     subr	z0.s, p0/m, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     subr	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     subr	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     subr	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     subr	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     subr	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     subr	z31.s, z31.s, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sunpkhi	z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sunpkhi	z31.h, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sunpkhi	z31.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sunpklo	z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sunpklo	z31.h, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sunpklo	z31.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     suqadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     suqadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     suqadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     suqadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sxtb	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sxtb	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sxtb	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sxtb	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sxtb	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sxtb	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sxth	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sxth	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sxth	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sxth	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sxtw	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     sxtw	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbl	z28.b, { z29.b, z30.b }, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbl	z28.d, { z29.d, z30.d }, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbl	z28.h, { z29.h, z30.h }, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbl	z28.s, { z29.s, z30.s }, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbl	z31.b, { z31.b }, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbl	z31.d, { z31.d }, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbl	z31.h, { z31.h }, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbl	z31.s, { z31.s }, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbx	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbx	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbx	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     tbx	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     trn1	p15.b, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     trn1	p15.d, p15.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     trn1	p15.h, p15.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     trn1	p15.s, p15.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     trn1	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     trn1	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     trn1	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     trn1	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     trn2	p15.b, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     trn2	p15.d, p15.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     trn2	p15.h, p15.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     trn2	p15.s, p15.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     trn2	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     trn2	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     trn2	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     trn2	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     uaba	z0.b, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     uaba	z0.d, z1.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     uaba	z0.h, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     uaba	z0.s, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     uabalb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     uabalb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     uabalb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     uabalt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     uabalt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     uabalt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uabd	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uabd	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uabd	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uabd	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uabdlb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uabdlb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uabdlb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uabdlt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uabdlt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uabdlt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     uadalp	z0.h, p0/m, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     uadalp	z29.s, p0/m, z30.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     uadalp	z30.d, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddlb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddlb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddlb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddlt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddlt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddlt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uaddv	d0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uaddv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uaddv	d0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uaddv	d0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddwb	z0.h, z1.h, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddwb	z29.s, z30.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddwb	z31.d, z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddwt	z0.h, z1.h, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddwt	z29.s, z30.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uaddwt	z31.d, z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	z0.d, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	z0.h, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	z0.h, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	z0.s, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ucvtf	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     23.00  udiv	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     12.00  udiv	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     23.00  udivr	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -      -      -     12.00  udivr	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     udot	z0.d, z1.h, z15.h[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     udot	z0.d, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     udot	z0.s, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     udot	z0.s, z1.b, z7.b[3]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uhadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uhadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uhadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uhadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uhsub	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uhsub	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uhsub	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uhsub	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uhsubr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uhsubr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uhsubr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uhsubr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umax	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umax	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umax	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umax	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umax	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umax	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umaxp	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umaxp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umaxp	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umaxp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     umaxv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     umaxv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     umaxv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     umaxv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umin	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umin	z31.b, p7/m, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umin	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umin	z31.d, p7/m, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umin	z31.h, p7/m, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     umin	z31.s, p7/m, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uminp	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uminp	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uminp	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uminp	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uminv	b0, p7, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uminv	d0, p7, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uminv	h0, p7, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uminv	s0, p7, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umlalb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umlalb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umlalb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umlalb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umlalb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umlalt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umlalt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umlalt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umlalt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umlalt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umlslb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umlslb	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umlslb	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umlslb	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umlslb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umlslt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umlslt	z0.d, z1.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umlslt	z0.h, z1.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umlslt	z0.s, z1.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umlslt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     ummla	z0.s, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umulh	z0.b, p7/m, z0.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umulh	z0.b, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umulh	z0.d, p7/m, z0.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umulh	z0.h, p7/m, z0.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umulh	z0.h, z1.h, z2.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umulh	z0.s, p7/m, z0.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umulh	z29.s, z30.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umulh	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umullb	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umullb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umullb	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umullb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umullb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umullt	z0.d, z1.s, z15.s[1]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umullt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umullt	z0.s, z1.h, z7.h[7]
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umullt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     umullt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqadd	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqadd	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqadd	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqadd	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqadd	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqadd	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqadd	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqadd	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqadd	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqadd	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqadd	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqadd	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqadd	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqadd	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqadd	z31.s, z31.s, #65280
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecb	w0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecb	w0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecb	w0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecb	w0, pow2, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecb	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecb	x0, #14
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecb	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecb	x0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecb	x0, vl1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecd	w0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecd	w0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecd	w0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecd	w0, pow2, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecd	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecd	x0, #14
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecd	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecd	x0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecd	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqdecd	z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqdecd	z0.d, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqdecd	z0.d, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqdecd	z0.d, pow2, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdech	w0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdech	w0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdech	w0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdech	w0, pow2, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdech	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdech	x0, #14
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdech	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdech	x0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdech	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqdech	z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqdech	z0.h, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqdech	z0.h, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqdech	z0.h, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uqdecp	wzr, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uqdecp	wzr, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uqdecp	wzr, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uqdecp	wzr, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uqdecp	x0, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uqdecp	x0, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uqdecp	x0, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uqdecp	x0, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqdecp	z0.d, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqdecp	z0.h, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqdecp	z0.s, p0.s
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecw	w0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecw	w0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecw	w0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecw	w0, pow2, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecw	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecw	x0, #14
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecw	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecw	x0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqdecw	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqdecw	z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqdecw	z0.s, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqdecw	z0.s, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqdecw	z0.s, pow2, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincb	w0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincb	w0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincb	w0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincb	w0, pow2, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincb	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincb	x0, #14
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincb	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincb	x0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincb	x0, vl1
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincd	w0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincd	w0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincd	w0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincd	w0, pow2, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincd	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincd	x0, #14
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincd	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincd	x0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincd	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqincd	z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqincd	z0.d, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqincd	z0.d, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqincd	z0.d, pow2, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqinch	w0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqinch	w0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqinch	w0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqinch	w0, pow2, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqinch	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqinch	x0, #14
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqinch	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqinch	x0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqinch	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqinch	z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqinch	z0.h, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqinch	z0.h, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqinch	z0.h, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uqincp	wzr, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uqincp	wzr, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uqincp	wzr, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uqincp	wzr, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uqincp	x0, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uqincp	x0, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uqincp	x0, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uqincp	x0, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqincp	z0.d, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqincp	z0.h, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqincp	z0.s, p0.s
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincw	w0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincw	w0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincw	w0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincw	w0, pow2, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincw	x0
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincw	x0, #14
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincw	x0, all, mul #16
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincw	x0, pow2
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     uqincw	x0, vl1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqincw	z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqincw	z0.s, all, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqincw	z0.s, pow2
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqincw	z0.s, pow2, mul #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqrshrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	z0.b, p0/m, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	z0.d, p0/m, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	z0.h, p0/m, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	z0.s, p0/m, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	z31.b, p0/m, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	z31.d, p0/m, z31.d, #63
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	z31.h, p0/m, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshl	z31.s, p0/m, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshrnb	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshrnb	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshrnb	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshrnb	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshrnb	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshrnb	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshrnt	z0.b, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshrnt	z0.h, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshrnt	z0.s, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshrnt	z31.b, z31.h, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshrnt	z31.h, z31.s, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqshrnt	z31.s, z31.d, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqsub	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqsub	z0.b, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqsub	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqsub	z0.d, z0.d, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqsub	z0.d, z0.d, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqsub	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqsub	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqsub	z0.h, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqsub	z0.h, z0.h, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqsub	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqsub	z0.s, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqsub	z0.s, z0.s, #0, lsl #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqsub	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqsub	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqsub	z31.b, z31.b, #255
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqsub	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqsub	z31.d, z31.d, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqsub	z31.h, z31.h, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqsub	z31.s, z31.s, #65280
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqsubr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqsubr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqsubr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqsubr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqxtnb	z0.b, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqxtnb	z0.h, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqxtnb	z0.s, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqxtnt	z0.b, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqxtnt	z0.h, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uqxtnt	z0.s, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     urecpe	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urhadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urhadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urhadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urhadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshl	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshl	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshl	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshl	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshlr	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshlr	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshlr	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshlr	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshr	z0.b, p0/m, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshr	z0.d, p0/m, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshr	z0.h, p0/m, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshr	z0.s, p0/m, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshr	z31.b, p0/m, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshr	z31.d, p0/m, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshr	z31.h, p0/m, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     urshr	z31.s, p0/m, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     ursqrte	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     ursra	z0.b, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     ursra	z0.d, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     ursra	z0.h, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     ursra	z0.s, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     ursra	z31.b, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     ursra	z31.d, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     ursra	z31.h, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00   1.00    -      -      -     ursra	z31.s, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ushllb	z0.d, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ushllb	z0.h, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ushllb	z0.s, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ushllb	z31.d, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ushllb	z31.h, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ushllb	z31.s, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ushllt	z0.d, z0.s, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ushllt	z0.h, z0.b, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ushllt	z0.s, z0.h, #0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ushllt	z31.d, z31.s, #31
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ushllt	z31.h, z31.b, #7
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     ushllt	z31.s, z31.h, #15
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -      -      -     0.50   0.50    -     usmmla	z0.s, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usqadd	z0.b, p0/m, z0.b, z1.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usqadd	z0.h, p0/m, z0.h, z1.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usqadd	z29.s, p7/m, z29.s, z30.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usqadd	z31.d, p7/m, z31.d, z30.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usra	z0.b, z0.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usra	z0.d, z0.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usra	z0.h, z0.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usra	z0.s, z0.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usra	z31.b, z31.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usra	z31.d, z31.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usra	z31.h, z31.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usra	z31.s, z31.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usublb	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usublb	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usublb	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usublt	z0.h, z1.b, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usublt	z29.s, z30.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usublt	z31.d, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usubwb	z0.h, z1.h, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usubwb	z29.s, z30.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usubwb	z31.d, z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usubwt	z0.h, z1.h, z2.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usubwt	z29.s, z30.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     usubwt	z31.d, z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uunpkhi	z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uunpkhi	z31.h, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uunpkhi	z31.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uunpklo	z31.d, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uunpklo	z31.h, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uunpklo	z31.s, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uxtb	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uxtb	z0.h, p0/m, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uxtb	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uxtb	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uxtb	z31.h, p7/m, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uxtb	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uxth	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uxth	z0.s, p0/m, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uxth	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uxth	z31.s, p7/m, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uxtw	z0.d, p0/m, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uxtw	z31.d, p7/m, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uzp1	p15.b, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uzp1	p15.d, p15.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uzp1	p15.h, p15.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uzp1	p15.s, p15.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uzp1	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uzp1	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uzp1	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uzp1	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uzp2	p15.b, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uzp2	p15.d, p15.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uzp2	p15.h, p15.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     uzp2	p15.s, p15.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uzp2	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uzp2	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uzp2	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     uzp2	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     whilege	p15.b, w0, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     whilege	p15.b, wzr, w0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     whilege	p15.b, x0, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     whilege	p15.b, xzr, x0
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     whilege	p15.d, w0, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     whilege	p15.d, x0, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     whilege	p15.h, w0, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     whilege	p15.h, x0, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     whilege	p15.s, w0, wzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     whilege	p15.s, x0, xzr
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     whilerw	p15.b, x30, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     whilerw	p15.d, x30, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     whilerw	p15.h, x30, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     whilerw	p15.s, x30, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     whilewr	p15.b, x30, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     whilewr	p15.d, x30, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     whilewr	p15.h, x30, x30
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     whilewr	p15.s, x30, x30
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     wrffr	p0.b
+# CHECK-NEXT: 0.33   0.33   0.33    -      -      -      -      -      -      -      -      -      -      -     wrffr	p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     xar	z0.b, z0.b, z1.b, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     xar	z0.d, z0.d, z1.d, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     xar	z0.h, z0.h, z1.h, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     xar	z0.s, z0.s, z1.s, #1
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     xar	z31.b, z31.b, z30.b, #8
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     xar	z31.d, z31.d, z30.d, #64
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     xar	z31.h, z31.h, z30.h, #16
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     xar	z31.s, z31.s, z30.s, #32
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     zip1	p0.b, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     zip1	p0.d, p0.d, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     zip1	p0.h, p0.h, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     zip1	p0.s, p0.s, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     zip1	p15.b, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     zip1	p15.d, p15.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     zip1	p15.h, p15.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     zip1	p15.s, p15.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip1	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip1	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip1	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip1	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip1	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip1	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip1	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip1	z31.s, z31.s, z31.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     zip2	p0.b, p0.b, p0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     zip2	p0.d, p0.d, p0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     zip2	p0.h, p0.h, p0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     zip2	p0.s, p0.s, p0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     zip2	p15.b, p15.b, p15.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     zip2	p15.d, p15.d, p15.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     zip2	p15.h, p15.h, p15.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     1.00    -      -      -      -     zip2	p15.s, p15.s, p15.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip2	z0.b, z0.b, z0.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip2	z0.d, z0.d, z0.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip2	z0.h, z0.h, z0.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip2	z0.s, z0.s, z0.s
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip2	z31.b, z31.b, z31.b
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip2	z31.d, z31.d, z31.d
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip2	z31.h, z31.h, z31.h
+# CHECK-NEXT:  -      -      -      -      -      -      -      -      -     0.50   0.50    -      -      -     zip2	z31.s, z31.s, z31.s

From e2b19ef6ac7e5635699730ff561e7f24775704c9 Mon Sep 17 00:00:00 2001
From: Alex Zinenko <zinenko@google.com>
Date: Mon, 19 Jun 2023 13:21:33 +0200
Subject: [PATCH 114/130] [mlir] mark libraries in mlir/examples as such

LLVM build system separates between `add_llvm_example_library` and
`add_llvm_library`, which is presumably used to package examples
separately from the regular library. Introduce a similar approach to
building example libraries in MLIR and use it for the transform dialect
tutorial.

Reviewed By: mehdi_amini

Differential Revision: https://reviews.llvm.org/D153265
---
 mlir/cmake/modules/AddMLIR.cmake              | 108 ++++++++++++------
 .../examples/transform/Ch2/lib/CMakeLists.txt |   9 +-
 .../examples/transform/Ch3/lib/CMakeLists.txt |   9 +-
 3 files changed, 82 insertions(+), 44 deletions(-)

diff --git a/mlir/cmake/modules/AddMLIR.cmake b/mlir/cmake/modules/AddMLIR.cmake
index f6fb8a639fde9..6acb01c305ad0 100644
--- a/mlir/cmake/modules/AddMLIR.cmake
+++ b/mlir/cmake/modules/AddMLIR.cmake
@@ -201,25 +201,9 @@ function(add_mlir_doc doc_filename output_file output_directory command)
   add_dependencies(mlir-doc ${output_file}DocGen)
 endfunction()
 
-# Declare an mlir library which can be compiled in libMLIR.so
-# In addition to everything that llvm_add_library accepts, this
-# also has the following option:
-# EXCLUDE_FROM_LIBMLIR
-#   Don't include this library in libMLIR.so.  This option should be used
-#   for test libraries, executable-specific libraries, or rarely used libraries
-#   with large dependencies.
-# ENABLE_AGGREGATION
-#   Forces generation of an OBJECT library, exports additional metadata,
-#   and installs additional object files needed to include this as part of an
-#   aggregate shared library.
-#   TODO: Make this the default for all MLIR libraries once all libraries
-#   are compatible with building an object library.
-function(add_mlir_library name)
-  cmake_parse_arguments(ARG
-    "SHARED;INSTALL_WITH_TOOLCHAIN;EXCLUDE_FROM_LIBMLIR;DISABLE_INSTALL;ENABLE_AGGREGATION"
-    ""
-    "ADDITIONAL_HEADERS;DEPENDS;LINK_COMPONENTS;LINK_LIBS"
-    ${ARGN})
+# Sets ${srcs} to contain the list of additional headers for the target. Extra
+# arguments are included into the list of additional headers.
+function(_set_mlir_additional_headers_as_srcs)
   set(srcs)
   if(MSVC_IDE OR XCODE)
     # Add public headers
@@ -245,13 +229,83 @@ function(add_mlir_library name)
       endif()
     endif()
   endif(MSVC_IDE OR XCODE)
-  if(srcs OR ARG_ADDITIONAL_HEADERS)
+  if(srcs OR ARGN)
     set(srcs
       ADDITIONAL_HEADERS
       ${srcs}
-      ${ARG_ADDITIONAL_HEADERS} # It may contain unparsed unknown args.
+      ${ARGN} # It may contain unparsed unknown args.
+      PARENT_SCOPE
       )
   endif()
+endfunction()
+
+# Checks that the LLVM components are not listed in the extra arguments,
+# assumed to be coming from the LINK_LIBS variable.
+function(_check_llvm_components_usage name)
+  # LINK_COMPONENTS is necessary to allow libLLVM.so to be properly
+  # substituted for individual library dependencies if LLVM_LINK_LLVM_DYLIB
+  # Perhaps this should be in llvm_add_library instead?  However, it fails
+  # on libclang-cpp.so
+  get_property(llvm_component_libs GLOBAL PROPERTY LLVM_COMPONENT_LIBS)
+  foreach(lib ${ARGN})
+    if(${lib} IN_LIST llvm_component_libs)
+      message(SEND_ERROR "${name} specifies LINK_LIBS ${lib}, but LINK_LIBS cannot be used for LLVM libraries.  Please use LINK_COMPONENTS instead.")
+    endif()
+  endforeach()
+endfunction()
+
+function(add_mlir_example_library name)
+  cmake_parse_arguments(ARG
+    "SHARED;DISABLE_INSTALL"
+    ""
+    "ADDITIONAL_HEADERS;DEPENDS;LINK_COMPONENTS;LINK_LIBS"
+    ${ARGN})
+  _set_mlir_additional_headers_as_srcs(${ARG_ADDITIONAL_HEADERS})
+  if (ARG_SHARED)
+    set(LIBTYPE SHARED)
+  else()
+    if(BUILD_SHARED_LIBS)
+      set(LIBTYPE SHARED)
+    else()
+      set(LIBTYPE STATIC)
+    endif()
+  endif()
+
+  # MLIR libraries uniformly depend on LLVMSupport.  Just specify it once here.
+  list(APPEND ARG_LINK_COMPONENTS Support)
+  _check_llvm_components_usage(${name} ${ARG_LINK_LIBS})
+
+  list(APPEND ARG_DEPENDS mlir-generic-headers)
+
+  llvm_add_library(${name} ${LIBTYPE} ${ARG_UNPARSED_ARGUMENTS} ${srcs} DEPENDS ${ARG_DEPENDS} LINK_COMPONENTS ${ARG_LINK_COMPONENTS} LINK_LIBS ${ARG_LINK_LIBS})
+  set_target_properties(${name} PROPERTIES FOLDER "Examples")
+  if (LLVM_BUILD_EXAMPLES AND NOT ${ARG_DISABLE_INSTALL})
+    add_mlir_library_install(${name})
+  else()
+    set_target_properties(${name} PROPERTIES EXCLUDE_FROM_ALL ON)
+  endif()
+endfunction()
+
+# Declare an mlir library which can be compiled in libMLIR.so
+# In addition to everything that llvm_add_library accepts, this
+# also has the following option:
+# EXCLUDE_FROM_LIBMLIR
+#   Don't include this library in libMLIR.so.  This option should be used
+#   for test libraries, executable-specific libraries, or rarely used libraries
+#   with large dependencies.
+# ENABLE_AGGREGATION
+#   Forces generation of an OBJECT library, exports additional metadata,
+#   and installs additional object files needed to include this as part of an
+#   aggregate shared library.
+#   TODO: Make this the default for all MLIR libraries once all libraries
+#   are compatible with building an object library.
+function(add_mlir_library name)
+  cmake_parse_arguments(ARG
+    "SHARED;INSTALL_WITH_TOOLCHAIN;EXCLUDE_FROM_LIBMLIR;DISABLE_INSTALL;ENABLE_AGGREGATION"
+    ""
+    "ADDITIONAL_HEADERS;DEPENDS;LINK_COMPONENTS;LINK_LIBS"
+    ${ARGN})
+  _set_mlir_additional_headers_as_srcs(${ARG_ADDITIONAL_HEADERS})
 
   # Is an object library needed.
   set(NEEDS_OBJECT_LIB OFF)
@@ -287,17 +341,7 @@ function(add_mlir_library name)
 
   # MLIR libraries uniformly depend on LLVMSupport.  Just specify it once here.
   list(APPEND ARG_LINK_COMPONENTS Support)
-
-  # LINK_COMPONENTS is necessary to allow libLLVM.so to be properly
-  # substituted for individual library dependencies if LLVM_LINK_LLVM_DYLIB
-  # Perhaps this should be in llvm_add_library instead?  However, it fails
-  # on libclang-cpp.so
-  get_property(llvm_component_libs GLOBAL PROPERTY LLVM_COMPONENT_LIBS)
-  foreach(lib ${ARG_LINK_LIBS})
-    if(${lib} IN_LIST llvm_component_libs)
-      message(SEND_ERROR "${name} specifies LINK_LIBS ${lib}, but LINK_LIBS cannot be used for LLVM libraries.  Please use LINK_COMPONENTS instead.")
-    endif()
-  endforeach()
+  _check_llvm_components_usage(${name} ${ARG_LINK_LIBS})
 
   list(APPEND ARG_DEPENDS mlir-generic-headers)
   llvm_add_library(${name} ${LIBTYPE} ${ARG_UNPARSED_ARGUMENTS} ${srcs} DEPENDS ${ARG_DEPENDS} LINK_COMPONENTS ${ARG_LINK_COMPONENTS} LINK_LIBS ${ARG_LINK_LIBS})
diff --git a/mlir/examples/transform/Ch2/lib/CMakeLists.txt b/mlir/examples/transform/Ch2/lib/CMakeLists.txt
index c89cd97006eb5..9c2327284292e 100644
--- a/mlir/examples/transform/Ch2/lib/CMakeLists.txt
+++ b/mlir/examples/transform/Ch2/lib/CMakeLists.txt
@@ -1,14 +1,11 @@
-add_mlir_library(
+# Outside examples, this should be `add_mlir_library`.
+add_mlir_example_library(
   # Library called MyExtension.
   MyExtensionCh2
 
   # Built from the following source files.
   MyExtension.cpp
 
-  # Do not make this part of the main library distribution.
-  # Needed for the example only.
-  EXCLUDE_FROM_LIBMLIR
-
   # Make includes visible without top-level path.
   ADDITIONAL_HEADER_DIRS
   ${PROJECT_SOURCE_DIR}/examples/transform/Ch2/include
@@ -18,7 +15,7 @@ add_mlir_library(
   MyExtensionCh2IncGen
 
   # Link in the transform dialect, an all generated dialects.
-  LINK_LIBS PUBLIC
+  LINK_LIBS PRIVATE
   MLIRTransformDialect
   MLIRFuncDialect
   MLIRSCFDialect
diff --git a/mlir/examples/transform/Ch3/lib/CMakeLists.txt b/mlir/examples/transform/Ch3/lib/CMakeLists.txt
index b43f79cc91a68..2c04a8bab85a2 100644
--- a/mlir/examples/transform/Ch3/lib/CMakeLists.txt
+++ b/mlir/examples/transform/Ch3/lib/CMakeLists.txt
@@ -1,14 +1,11 @@
-add_mlir_library(
+# Outside examples, this should be `add_mlir_library`.
+add_mlir_example_library(
   # Library called MyExtension.
   MyExtensionCh3
 
   # Built from the following source files.
   MyExtension.cpp
 
-  # Do not make this part of the main library distribution.
-  # Needed for the example only.
-  EXCLUDE_FROM_LIBMLIR
-
   # Make includes visible without top-level path.
   ADDITIONAL_HEADER_DIRS
   ${PROJECT_SOURCE_DIR}/examples/transform/Ch3/include
@@ -18,7 +15,7 @@ add_mlir_library(
   MyExtensionCh3IncGen
 
   # Link in the transform dialect, an all generated dialects.
-  LINK_LIBS PUBLIC
+  LINK_LIBS PRIVATE
   MLIRTransformDialect
   MLIRFuncDialect
   MLIRSCFDialect

From f6c1796b446f4c6cbff082fe3dd3f982171a8597 Mon Sep 17 00:00:00 2001
From: Alexey Lapshin <a.v.lapshin@mail.ru>
Date: Mon, 19 Jun 2023 17:04:19 +0200
Subject: [PATCH 115/130] [DWARFLinker] add
 DWARFUnit::getIndexedAddressOffset().

This patch is a followup for D153162. It cures one more place
where indexed address was incorrectly read. It also moves handling
of indexed address into DWARFUnit.

Differential Revision: https://reviews.llvm.org/D153297
---
 llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h |   9 ++
 llvm/lib/DWARFLinker/DWARFLinker.cpp          |  14 ++-
 .../dsymutil/Inputs/dwarf5-dw-op-addrx.o      | Bin 1856 -> 2320 bytes
 .../dsymutil/X86/dwarf5-dw-op-addrx.test      |  81 +++++++++++++++++-
 llvm/tools/dsymutil/DwarfLinkerForBinary.cpp  |  16 ++--
 5 files changed, 99 insertions(+), 21 deletions(-)

diff --git a/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h b/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h
index 8d6310a7f2745..b42c951598d6f 100644
--- a/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h
+++ b/llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h
@@ -355,6 +355,15 @@ class DWARFUnit {
     return AddrOffsetSectionBase;
   }
 
+  /// Returns offset to the indexed address value inside .debug_addr section.
+  std::optional<uint64_t> getIndexedAddressOffset(uint64_t Index) {
+    if (std::optional<uint64_t> AddrOffsetSectionBase =
+            getAddrOffsetSectionBase())
+      return *AddrOffsetSectionBase + Index * getAddressByteSize();
+
+    return std::nullopt;
+  }
+
   /// Recursively update address to Die map.
   void updateAddressDieMap(DWARFDie Die);
 
diff --git a/llvm/lib/DWARFLinker/DWARFLinker.cpp b/llvm/lib/DWARFLinker/DWARFLinker.cpp
index 18b81468fc906..0c84bf161aa15 100644
--- a/llvm/lib/DWARFLinker/DWARFLinker.cpp
+++ b/llvm/lib/DWARFLinker/DWARFLinker.cpp
@@ -489,16 +489,14 @@ DWARFLinker::getVariableRelocAdjustment(AddressesMap &RelocMgr,
     } break;
     case dwarf::DW_OP_constx:
     case dwarf::DW_OP_addrx: {
-      if (std::optional<uint64_t> AddrOffsetSectionBase =
-              DIE.getDwarfUnit()->getAddrOffsetSectionBase()) {
-        uint64_t StartOffset = *AddrOffsetSectionBase + Op.getRawOperand(0);
-        uint64_t EndOffset =
-            StartOffset + DIE.getDwarfUnit()->getAddressByteSize();
-
+      if (std::optional<uint64_t> AddressOffset =
+              DIE.getDwarfUnit()->getIndexedAddressOffset(
+                  Op.getRawOperand(0))) {
         // Check relocation for the address.
         if (std::optional<int64_t> RelocAdjustment =
-                RelocMgr.getExprOpAddressRelocAdjustment(*U, Op, StartOffset,
-                                                         EndOffset))
+                RelocMgr.getExprOpAddressRelocAdjustment(
+                    *U, Op, *AddressOffset,
+                    *AddressOffset + DIE.getDwarfUnit()->getAddressByteSize()))
           return *RelocAdjustment;
       }
     } break;
diff --git a/llvm/test/tools/dsymutil/Inputs/dwarf5-dw-op-addrx.o b/llvm/test/tools/dsymutil/Inputs/dwarf5-dw-op-addrx.o
index fb2a6d3242ec5b9e49fac277fba58145b21f6aad..62d1b03eb20e5bb5f50194c7698935f02aaf69c0 100644
GIT binary patch
delta 862
zcmZ8fziSjh6rOpzJG*<iTXSa<BT23Z-VsF1-DR^EV(tndCs>H6M2LhdECfw)g&--C
z=880$<cW<<78HzyTG%Km0sjIeg?6S0<WI0r@SE8w;tjm-$D23vcK7Xmv{l+_m)0C%
z){>j?nzMcVvFvOgE*7t48S&ZjD{jhGu^=;h-)8UfCbExg_KnSc<Darss!(R_rdU@z
zMGIq9n_jZT8!8hAz7RR?Q+;&9R+MeU4^-#}oecOp#l>B_-L~7m`Ha&V{bjSuHk;r&
zz0L>RbK=sxebZK!6;FNs=qwZ(0h%Ip0G?;hosc0<yA^J``S50E^Jsei&ezM_bt_}f
zQ1u+Ki+G86jd+WAkNCiS_erJO>(vCHuA!k}MnkM&Rzsqp$)~-CXZ2}MNOToKio8`Y
z6!HtNvP$ZL-XcmR`9?aF5(P%OlsXg|=~3!Z&PYY6M?*&Xlq$*_8Bpp|!N`!(zz=mx
zc`FjRJqYiVhsodILvI4FJYr*bM<vw$%X4a6F7a(uE9b8s&Go-5#^;XT9ao2TKH*Sk
zzc)q`O5RgfI{J^ntW^gb3K6xz=oTel3?sS-#w4RE*aMfrxF{L{V>;0=7?(!;M<weq
z3@&qZv0nP{Nk28<CkOoafY(h<o-Ymf{Qv%v5Gxp#Sb5OBw`ye5U}A9AU~F*4V8dYD
HU`_l3AHH@|

delta 448
zcmbOrbbxPygV0PS1~7<VfzUIVCI<B}YD|3SE@%J|VVDY~J0@E)+6x|q@^YZG?Bq_M
zs39YS8wM5aoP3kfUZ@=^3Z)qCvrg7xvXkwA@+F{(AFx6gU)UHJm?pO}8A~mIs!)Kc
z_yJVG3}mygPrk%tCU^iU?EsaQm@LU`FU0`$N)%Ky0%#lv@UTxVVm1?sf=b&%L>V@*
zP2R<9$1B1Pv=k_00;G8+v$DuDGESbzX3QDt(fQQyK<r_a$%!oLlUK0xOpakKl@0=`
zWMyFF0GYwazyoCgIZTX`->|BSv4f=;!`K*@0zpdIS%KjyGP#jWe)0o03nqnslcm_@
z7{ex8vKz_0f3ZjL_rAr=Mr*sC9@J^72CC$Qx{-lFcyc$pCl>>Q0N5f1w#hOa7Ca2`
OiA6;W@wthac?<v|aYY~i

diff --git a/llvm/test/tools/dsymutil/X86/dwarf5-dw-op-addrx.test b/llvm/test/tools/dsymutil/X86/dwarf5-dw-op-addrx.test
index 8951470f4c5b6..08a12022bfe53 100644
--- a/llvm/test/tools/dsymutil/X86/dwarf5-dw-op-addrx.test
+++ b/llvm/test/tools/dsymutil/X86/dwarf5-dw-op-addrx.test
@@ -4,6 +4,17 @@
 
 ## cat dwarf5-dw-op-addrx.c
 
+## char arr[40];
+## char arr2[40];
+## char arr3[40];
+## char arr4[40];
+## char arr5[40];
+## char arr6[40];
+## char arr7[40];
+## char arr8[40];
+## char arr9[40];
+## int main() { return 0;}
+
 ## $ clang -gdwarf-5 dwarf5-dw-op-addrx.c -c -O2 -o dwarf5-dw-op-addrx.o
 
 #RUN: dsymutil -oso-prepend-path %p/../Inputs -y %s -o %t.dSYM
@@ -21,19 +32,75 @@
 #DWARF-CHECK: DW_AT_low_pc {{.*}}0x0000000100000fb0
 #DWARF-CHECK: DW_TAG_variable
 #DWARF-CHECK:   DW_AT_name {{.*}}"arr"
+#DWARF-CHECK:   DW_AT_location {{.*}}(DW_OP_addr 0x100001000)
+#DWARF-CHECK: DW_TAG_variable
+#DWARF-CHECK:   DW_AT_name {{.*}}"arr2"
 #DWARF-CHECK:   DW_AT_location {{.*}}(DW_OP_addr 0x100002000)
+#DWARF-CHECK: DW_TAG_variable
+#DWARF-CHECK:   DW_AT_name {{.*}}"arr3"
+#DWARF-CHECK:   DW_AT_location {{.*}}(DW_OP_addr 0x100003000)
+#DWARF-CHECK: DW_TAG_variable
+#DWARF-CHECK:   DW_AT_name {{.*}}"arr4"
+#DWARF-CHECK:   DW_AT_location {{.*}}(DW_OP_addr 0x100004000)
+#DWARF-CHECK: DW_TAG_variable
+#DWARF-CHECK:   DW_AT_name {{.*}}"arr5"
+#DWARF-CHECK:   DW_AT_location {{.*}}(DW_OP_addr 0x100005000)
+#DWARF-CHECK: DW_TAG_variable
+#DWARF-CHECK:   DW_AT_name {{.*}}"arr6"
+#DWARF-CHECK:   DW_AT_location {{.*}}(DW_OP_addr 0x100006000)
+#DWARF-CHECK: DW_TAG_variable
+#DWARF-CHECK:   DW_AT_name {{.*}}"arr7"
+#DWARF-CHECK:   DW_AT_location {{.*}}(DW_OP_addr 0x100007000)
+#DWARF-CHECK: DW_TAG_variable
+#DWARF-CHECK:   DW_AT_name {{.*}}"arr8"
+#DWARF-CHECK:   DW_AT_location {{.*}}(DW_OP_addr 0x100008000)
+#DWARF-CHECK: DW_TAG_variable
+#DWARF-CHECK:   DW_AT_name {{.*}}"arr9"
+#DWARF-CHECK:   DW_AT_location {{.*}}(DW_OP_addr 0x100009000)
 #DWARF-CHECK-NOT: .debug_addr
 
 #UPD-DWARF-CHECK: DW_TAG_compile_unit
 #UPD-DWARF-CHECK: DW_AT_name {{.*}}"dwarf5-dw-op-addrx.c"
-#UPD-DWARF-CHECK: DW_AT_low_pc [DW_FORM_addrx]      (indexed (00000001) address = 0x0000000000000000)
+#UPD-DWARF-CHECK: DW_AT_low_pc [DW_FORM_addrx]      (indexed (00000009) address = 0x0000000000000000)
 #UPD-DWARF-CHECK: DW_AT_high_pc [DW_FORM_data4]     (0x00000008)
 #UPD-DWARF-CHECK: DW_AT_addr_base [DW_FORM_sec_offset]      (0x00000008)
 #UPD-DWARF-CHECK: DW_TAG_variable
 #UPD-DWARF-CHECK:   DW_AT_name {{.*}}"arr"
 #UPD-DWARF-CHECK:   DW_AT_location [DW_FORM_exprloc]        (DW_OP_addrx 0x0)
+#UPD-DWARF-CHECK: DW_TAG_variable
+#UPD-DWARF-CHECK:   DW_AT_name {{.*}}"arr2"
+#UPD-DWARF-CHECK:   DW_AT_location [DW_FORM_exprloc]        (DW_OP_addrx 0x1)
+#UPD-DWARF-CHECK: DW_TAG_variable
+#UPD-DWARF-CHECK:   DW_AT_name {{.*}}"arr3"
+#UPD-DWARF-CHECK:   DW_AT_location [DW_FORM_exprloc]        (DW_OP_addrx 0x2)
+#UPD-DWARF-CHECK: DW_TAG_variable
+#UPD-DWARF-CHECK:   DW_AT_name {{.*}}"arr4"
+#UPD-DWARF-CHECK:   DW_AT_location [DW_FORM_exprloc]        (DW_OP_addrx 0x3)
+#UPD-DWARF-CHECK: DW_TAG_variable
+#UPD-DWARF-CHECK:   DW_AT_name {{.*}}"arr5"
+#UPD-DWARF-CHECK:   DW_AT_location [DW_FORM_exprloc]        (DW_OP_addrx 0x4)
+#UPD-DWARF-CHECK: DW_TAG_variable
+#UPD-DWARF-CHECK:   DW_AT_name {{.*}}"arr6"
+#UPD-DWARF-CHECK:   DW_AT_location [DW_FORM_exprloc]        (DW_OP_addrx 0x5)
+#UPD-DWARF-CHECK: DW_TAG_variable
+#UPD-DWARF-CHECK:   DW_AT_name {{.*}}"arr7"
+#UPD-DWARF-CHECK:   DW_AT_location [DW_FORM_exprloc]        (DW_OP_addrx 0x6)
+#UPD-DWARF-CHECK: DW_TAG_variable
+#UPD-DWARF-CHECK:   DW_AT_name {{.*}}"arr8"
+#UPD-DWARF-CHECK:   DW_AT_location [DW_FORM_exprloc]        (DW_OP_addrx 0x7)
+#UPD-DWARF-CHECK: DW_TAG_variable
+#UPD-DWARF-CHECK:   DW_AT_name {{.*}}"arr9"
+#UPD-DWARF-CHECK:   DW_AT_location [DW_FORM_exprloc]        (DW_OP_addrx 0x8)
 #UPD-DWARF-CHECK: .debug_addr contents:
-#UPD-DWARF-CHECK: 0x00000000: Address table header: length = 0x00000014, format = DWARF32, version = 0x0005, addr_size = 0x08, seg_size = 0x00
+#UPD-DWARF-CHECK: 0x00000000: Address table header: length = 0x00000054, format = DWARF32, version = 0x0005, addr_size = 0x08, seg_size = 0x00
+#UPD-DWARF-CHECK: 0x0000000000000000
+#UPD-DWARF-CHECK: 0x0000000000000000
+#UPD-DWARF-CHECK: 0x0000000000000000
+#UPD-DWARF-CHECK: 0x0000000000000000
+#UPD-DWARF-CHECK: 0x0000000000000000
+#UPD-DWARF-CHECK: 0x0000000000000000
+#UPD-DWARF-CHECK: 0x0000000000000000
+#UPD-DWARF-CHECK: 0x0000000000000000
 #UPD-DWARF-CHECK: 0x0000000000000000
 #UPD-DWARF-CHECK: 0x0000000000000000
 
@@ -44,4 +111,12 @@ objects:
     timestamp:       1676048242
     symbols:
       - { sym: _main, objAddr: 0x0000000000000000, binAddr: 0x0000000100000FB0, size: 0x00000008 }
-      - { sym: _arr, binAddr: 0x0000000100002000, size: 0x00000000 }
+      - { sym: _arr, binAddr: 0x0000000100001000, size: 0x00000008 }
+      - { sym: _arr2, binAddr: 0x0000000100002000, size: 0x00000008 }
+      - { sym: _arr3, binAddr: 0x0000000100003000, size: 0x00000008 }
+      - { sym: _arr4, binAddr: 0x0000000100004000, size: 0x00000008 }
+      - { sym: _arr5, binAddr: 0x0000000100005000, size: 0x00000008 }
+      - { sym: _arr6, binAddr: 0x0000000100006000, size: 0x00000008 }
+      - { sym: _arr7, binAddr: 0x0000000100007000, size: 0x00000008 }
+      - { sym: _arr8, binAddr: 0x0000000100008000, size: 0x00000008 }
+      - { sym: _arr9, binAddr: 0x0000000100009000, size: 0x00000008 }
diff --git a/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp b/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp
index 09a69a35a23ca..4b281d5f19801 100644
--- a/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp
+++ b/llvm/tools/dsymutil/DwarfLinkerForBinary.cpp
@@ -1079,16 +1079,12 @@ std::optional<int64_t> DwarfLinkerForBinary::AddressManager<
   case dwarf::DW_FORM_addrx3:
   case dwarf::DW_FORM_addrx4: {
     std::optional<DWARFFormValue> AddrValue = DIE.find(dwarf::DW_AT_low_pc);
-    if (std::optional<uint64_t> AddrOffsetSectionBase =
-            DIE.getDwarfUnit()->getAddrOffsetSectionBase()) {
-      // Addrx is a index into the debug_addr section, not an offset, so we need
-      // to multiply by byte size.
-      const uint64_t AddrSize = DIE.getDwarfUnit()->getAddressByteSize();
-      const uint64_t StartOffset =
-          *AddrOffsetSectionBase + (AddrValue->getRawUValue() * AddrSize);
-      const uint64_t EndOffset = StartOffset + AddrSize;
-      return hasValidRelocationAt(ValidDebugAddrRelocs, StartOffset, EndOffset);
-    }
+    if (std::optional<uint64_t> AddressOffset =
+            DIE.getDwarfUnit()->getIndexedAddressOffset(
+                AddrValue->getRawUValue()))
+      return hasValidRelocationAt(ValidDebugAddrRelocs, *AddressOffset,
+                                  *AddressOffset +
+                                      DIE.getDwarfUnit()->getAddressByteSize());
 
     Linker.reportWarning("no base offset for address table", SrcFileName);
     return std::nullopt;

From 6bea8331f9e09ba94a225c65becd4224a1a473af Mon Sep 17 00:00:00 2001
From: Vladislav Dzhidzhoev <vdzhidzhoev@accesssoftek.com>
Date: Tue, 20 Jun 2023 13:08:47 +0200
Subject: [PATCH 116/130] Revert "Reland "[DebugMetadata][DwarfDebug] Support
 function-local types in lexical block scopes (4/7)" (2)"

This reverts commit cb9ac7051589ea0d05507f9370d0716bef86b4ae.
It causes an assert in clang:
virtual void llvm::DwarfDebug::endFunctionImpl(const llvm::MachineFunction*): Assertion `LScopes.getAbstractScopesList().size() == NumAbstractSubprograms && "getOrCreateAbstractScope() inserted an abstract subprogram scope"' failed.
https://bugs.chromium.org/p/chromium/issues/detail?id=1456288#c2
---
 .../CodeGen/debug-info-codeview-unnamed.c     |  16 +-
 clang/test/CodeGen/debug-info-unused-types.c  |  14 +-
 .../test/CodeGen/debug-info-unused-types.cpp  |  12 +-
 clang/test/CodeGenCXX/debug-info-access.cpp   |   2 +-
 .../CodeGenCXX/debug-info-anon-union-vars.cpp |  12 +-
 .../debug-info-codeview-unnamed.cpp           | 110 ++---
 .../debug-info-gline-tables-only-codeview.cpp |   4 +-
 clang/test/CodeGenCXX/debug-lambda-this.cpp   |   4 +-
 llvm/include/llvm/IR/DIBuilder.h              |   6 +-
 llvm/lib/Bitcode/Reader/MetadataLoader.cpp    |  87 ++--
 .../CodeGen/AsmPrinter/DwarfCompileUnit.cpp   |  60 +--
 .../lib/CodeGen/AsmPrinter/DwarfCompileUnit.h |  16 +-
 llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp    |  13 +-
 llvm/lib/IR/DIBuilder.cpp                     |  34 +-
 llvm/lib/IR/Verifier.cpp                      |   6 +-
 llvm/test/Bitcode/upgrade-cu-locals.ll        |  68 ++-
 llvm/test/Bitcode/upgrade-cu-locals.ll.bc     | Bin 2780 -> 2688 bytes
 .../DebugInfo/Generic/inlined-local-type.ll   | 128 ------
 .../Generic/lexical-block-retained-types.ll   |  55 ---
 .../DebugInfo/Generic/lexical-block-types.ll  | 425 ------------------
 .../split-dwarf-local-import.ll               |   1 +
 .../split-dwarf-local-import2.ll              |   1 +
 .../split-dwarf-local-import3.ll              |   0
 .../Generic/verifier-invalid-disubprogram.ll  |   2 +-
 .../X86/local-type-as-template-parameter.ll   | 161 -------
 llvm/test/DebugInfo/X86/set.ll                |   4 +-
 26 files changed, 189 insertions(+), 1052 deletions(-)
 delete mode 100644 llvm/test/DebugInfo/Generic/inlined-local-type.ll
 delete mode 100644 llvm/test/DebugInfo/Generic/lexical-block-retained-types.ll
 delete mode 100644 llvm/test/DebugInfo/Generic/lexical-block-types.ll
 rename llvm/test/DebugInfo/{X86 => Generic}/split-dwarf-local-import.ll (98%)
 rename llvm/test/DebugInfo/{X86 => Generic}/split-dwarf-local-import2.ll (98%)
 rename llvm/test/DebugInfo/{X86 => Generic}/split-dwarf-local-import3.ll (100%)
 delete mode 100644 llvm/test/DebugInfo/X86/local-type-as-template-parameter.ll

diff --git a/clang/test/CodeGen/debug-info-codeview-unnamed.c b/clang/test/CodeGen/debug-info-codeview-unnamed.c
index 16ffb3682236f..bd2a7543e56b2 100644
--- a/clang/test/CodeGen/debug-info-codeview-unnamed.c
+++ b/clang/test/CodeGen/debug-info-codeview-unnamed.c
@@ -8,23 +8,23 @@ int main(int argc, char* argv[], char* arge[]) {
   //
   struct { int bar; } one = {42};
   //
-  // LINUX:      [[TYPE_OF_ONE:![0-9]+]] = distinct !DICompositeType(
+  // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "one"
+  // LINUX-SAME:     type: [[TYPE_OF_ONE:![0-9]+]]
+  // LINUX-SAME:     )
+  // LINUX:      [[TYPE_OF_ONE]] = distinct !DICompositeType(
   // LINUX-SAME:     tag: DW_TAG_structure_type
   // LINUX-NOT:      name:
   // LINUX-NOT:      identifier:
   // LINUX-SAME:     )
-  // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "one"
-  // LINUX-SAME:     type: [[TYPE_OF_ONE]]
-  // LINUX-SAME:     )
   //
-  // MSVC:       [[TYPE_OF_ONE:![0-9]+]] = distinct !DICompositeType
+  // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "one"
+  // MSVC-SAME:      type: [[TYPE_OF_ONE:![0-9]+]]
+  // MSVC-SAME:      )
+  // MSVC:       [[TYPE_OF_ONE]] = distinct !DICompositeType
   // MSVC-SAME:      tag: DW_TAG_structure_type
   // MSVC-NOT:       name:
   // MSVC-NOT:       identifier:
   // MSVC-SAME:      )
-  // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "one"
-  // MSVC-SAME:      type: [[TYPE_OF_ONE]]
-  // MSVC-SAME:      )
 
   return 0;
 }
diff --git a/clang/test/CodeGen/debug-info-unused-types.c b/clang/test/CodeGen/debug-info-unused-types.c
index 5bcc71f710a5c..3e9f7b07658e3 100644
--- a/clang/test/CodeGen/debug-info-unused-types.c
+++ b/clang/test/CodeGen/debug-info-unused-types.c
@@ -18,13 +18,13 @@ void quux(void) {
 // CHECK: !DICompileUnit{{.+}}retainedTypes: [[RETTYPES:![0-9]+]]
 // CHECK: [[TYPE0:![0-9]+]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "bar"
 // CHECK: [[TYPE1:![0-9]+]] = !DIEnumerator(name: "BAR"
-// CHECK: [[RETTYPES]] = !{[[TYPE2:![0-9]+]], [[TYPE3:![0-9]+]], [[TYPE0]], [[TYPE4:![0-9]+]], {{![0-9]+}}, [[TYPE5:![0-9]+]], [[TYPE6:![0-9]+]], [[TYPE8:![0-9]+]]}
-// CHECK: [[TYPE2]] = !DIDerivedType(tag: DW_TAG_typedef, name: "my_int"
-// CHECK: [[TYPE3]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "foo"
-// CHECK: [[TYPE4]] = distinct !DICompositeType(tag: DW_TAG_union_type, name: "baz"
-// CHECK: [[TYPE5]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "y"
-// CHECK: [[TYPE6]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "z"
-// CHECK: [[TYPE7:![0-9]+]] = !DIEnumerator(name: "Z"
+// CHECK: [[TYPE2:![0-9]+]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "z"
+// CHECK: [[TYPE3:![0-9]+]] = !DIEnumerator(name: "Z"
+// CHECK: [[RETTYPES]] = !{[[TYPE4:![0-9]+]], [[TYPE5:![0-9]+]], [[TYPE0]], [[TYPE6:![0-9]+]], {{![0-9]+}}, [[TYPE7:![0-9]+]], [[TYPE2]], [[TYPE8:![0-9]+]]}
+// CHECK: [[TYPE4]] = !DIDerivedType(tag: DW_TAG_typedef, name: "my_int"
+// CHECK: [[TYPE5]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "foo"
+// CHECK: [[TYPE6]] = distinct !DICompositeType(tag: DW_TAG_union_type, name: "baz"
+// CHECK: [[TYPE7]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "y"
 // CHECK: [[TYPE8]] = distinct !DICompositeType(tag: DW_TAG_union_type, name: "w"
 
 // Check that debug info is not emitted for the typedef, struct, enum, and
diff --git a/clang/test/CodeGen/debug-info-unused-types.cpp b/clang/test/CodeGen/debug-info-unused-types.cpp
index 715fada47eab8..023cac159faa4 100644
--- a/clang/test/CodeGen/debug-info-unused-types.cpp
+++ b/clang/test/CodeGen/debug-info-unused-types.cpp
@@ -13,12 +13,12 @@ void quux() {
 // CHECK: !DICompileUnit{{.+}}retainedTypes: [[RETTYPES:![0-9]+]]
 // CHECK: [[TYPE0:![0-9]+]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "baz"
 // CHECK: [[TYPE1:![0-9]+]] = !DIEnumerator(name: "BAZ"
-// CHECK: [[RETTYPES]] = !{[[TYPE2:![0-9]+]], [[TYPE3:![0-9]+]], [[TYPE0]], {{![0-9]+}}, [[TYPE4:![0-9]+]], [[TYPE5:![0-9]+]]}
-// CHECK: [[TYPE2]] = !DIDerivedType(tag: DW_TAG_typedef, name: "foo"
-// CHECK: [[TYPE3]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "bar"
-// CHECK: [[TYPE4]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "y"
-// CHECK: [[TYPE5]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "z"
-// CHECK: [[TYPE6:![0-9]+]] = !DIEnumerator(name: "Z"
+// CHECK: [[TYPE2:![0-9]+]] = !DICompositeType(tag: DW_TAG_enumeration_type, name: "z"
+// CHECK: [[TYPE3:![0-9]+]] = !DIEnumerator(name: "Z"
+// CHECK: [[RETTYPES]] = !{[[TYPE4:![0-9]+]], [[TYPE5:![0-9]+]], [[TYPE0]], {{![0-9]+}}, [[TYPE6:![0-9]+]], [[TYPE2]]}
+// CHECK: [[TYPE4]] = !DIDerivedType(tag: DW_TAG_typedef, name: "foo"
+// CHECK: [[TYPE5]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "bar"
+// CHECK: [[TYPE6]] = distinct !DICompositeType(tag: DW_TAG_class_type, name: "y"
 
 // NODBG-NOT: !DI{{CompositeType|Enumerator|DerivedType}}
 
diff --git a/clang/test/CodeGenCXX/debug-info-access.cpp b/clang/test/CodeGenCXX/debug-info-access.cpp
index 7c0bf8ccb0384..9f2c044843d0f 100644
--- a/clang/test/CodeGenCXX/debug-info-access.cpp
+++ b/clang/test/CodeGenCXX/debug-info-access.cpp
@@ -18,9 +18,9 @@ class B : public A {
   static int public_static;
 
 protected:
-  // CHECK-DAG: !DIDerivedType(tag: DW_TAG_typedef, name: "prot_using",{{.*}} line: [[@LINE+3]],{{.*}} flags: DIFlagProtected)
   // CHECK-DAG: !DIDerivedType(tag: DW_TAG_typedef, name: "prot_typedef",{{.*}} line: [[@LINE+1]],{{.*}} flags: DIFlagProtected)
   typedef int prot_typedef;
+  // CHECK-DAG: !DIDerivedType(tag: DW_TAG_typedef, name: "prot_using",{{.*}} line: [[@LINE+1]],{{.*}} flags: DIFlagProtected)
   using prot_using = prot_typedef;
   prot_using prot_member;
 
diff --git a/clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp b/clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp
index c91cf83c0405f..61b3c7c0526c8 100644
--- a/clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp
+++ b/clang/test/CodeGenCXX/debug-info-anon-union-vars.cpp
@@ -51,13 +51,13 @@ void instantiate(int x) {
 // CHECK: !DIGlobalVariable(name: "b",{{.*}} file: [[FILE]], line: 6,{{.*}} isLocal: true, isDefinition: true
 // CHECK: !DIGlobalVariable(name: "result", {{.*}} isLocal: false, isDefinition: true
 // CHECK: !DIGlobalVariable(name: "value", {{.*}} isLocal: false, isDefinition: true
-// CHECK: ![[UNION:[0-9]+]] = distinct !DICompositeType(tag: DW_TAG_union_type,
-// CHECK-NOT: name:
-// CHECK: elements
-// CHECK: !DIDerivedType(tag: DW_TAG_member, name: "i", scope: ![[UNION]],
-// CHECK: !DIDerivedType(tag: DW_TAG_member, name: "c", scope: ![[UNION]],
 // CHECK: !DILocalVariable(name: "i", {{.*}}, flags: DIFlagArtificial
 // CHECK: !DILocalVariable(name: "c", {{.*}}, flags: DIFlagArtificial
 // CHECK: !DILocalVariable(
 // CHECK-NOT: name:
-// CHECK: type: ![[UNION]]
+// CHECK: type: ![[UNION:[0-9]+]]
+// CHECK: ![[UNION]] = distinct !DICompositeType(tag: DW_TAG_union_type,
+// CHECK-NOT: name:
+// CHECK: elements
+// CHECK: !DIDerivedType(tag: DW_TAG_member, name: "i", scope: ![[UNION]],
+// CHECK: !DIDerivedType(tag: DW_TAG_member, name: "c", scope: ![[UNION]],
diff --git a/clang/test/CodeGenCXX/debug-info-codeview-unnamed.cpp b/clang/test/CodeGenCXX/debug-info-codeview-unnamed.cpp
index 9602ac1b02497..b4c79936ab33e 100644
--- a/clang/test/CodeGenCXX/debug-info-codeview-unnamed.cpp
+++ b/clang/test/CodeGenCXX/debug-info-codeview-unnamed.cpp
@@ -3,60 +3,6 @@
 
 int main(int argc, char* argv[], char* arge[]) {
   //
-  // LINUX:      [[TYPE_OF_ONE:![0-9]+]] = distinct !DICompositeType(
-  // LINUX-SAME:     tag: DW_TAG_structure_type
-  // LINUX-NOT:      name:
-  // LINUX-NOT:      identifier:
-  // LINUX-SAME:     )
-  //
-  // MSVC:       [[TYPE_OF_ONE:![0-9]+]] = distinct !DICompositeType
-  // MSVC-SAME:      tag: DW_TAG_structure_type
-  // MSVC-SAME:      name: "<unnamed-type-one>"
-  // MSVC-SAME:      identifier: ".?AU<unnamed-type-one>@?1??main@@9@"
-  // MSVC-SAME:      )
-
-
-  //
-  // LINUX:      [[TYPE_OF_TWO:![0-9]+]] = distinct !DICompositeType(
-  // LINUX-SAME:     tag: DW_TAG_structure_type
-  // LINUX-NOT:      name:
-  // LINUX-NOT:      identifier:
-  // LINUX-SAME:     )
-  //
-  // MSVC:       [[TYPE_OF_TWO:![0-9]+]] = distinct !DICompositeType
-  // MSVC-SAME:      tag: DW_TAG_structure_type
-  // MSVC-SAME:      name: "<unnamed-type-two>"
-  // MSVC-SAME:      identifier: ".?AU<unnamed-type-two>@?2??main@@9@"
-  // MSVC-SAME:      )
-
-
-  //
-  // LINUX:      [[TYPE_OF_THREE:![0-9]+]] = distinct !DICompositeType(
-  // LINUX-SAME:     tag: DW_TAG_structure_type
-  // LINUX-SAME:     name: "named"
-  // LINUX-NOT:      identifier:
-  // LINUX-SAME:     )
-  //
-  // MSVC:       [[TYPE_OF_THREE:![0-9]+]] = distinct !DICompositeType
-  // MSVC-SAME:      tag: DW_TAG_structure_type
-  // MSVC-SAME:      name: "named"
-  // MSVC-SAME:      identifier: ".?AUnamed@?1??main@@9@"
-  // MSVC-SAME:      )
-
-  //
-  // LINUX:      [[TYPE_OF_FOUR:![0-9]+]] = distinct !DICompositeType(
-  // LINUX-SAME:     tag: DW_TAG_class_type
-  // LINUX-NOT:      name:
-  // LINUX-NOT:      identifier:
-  // LINUX-SAME:     )
-  //
-  // MSVC:       [[TYPE_OF_FOUR:![0-9]+]] = distinct !DICompositeType
-  // MSVC-SAME:      tag: DW_TAG_class_type
-  // MSVC-SAME:      name: "<lambda_0>"
-  // MSVC-SAME:      identifier: ".?AV<lambda_0>@?0??main@@9@"
-  // MSVC-SAME:      )
-
-
   // In CodeView, the LF_MFUNCTION entry for "bar()" refers to the forward
   // reference of the unnamed struct. Visual Studio requires a unique
   // identifier to match the LF_STRUCTURE forward reference to the definition.
@@ -64,11 +10,21 @@ int main(int argc, char* argv[], char* arge[]) {
   struct { void bar() {} } one;
   //
   // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "one"
-  // LINUX-SAME:     type: [[TYPE_OF_ONE]]
+  // LINUX-SAME:     type: [[TYPE_OF_ONE:![0-9]+]]
+  // LINUX-SAME:     )
+  // LINUX:      [[TYPE_OF_ONE]] = distinct !DICompositeType(
+  // LINUX-SAME:     tag: DW_TAG_structure_type
+  // LINUX-NOT:      name:
+  // LINUX-NOT:      identifier:
   // LINUX-SAME:     )
   //
   // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "one"
-  // MSVC-SAME:      type: [[TYPE_OF_ONE]]
+  // MSVC-SAME:      type: [[TYPE_OF_ONE:![0-9]+]]
+  // MSVC-SAME:      )
+  // MSVC:       [[TYPE_OF_ONE]] = distinct !DICompositeType
+  // MSVC-SAME:      tag: DW_TAG_structure_type
+  // MSVC-SAME:      name: "<unnamed-type-one>"
+  // MSVC-SAME:      identifier: ".?AU<unnamed-type-one>@?1??main@@9@"
   // MSVC-SAME:      )
 
 
@@ -80,11 +36,21 @@ int main(int argc, char* argv[], char* arge[]) {
   int decltype(two)::*ptr2unnamed = &decltype(two)::bar;
   //
   // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "two"
-  // LINUX-SAME:     type: [[TYPE_OF_TWO]]
+  // LINUX-SAME:     type: [[TYPE_OF_TWO:![0-9]+]]
+  // LINUX-SAME:     )
+  // LINUX:      [[TYPE_OF_TWO]] = distinct !DICompositeType(
+  // LINUX-SAME:     tag: DW_TAG_structure_type
+  // LINUX-NOT:      name:
+  // LINUX-NOT:      identifier:
   // LINUX-SAME:     )
   //
   // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "two"
-  // MSVC-SAME:      type: [[TYPE_OF_TWO]]
+  // MSVC-SAME:      type: [[TYPE_OF_TWO:![0-9]+]]
+  // MSVC-SAME:      )
+  // MSVC:       [[TYPE_OF_TWO]] = distinct !DICompositeType
+  // MSVC-SAME:      tag: DW_TAG_structure_type
+  // MSVC-SAME:      name: "<unnamed-type-two>"
+  // MSVC-SAME:      identifier: ".?AU<unnamed-type-two>@?2??main@@9@"
   // MSVC-SAME:      )
 
 
@@ -95,11 +61,21 @@ int main(int argc, char* argv[], char* arge[]) {
   struct named { int bar; int named::* p2mem; } three = { 42, &named::bar };
   //
   // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "three"
-  // LINUX-SAME:     type: [[TYPE_OF_THREE]]
+  // LINUX-SAME:     type: [[TYPE_OF_THREE:![0-9]+]]
+  // LINUX-SAME:     )
+  // LINUX:      [[TYPE_OF_THREE]] = distinct !DICompositeType(
+  // LINUX-SAME:     tag: DW_TAG_structure_type
+  // LINUX-SAME:     name: "named"
+  // LINUX-NOT:      identifier:
   // LINUX-SAME:     )
   //
   // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "three"
-  // MSVC-SAME:      type: [[TYPE_OF_THREE]]
+  // MSVC-SAME:      type: [[TYPE_OF_THREE:![0-9]+]]
+  // MSVC-SAME:      )
+  // MSVC:       [[TYPE_OF_THREE]] = distinct !DICompositeType
+  // MSVC-SAME:      tag: DW_TAG_structure_type
+  // MSVC-SAME:      name: "named"
+  // MSVC-SAME:      identifier: ".?AUnamed@?1??main@@9@"
   // MSVC-SAME:      )
 
 
@@ -111,11 +87,21 @@ int main(int argc, char* argv[], char* arge[]) {
   auto four = [argc](int i) -> int { return argc == i ? 1 : 0; };
   //
   // LINUX:      !{{[0-9]+}} = !DILocalVariable(name: "four"
-  // LINUX-SAME:     type: [[TYPE_OF_FOUR]]
+  // LINUX-SAME:     type: [[TYPE_OF_FOUR:![0-9]+]]
+  // LINUX-SAME:     )
+  // LINUX:      [[TYPE_OF_FOUR]] = distinct !DICompositeType(
+  // LINUX-SAME:     tag: DW_TAG_class_type
+  // LINUX-NOT:      name:
+  // LINUX-NOT:      identifier:
   // LINUX-SAME:     )
   //
   // MSVC:       !{{[0-9]+}} = !DILocalVariable(name: "four"
-  // MSVC-SAME:      type: [[TYPE_OF_FOUR]]
+  // MSVC-SAME:      type: [[TYPE_OF_FOUR:![0-9]+]]
+  // MSVC-SAME:      )
+  // MSVC:       [[TYPE_OF_FOUR]] = distinct !DICompositeType
+  // MSVC-SAME:      tag: DW_TAG_class_type
+  // MSVC-SAME:      name: "<lambda_0>"
+  // MSVC-SAME:      identifier: ".?AV<lambda_0>@?0??main@@9@"
   // MSVC-SAME:      )
 
   return 0;
diff --git a/clang/test/CodeGenCXX/debug-info-gline-tables-only-codeview.cpp b/clang/test/CodeGenCXX/debug-info-gline-tables-only-codeview.cpp
index 122e4aa62ea7d..6b9c9a243decd 100644
--- a/clang/test/CodeGenCXX/debug-info-gline-tables-only-codeview.cpp
+++ b/clang/test/CodeGenCXX/debug-info-gline-tables-only-codeview.cpp
@@ -51,9 +51,9 @@ void test() {
   // CHECK-SAME:                              name: "<lambda_2_1>",
   c.lambda_params();
 
-  // CHECK: ![[LAMBDA1:[0-9]+]] = !DICompositeType(tag: DW_TAG_class_type,
+  // CHECK: !DISubprogram(name: "operator()", scope: ![[LAMBDA1:[0-9]+]],
+  // CHECK: ![[LAMBDA1]] = !DICompositeType(tag: DW_TAG_class_type,
   // CHECK-SAME:                            name: "<lambda_1>",
   // CHECK-SAME:                            flags: DIFlagFwdDecl
-  // CHECK: !DISubprogram(name: "operator()", scope: ![[LAMBDA1]],
   c.lambda2();
 }
diff --git a/clang/test/CodeGenCXX/debug-lambda-this.cpp b/clang/test/CodeGenCXX/debug-lambda-this.cpp
index 3d659e7bfd004..eecbac6520ac9 100644
--- a/clang/test/CodeGenCXX/debug-lambda-this.cpp
+++ b/clang/test/CodeGenCXX/debug-lambda-this.cpp
@@ -13,10 +13,10 @@ int D::d(int x) {
 }
 
 // CHECK: ![[D:[0-9]+]] = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "D",
+// CHECK: ![[POINTER:.*]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: ![[D]], size: 64)
 // CHECK: !DIDerivedType(tag: DW_TAG_member, name: "this",
 // CHECK-SAME:           line: 11
-// CHECK-SAME:           baseType: ![[POINTER:[0-9]+]]
+// CHECK-SAME:           baseType: ![[POINTER]]
 // CHECK-SAME:           size: 64
 // CHECK-NOT:            offset: 0
 // CHECK-SAME:           ){{$}}
-// CHECK: ![[POINTER]] = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: ![[D]], size: 64)
diff --git a/llvm/include/llvm/IR/DIBuilder.h b/llvm/include/llvm/IR/DIBuilder.h
index 0aa180aec6d8a..ecd6dd7b0a4f8 100644
--- a/llvm/include/llvm/IR/DIBuilder.h
+++ b/llvm/include/llvm/IR/DIBuilder.h
@@ -49,7 +49,7 @@ namespace llvm {
     Function *LabelFn;       ///< llvm.dbg.label
     Function *AssignFn;      ///< llvm.dbg.assign
 
-    SmallVector<TrackingMDNodeRef, 4> EnumTypes;
+    SmallVector<TrackingMDNodeRef, 4> AllEnumTypes;
     /// Track the RetainTypes, since they can be updated later on.
     SmallVector<TrackingMDNodeRef, 4> AllRetainTypes;
     SmallVector<DISubprogram *, 4> AllSubprograms;
@@ -64,8 +64,8 @@ namespace llvm {
     SmallVector<TrackingMDNodeRef, 4> UnresolvedNodes;
     bool AllowUnresolvedNodes;
 
-    /// Each subprogram's preserved local variables, labels, imported entities,
-    /// and types.
+    /// Each subprogram's preserved local variables, labels and imported
+    /// entities.
     ///
     /// Do not use a std::vector.  Some versions of libc++ apparently copy
     /// instead of move on grow operations, and TrackingMDRef is expensive to
diff --git a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
index 321eae6fcb3b5..0a9a80688a419 100644
--- a/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
+++ b/llvm/lib/Bitcode/Reader/MetadataLoader.cpp
@@ -548,8 +548,6 @@ class MetadataLoader::MetadataLoaderImpl {
 
   /// Move local imports from DICompileUnit's 'imports' field to
   /// DISubprogram's retainedNodes.
-  /// Move fucntion-local enums from DICompileUnit's enums
-  /// to DISubprogram's retainedNodes.
   void upgradeCULocals() {
     if (NamedMDNode *CUNodes = TheModule.getNamedMetadata("llvm.dbg.cu")) {
       for (unsigned I = 0, E = CUNodes->getNumOperands(); I != E; ++I) {
@@ -557,71 +555,48 @@ class MetadataLoader::MetadataLoaderImpl {
         if (!CU)
           continue;
 
-        SetVector<Metadata *> MetadataToRemove;
-        // Collect imported entities to be moved.
-        if (CU->getRawImportedEntities()) {
+        if (auto *RawImported = CU->getRawImportedEntities()) {
+          // Collect a set of imported entities to be moved.
+          SetVector<Metadata *> EntitiesToRemove;
           for (Metadata *Op : CU->getImportedEntities()->operands()) {
             auto *IE = cast<DIImportedEntity>(Op);
-            if (dyn_cast_or_null<DILocalScope>(IE->getScope())) {
-              MetadataToRemove.insert(IE);
+            if (auto *S = dyn_cast_or_null<DILocalScope>(IE->getScope())) {
+              EntitiesToRemove.insert(IE);
             }
           }
-        }
-        // Collect enums to be moved.
-        if (CU->getRawEnumTypes()) {
-          for (Metadata *Op : CU->getEnumTypes()->operands()) {
-            auto *Enum = cast<DICompositeType>(Op);
-            if (dyn_cast_or_null<DILocalScope>(Enum->getScope())) {
-              MetadataToRemove.insert(Enum);
-            }
-          }
-        }
 
-        if (!MetadataToRemove.empty()) {
-          // Make a new list of CU's 'imports'.
-          SmallVector<Metadata *> NewImports;
-          if (CU->getRawImportedEntities())
-            for (Metadata *Op : CU->getImportedEntities()->operands())
-              if (!MetadataToRemove.contains(Op))
+          if (!EntitiesToRemove.empty()) {
+            // Make a new list of CU's 'imports'.
+            SmallVector<Metadata *> NewImports;
+            for (Metadata *Op : CU->getImportedEntities()->operands()) {
+              if (!EntitiesToRemove.contains(cast<DIImportedEntity>(Op))) {
                 NewImports.push_back(Op);
+              }
+            }
 
-          // Make a new list of CU's 'enums'.
-          SmallVector<Metadata *> NewEnums;
-          if (CU->getRawEnumTypes())
-            for (Metadata *Op : CU->getEnumTypes()->operands())
-              if (!MetadataToRemove.contains(Op))
-                NewEnums.push_back(Op);
-
-          // Find DISubprogram corresponding to each entity.
-          std::map<DISubprogram *, SmallVector<Metadata *>> SPToEntities;
-          for (auto *I : MetadataToRemove) {
-            DILocalScope *Scope = nullptr;
-            if (auto *Entity = dyn_cast<DIImportedEntity>(I))
-              Scope = cast<DILocalScope>(Entity->getScope());
-            else if (auto *Enum = dyn_cast<DICompositeType>(I))
-              Scope = cast<DILocalScope>(Enum->getScope());
-
-            if (auto *SP = findEnclosingSubprogram(Scope)) {
-              SPToEntities[SP].push_back(I);
+            // Find DISubprogram corresponding to each entity.
+            std::map<DISubprogram *, SmallVector<Metadata *>> SPToEntities;
+            for (auto *I : EntitiesToRemove) {
+              auto *Entity = cast<DIImportedEntity>(I);
+              if (auto *SP = findEnclosingSubprogram(
+                      cast<DILocalScope>(Entity->getScope()))) {
+                SPToEntities[SP].push_back(Entity);
+              }
             }
-          }
 
-          // Update DISubprograms' retainedNodes.
-          for (auto I = SPToEntities.begin(); I != SPToEntities.end(); ++I) {
-            auto *SP = I->first;
-            auto RetainedNodes = SP->getRetainedNodes();
-            SmallVector<Metadata *> MDs(RetainedNodes.begin(),
-                                        RetainedNodes.end());
-            MDs.append(I->second);
-            SP->replaceRetainedNodes(MDNode::get(Context, MDs));
-          }
+            // Update DISubprograms' retainedNodes.
+            for (auto I = SPToEntities.begin(); I != SPToEntities.end(); ++I) {
+              auto *SP = I->first;
+              auto RetainedNodes = SP->getRetainedNodes();
+              SmallVector<Metadata *> MDs(RetainedNodes.begin(),
+                                          RetainedNodes.end());
+              MDs.append(I->second);
+              SP->replaceRetainedNodes(MDNode::get(Context, MDs));
+            }
 
-          // Remove entities with local scope from CU.
-          if (CU->getRawImportedEntities())
+            // Remove entities with local scope from CU.
             CU->replaceImportedEntities(MDTuple::get(Context, NewImports));
-          // Remove enums with local scope from CU.
-          if (CU->getRawEnumTypes())
-            CU->replaceEnumTypes(MDTuple::get(Context, NewEnums));
+          }
         }
       }
     }
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
index bf8bdbb7da49f..58ed21379d29b 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
@@ -598,9 +598,10 @@ void DwarfCompileUnit::constructScopeDIE(LexicalScope *Scope,
     return;
 
   // Emit lexical blocks.
-  DIE *ScopeDIE = getOrCreateLexicalBlockDIE(Scope, ParentScopeDIE);
+  DIE *ScopeDIE = constructLexicalScopeDIE(Scope);
   assert(ScopeDIE && "Scope DIE should not be null.");
 
+  ParentScopeDIE.addChild(ScopeDIE);
   createAndAddScopeChildren(Scope, *ScopeDIE);
 }
 
@@ -720,39 +721,24 @@ DIE *DwarfCompileUnit::constructInlinedScopeDIE(LexicalScope *Scope,
   return ScopeDIE;
 }
 
-DIE *DwarfCompileUnit::getOrCreateLexicalBlockDIE(LexicalScope *Scope,
-                                                  DIE &ParentScopeDIE) {
+// Construct new DW_TAG_lexical_block for this scope and attach
+// DW_AT_low_pc/DW_AT_high_pc labels.
+DIE *DwarfCompileUnit::constructLexicalScopeDIE(LexicalScope *Scope) {
   if (DD->isLexicalScopeDIENull(Scope))
     return nullptr;
   const auto *DS = Scope->getScopeNode();
-  DIE *ScopeDIE = nullptr;
-
-  // FIXME: We may have a concrete DIE for this scope already created.
-  // This may happen when we emit local variables for an abstract tree of
-  // an inlined function: if a local variable has a templated type with
-  // a function-local type as a template parameter. See PR55680 for details
-  // (see also llvm/test/DebugInfo/Generic/local-type-as-template-parameter.ll).
-  if (!Scope->isAbstractScope() && !Scope->getInlinedAt()) {
-    if (auto It = LexicalBlockDIEs.find(DS); It != LexicalBlockDIEs.end()) {
-      ScopeDIE = It->second;
-      assert(!ScopeDIE->findAttribute(dwarf::DW_AT_low_pc) &&
-             !ScopeDIE->findAttribute(dwarf::DW_AT_ranges));
-      assert(ScopeDIE->getParent() == &ParentScopeDIE);
-    }
-  }
-  if (!ScopeDIE) {
-    ScopeDIE = DIE::get(DIEValueAllocator, dwarf::DW_TAG_lexical_block);
-    ParentScopeDIE.addChild(ScopeDIE);
-
-    if (Scope->isAbstractScope()) {
-      assert(!getAbstractScopeDIEs().count(DS) &&
-             "Abstract DIE for this scope exists!");
-      getAbstractScopeDIEs()[DS] = ScopeDIE;
-      return ScopeDIE;
-    }
 
-    if (!Scope->getInlinedAt())
-      LexicalBlockDIEs[DS] = ScopeDIE;
+  auto ScopeDIE = DIE::get(DIEValueAllocator, dwarf::DW_TAG_lexical_block);
+  if (Scope->isAbstractScope()) {
+    assert(!getAbstractScopeDIEs().count(DS) &&
+           "Abstract DIE for this scope exists!");
+    getAbstractScopeDIEs()[DS] = ScopeDIE;
+    return ScopeDIE;
+  }
+  if (!Scope->getInlinedAt()) {
+    assert(!LexicalBlockDIEs.count(DS) &&
+           "Concrete out-of-line DIE for this scope exists!");
+    LexicalBlockDIEs[DS] = ScopeDIE;
   }
 
   attachRangesOrLowHighPC(*ScopeDIE, Scope->getRanges());
@@ -1695,21 +1681,15 @@ void DwarfCompileUnit::createBaseTypeDIEs() {
   }
 }
 
-DIE *DwarfCompileUnit::getLocalContextDIE(const DILexicalBlock *LB) {
+DIE *DwarfCompileUnit::getLexicalBlockDIE(const DILexicalBlock *LB) {
   // Assume if there is an abstract tree all the DIEs are already emitted.
   bool isAbstract = getAbstractScopeDIEs().count(LB->getSubprogram());
   if (isAbstract && getAbstractScopeDIEs().count(LB))
     return getAbstractScopeDIEs()[LB];
   assert(!isAbstract && "Missed lexical block DIE in abstract tree!");
 
-  // Check if we have a concrete DIE.
-  if (auto It = LexicalBlockDIEs.find(LB); It != LexicalBlockDIEs.end())
-    return It->second;
-
-  // If nothing available found, we cannot just create a new lexical block,
-  // because it isn't known where to put it into the DIE tree.
-  // So, we may only try to find the most close avaiable parent DIE.
-  return getOrCreateContextDIE(LB->getScope()->getNonLexicalBlockFileScope());
+  // Return a concrete DIE if it exists or nullptr otherwise.
+  return LexicalBlockDIEs.lookup(LB);
 }
 
 DIE *DwarfCompileUnit::getOrCreateContextDIE(const DIScope *Context) {
@@ -1717,7 +1697,7 @@ DIE *DwarfCompileUnit::getOrCreateContextDIE(const DIScope *Context) {
     if (auto *LFScope = dyn_cast<DILexicalBlockFile>(Context))
       Context = LFScope->getNonLexicalBlockFileScope();
     if (auto *LScope = dyn_cast<DILexicalBlock>(Context))
-      return getLocalContextDIE(LScope);
+      return getLexicalBlockDIE(LScope);
 
     // Otherwise the context must be a DISubprogram.
     auto *SPScope = cast<DISubprogram>(Context);
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
index 5662087bec322..6ef73ebd4f7f2 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
@@ -210,9 +210,14 @@ class DwarfCompileUnit final : public DwarfUnit {
   /// DIE to represent this concrete inlined copy of the function.
   DIE *constructInlinedScopeDIE(LexicalScope *Scope, DIE &ParentScopeDIE);
 
-  /// Get if available or create a new DW_TAG_lexical_block for the given
-  /// LexicalScope and attach DW_AT_low_pc/DW_AT_high_pc labels.
-  DIE *getOrCreateLexicalBlockDIE(LexicalScope *Scope, DIE &ParentDIE);
+  /// Construct new DW_TAG_lexical_block for this scope and
+  /// attach DW_AT_low_pc/DW_AT_high_pc labels.
+  DIE *constructLexicalScopeDIE(LexicalScope *Scope);
+
+  /// Get a DIE for the given DILexicalBlock.
+  /// Note that this function assumes that the DIE has been already created
+  /// and it's an error, if it hasn't.
+  DIE *getLexicalBlockDIE(const DILexicalBlock *LB);
 
   /// constructVariableDIE - Construct a DIE for the given DbgVariable.
   DIE *constructVariableDIE(DbgVariable &DV, bool Abstract = false);
@@ -229,11 +234,6 @@ class DwarfCompileUnit final : public DwarfUnit {
   /// This instance of 'getOrCreateContextDIE()' can handle DILocalScope.
   DIE *getOrCreateContextDIE(const DIScope *Ty) override;
 
-  /// Get DW_TAG_lexical_block for the given DILexicalBlock if available,
-  /// or the most close parent DIE, if no correspoding DW_TAG_lexical_block
-  /// exists.
-  DIE *getLocalContextDIE(const DILexicalBlock *LB);
-
   /// Construct a DIE for this subprogram scope.
   DIE &constructSubprogramScopeDIE(const DISubprogram *Sub,
                                    LexicalScope *Scope);
diff --git a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index f0c13edaf4bd3..df2161410ad08 100644
--- a/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -1229,13 +1229,12 @@ void DwarfDebug::beginModule(Module *M) {
         CU.getOrCreateGlobalVariableDIE(GV, sortGlobalExprs(GVMap[GV]));
     }
 
-    for (auto *Ty : CUNode->getEnumTypes()) {
-      assert(!isa_and_nonnull<DILocalScope>(Ty->getScope()) &&
-             "Unexpected function-local entity in 'enums' CU field.");
+    for (auto *Ty : CUNode->getEnumTypes())
       CU.getOrCreateTypeDIE(cast<DIType>(Ty));
-    }
 
     for (auto *Ty : CUNode->getRetainedTypes()) {
+      // The retained types array by design contains pointers to
+      // MDNodes rather than DIRefs. Unique them here.
       if (DIType *RT = dyn_cast<DIType>(Ty))
         // There is no point in force-emitting a forward declaration.
         CU.getOrCreateTypeDIE(RT);
@@ -1429,13 +1428,9 @@ void DwarfDebug::endModule() {
              "Unexpected function-local entity in 'imports' CU field.");
       CU->getOrCreateImportedEntityDIE(IE);
     }
-
-    // Emit function-local entities.
     for (const auto *D : CU->getDeferredLocalDecls()) {
       if (auto *IE = dyn_cast<DIImportedEntity>(D))
         CU->getOrCreateImportedEntityDIE(IE);
-      else if (auto *Ty = dyn_cast<DIType>(D))
-        CU->getOrCreateTypeDIE(Ty);
       else
         llvm_unreachable("Unexpected local retained node!");
     }
@@ -1533,8 +1528,6 @@ static const DILocalScope *getRetainedNodeScope(const MDNode *N) {
     S = L->getScope();
   else if (const auto *IE = dyn_cast<DIImportedEntity>(N))
     S = IE->getScope();
-  else if (const auto *T = dyn_cast<DIType>(N))
-    S = T->getScope();
   else
     llvm_unreachable("Unexpected retained node!");
 
diff --git a/llvm/lib/IR/DIBuilder.cpp b/llvm/lib/IR/DIBuilder.cpp
index 242ca44794262..ae1a1a37d7c74 100644
--- a/llvm/lib/IR/DIBuilder.cpp
+++ b/llvm/lib/IR/DIBuilder.cpp
@@ -29,7 +29,7 @@ DIBuilder::DIBuilder(Module &m, bool AllowUnresolvedNodes, DICompileUnit *CU)
       AllowUnresolvedNodes(AllowUnresolvedNodes) {
   if (CUNode) {
     if (const auto &ETs = CUNode->getEnumTypes())
-      EnumTypes.assign(ETs.begin(), ETs.end());
+      AllEnumTypes.assign(ETs.begin(), ETs.end());
     if (const auto &RTs = CUNode->getRetainedTypes())
       AllRetainTypes.assign(RTs.begin(), RTs.end());
     if (const auto &GVs = CUNode->getGlobalVariables())
@@ -66,10 +66,10 @@ void DIBuilder::finalize() {
     return;
   }
 
-  if (!EnumTypes.empty())
+  if (!AllEnumTypes.empty())
     CUNode->replaceEnumTypes(MDTuple::get(
-        VMContext, SmallVector<Metadata *, 16>(EnumTypes.begin(),
-                                               EnumTypes.end())));
+        VMContext, SmallVector<Metadata *, 16>(AllEnumTypes.begin(),
+                                               AllEnumTypes.end())));
 
   SmallVector<Metadata *, 16> RetainValues;
   // Declarations and definitions of the same type may be retained. Some
@@ -334,13 +334,10 @@ DIDerivedType *DIBuilder::createTypedef(DIType *Ty, StringRef Name,
                                         DIScope *Context, uint32_t AlignInBits,
                                         DINode::DIFlags Flags,
                                         DINodeArray Annotations) {
-  auto *T =
-      DIDerivedType::get(VMContext, dwarf::DW_TAG_typedef, Name, File, LineNo,
-                         getNonCompileUnitScope(Context), Ty, 0, AlignInBits, 0,
-                         std::nullopt, Flags, nullptr, Annotations);
-  if (isa_and_nonnull<DILocalScope>(Context))
-    getSubprogramNodesTrackingVector(Context).emplace_back(T);
-  return T;
+  return DIDerivedType::get(VMContext, dwarf::DW_TAG_typedef, Name, File,
+                            LineNo, getNonCompileUnitScope(Context), Ty, 0,
+                            AlignInBits, 0, std::nullopt, Flags, nullptr,
+                            Annotations);
 }
 
 DIDerivedType *DIBuilder::createFriend(DIType *Ty, DIType *FriendTy) {
@@ -488,8 +485,6 @@ DICompositeType *DIBuilder::createClassType(
       OffsetInBits, Flags, Elements, 0, VTableHolder,
       cast_or_null<MDTuple>(TemplateParams), UniqueIdentifier);
   trackIfUnresolved(R);
-  if (isa_and_nonnull<DILocalScope>(Context))
-    getSubprogramNodesTrackingVector(Context).emplace_back(R);
   return R;
 }
 
@@ -503,8 +498,6 @@ DICompositeType *DIBuilder::createStructType(
       getNonCompileUnitScope(Context), DerivedFrom, SizeInBits, AlignInBits, 0,
       Flags, Elements, RunTimeLang, VTableHolder, nullptr, UniqueIdentifier);
   trackIfUnresolved(R);
-  if (isa_and_nonnull<DILocalScope>(Context))
-    getSubprogramNodesTrackingVector(Context).emplace_back(R);
   return R;
 }
 
@@ -517,8 +510,6 @@ DICompositeType *DIBuilder::createUnionType(
       getNonCompileUnitScope(Scope), nullptr, SizeInBits, AlignInBits, 0, Flags,
       Elements, RunTimeLang, nullptr, nullptr, UniqueIdentifier);
   trackIfUnresolved(R);
-  if (isa_and_nonnull<DILocalScope>(Scope))
-    getSubprogramNodesTrackingVector(Scope).emplace_back(R);
   return R;
 }
 
@@ -551,10 +542,7 @@ DICompositeType *DIBuilder::createEnumerationType(
       getNonCompileUnitScope(Scope), UnderlyingType, SizeInBits, AlignInBits, 0,
       IsScoped ? DINode::FlagEnumClass : DINode::FlagZero, Elements, 0, nullptr,
       nullptr, UniqueIdentifier);
-  if (isa_and_nonnull<DILocalScope>(Scope))
-    getSubprogramNodesTrackingVector(Scope).emplace_back(CTy);
-  else
-    EnumTypes.emplace_back(CTy);
+  AllEnumTypes.emplace_back(CTy);
   trackIfUnresolved(CTy);
   return CTy;
 }
@@ -652,8 +640,6 @@ DIBuilder::createForwardDecl(unsigned Tag, StringRef Name, DIScope *Scope,
       SizeInBits, AlignInBits, 0, DINode::FlagFwdDecl, nullptr, RuntimeLang,
       nullptr, nullptr, UniqueIdentifier);
   trackIfUnresolved(RetTy);
-  if (isa_and_nonnull<DILocalScope>(Scope))
-    getSubprogramNodesTrackingVector(Scope).emplace_back(RetTy);
   return RetTy;
 }
 
@@ -670,8 +656,6 @@ DICompositeType *DIBuilder::createReplaceableCompositeType(
           nullptr, Annotations)
           .release();
   trackIfUnresolved(RetTy);
-  if (isa_and_nonnull<DILocalScope>(Scope))
-    getSubprogramNodesTrackingVector(Scope).emplace_back(RetTy);
   return RetTy;
 }
 
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 8f929dc7f48ab..293d443d754a4 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -1384,9 +1384,9 @@ void Verifier::visitDISubprogram(const DISubprogram &N) {
     CheckDI(Node, "invalid retained nodes list", &N, RawNode);
     for (Metadata *Op : Node->operands()) {
       CheckDI(Op && (isa<DILocalVariable>(Op) || isa<DILabel>(Op) ||
-                     isa<DIImportedEntity>(Op) || isa<DIType>(Op)),
-              "invalid retained nodes, expected DILocalVariable, DILabel, "
-              "DIImportedEntity or DIType",
+                     isa<DIImportedEntity>(Op)),
+              "invalid retained nodes, expected DILocalVariable, DILabel or "
+              "DIImportedEntity",
               &N, Node, Op);
     }
   }
diff --git a/llvm/test/Bitcode/upgrade-cu-locals.ll b/llvm/test/Bitcode/upgrade-cu-locals.ll
index 7052ab379bea4..9a590f0fc0774 100644
--- a/llvm/test/Bitcode/upgrade-cu-locals.ll
+++ b/llvm/test/Bitcode/upgrade-cu-locals.ll
@@ -1,4 +1,4 @@
-; Test moving of local imports/enums from DICompileUnit to DISubprogram's 'retainedNodes'
+; Test moving of local imports from DICompileUnit's 'imports' to DISubprogram's 'retainedNodes'
 ;
 ; RUN: llvm-dis -o - %s.bc | FileCheck %s
 
@@ -31,36 +31,31 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo
 ; CHECK: !4 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t4"
 
 ; CHECK: !5 = !{}
-; CHECK: !6 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !7, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !8, imports: !5, nameTableKind: GNU)
-; CHECK: !7 = !DIFile(filename: "b.cpp"
-; CHECK: !8 = !{!9}
-; CHECK: !9 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum2", scope: !6, file: !7, line: 4, size: 8, align: 8, elements: !5)
-
-; CHECK: !16 = distinct !DISubprogram(name: "main", scope: !7, file: !7, line: 2, type: !17, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !20)
-; CHECK: !20 = !{!21}
-; CHECK: !21 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !22, entity: !25,
-; CHECK: !22 = !DILexicalBlock(scope: !23, file: !7, line: 7, column: 35)
-; CHECK: !23 = !DILexicalBlock(scope: !24, file: !7, line: 7, column: 35)
-; CHECK: !24 = !DILexicalBlock(scope: !16, file: !7, line: 7, column: 35)
-; CHECK: !25 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t5", scope: !22,
-
-; CHECK: !27 = distinct !DISubprogram(name: "f1", linkageName: "_Z2f1v", scope: !1, file: !1, line: 3, type: !28, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !30)
-; CHECK: !30 = !{!31, !34, !36}
-; CHECK: !31 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !27, entity: !32,
-; CHECK: !32 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t1",
-; CHECK: !34 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !27, entity: !35,
-; CHECK: !35 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t2",
-; CHECK: !36 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !27, entity: !37,
-; CHECK: !37 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t3",
-
-; CHECK: !42 = distinct !DISubprogram(name: "main2", scope: !7, file: !7, line: 10, type: !17, scopeLine: 10, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !43)
-; CHECK: !43 = !{!44, !46, !48, !49}
-; CHECK: !44 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !42, entity: !45,
-; CHECK: !45 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t6"
-; CHECK: !46 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !42, entity: !47,
-; CHECK: !47 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t7",
-; CHECK: !48 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum", scope: !42, file: !7, line: 3, size: 8, align: 8, elements: !5)
-; CHECK: !49 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum3", scope: !42, file: !7, line: 5, size: 8, align: 8, elements: !5)
+; CHECK: !6 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !7, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, imports: !5, nameTableKind: GNU)
+
+; CHECK: !14 = distinct !DISubprogram(name: "main", scope: !7, file: !7, line: 2, type: !15, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !18)
+; CHECK: !18 = !{!19}
+; CHECK: !19 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !20, entity: !23,
+; CHECK: !20 = !DILexicalBlock(scope: !21, file: !7, line: 7, column: 35)
+; CHECK: !21 = !DILexicalBlock(scope: !22, file: !7, line: 7, column: 35)
+; CHECK: !22 = !DILexicalBlock(scope: !14, file: !7, line: 7, column: 35)
+; CHECK: !23 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t5", scope: !20,
+
+; CHECK: !25 = distinct !DISubprogram(name: "f1", linkageName: "_Z2f1v", scope: !1, file: !1, line: 3, type: !26, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !28)
+; CHECK: !28 = !{!29, !32, !34}
+; CHECK: !29 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !25, entity: !30,
+; CHECK: !30 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t1",
+; CHECK: !32 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !25, entity: !33,
+; CHECK: !33 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t2",
+; CHECK: !34 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !25, entity: !35,
+; CHECK: !35 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t3",
+
+; CHECK: !40 = distinct !DISubprogram(name: "main2", scope: !7, file: !7, line: 10, type: !15, scopeLine: 10, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !6, retainedNodes: !41)
+; CHECK: !41 = !{!42, !44}
+; CHECK: !42 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !40, entity: !43,
+; CHECK: !43 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t6"
+; CHECK: !44 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !40, entity: !45,
+; CHECK: !45 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t7",
 
 
 !0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, imports: !2, nameTableKind: GNU)
@@ -87,7 +82,7 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo
 ; Leave t4 in CU
 !14 = !DIImportedEntity(tag: DW_TAG_imported_declaration, scope: !0, entity: !15, file: !1, line: 3)
 !15 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "t4", scope: !0, file: !1, line: 1, size: 8, flags: DIFlagTypePassByValue, elements: !7, identifier: "_ZTSN2ns2t4E")
-!16 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !17, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, imports: !18, enums: !50, nameTableKind: GNU)
+!16 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !17, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, imports: !18, nameTableKind: GNU)
 !17 = !DIFile(filename: "b.cpp", directory: "/")
 !18 = !{!19, !28, !31}
 
@@ -121,12 +116,3 @@ attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memo
 !41 = distinct !DILocation(line: 3, column: 3, scope: !23)
 !42 = !DILocation(line: 3, column: 41, scope: !4, inlinedAt: !41)
 !43 = !DILocation(line: 4, column: 1, scope: !23)
-
-!50 = !{!51, !52, !53}
-; Move to main2
-!51 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum", scope: !29, file: !17, line: 3, size: 8, align: 8, elements: !7)
-; Leave in b.cpp's CU
-!52 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum2", scope: !16, file: !17, line: 4, size: 8, align: 8, elements: !7)
-; Move to main2
-!53 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum3", scope: !29, file: !17, line: 5, size: 8, align: 8, elements: !7)
-
diff --git a/llvm/test/Bitcode/upgrade-cu-locals.ll.bc b/llvm/test/Bitcode/upgrade-cu-locals.ll.bc
index f73d05668af5b16f9ee03e400e2877faab339a0d..9d0ea8fd9a3704ea1fb3a92250500addc75f0587 100644
GIT binary patch
delta 1011
zcmYjPT}&fY6h1R;?+nAR8K7ukpbT3~NOmQ+|Ckt^#_fv6bXy??4aQDuk;YOAbd5%0
zrv<}iQ#P<|%$oR6v-sc!%?9*sAEv1*sKCpvhU}79AB@J-1png8D&A>B@aE3>`OdxH
z`DW%Ebf_IG{nX^@Bi~TU>Tgesr^SEYyYtha=j~yWY!HBg<7MPOE-e79rviCYP}RO~
z{gy^qhXoMu+C9!tM`Yd7K=*rCbF3X;n<h+~OH`|YIw-tvv~}+*HOenH<t0qpa4y8=
zHS>ZdEU$I902B_4N`Kn>%1fgo0Y`Gqk@ou2o|DYupHkvUM?9G>Oq(ubP_5n(a4sm=
zSie0NwAk3&#7VXnE9Y+cR78YGZ0=kSKu|xb4S$DwriFb<1z>Yhhyjpv9A9-m_myaT
zL(g*3@Li&q)A9VPEq8uL*&bj)mDsk61ggK}NN7uR?PWy%O-xFsCjs;lE_N)IMV>_D
zKJ<VL#yUlNktuhbP{;%YAPW0`^xgcQLag7?M(?gN<`w4afdQb^OCrcbyRM_~7ovJk
zNHh~b24E!gv^6YS+3S+7BYC33XFb+2(9pq;LKfhUKA1%l7^Y!o3AQ_nWw=G*URruh
zMutPA&9iE$Y~eK#oI*-qW-;T0g%$|!2;gj`Q;a5?4q3o2P@nm%-O&sQe%1rxYjsbU
z=Cxy{^=ua}TlJP3D30MR*uj5hdmoe^XJpi20TX^o;0UY4X?z)1s<H2RgJn{|6$#z<
zR;n`*f?W)!5|HZNntA7khsjPT59ziZB&B9$Lu%dtZpAW|tfCTdC6Y?b=`i<!I2Ng*
zNfkNTu9e33_iOjYx4+mM|MHjft$U@G=auN@>vPZ8&dgmG{g!t_+n1JiKi&Q4*X8vq
zpX}VuKUzy9i#hYNWpjYqoMKH>Ktt-x<rKPch!#4@&scBPKP}i01Gz9h`{B{i(UTAU
zA-4KAF_pZ}UMzRsq;W(hvxgQP(I6jBp>;^$Xi)Hp1+A%zQVaK+&J4<A;u~v5YD|{n
zlyG{H{@geu1nPyCNvpFYYBLlg1s69s5SfVLsK@JZx`(jG?sT~nm&;-I+LfWlT#gBk
cSBb^kZo4~HxXhgw;!~;V+4S6CBGKsbANlGKqW}N^

delta 1080
zcmYk5e`p(J7{{NxT<<RFwOop6V|rQMH7e;k%bOqVBKYItmQkU&t8>^v2T79^Yj3eX
zCa4ST^$4!hY-CFq`$I=`D*iELuKy_e=R!ISm`cGRL56hTA0ibxhWMA%_q{6H%X`oB
z-1ohE-p}*!zJ}+nr}PA4Tzv8e=ETMO&yU}{_3PDJt*yNt=um-!|A&k}H9J|6zN*x)
zz`pltPnw1RR8~Q?N^cBEwg8N+ilr9PDjEQ4=T<*p+z#fi@<&_{iTssC`dw0W77=EH
z*c_9SV-gA2gRx8VvGQFm-JQ(6b&5U_YNDso)49lW?sWONHl>_yJF*18yXh_f*wo~p
zm<*emHz?jF4h4_b4>b}uctVLpliLG87yzu1pRs>l{*5sRvWb=j0!DRwX=wDE!o5S^
zOf<4d(z>W#%y#_LCnWZe`V1s?y-g$s)sK#yRhD8G6zL2mOMNo{bO~2Hg=K}-PyV<3
zq#>kxqe*`rGx=)KWD%^aF+h`3j$?WG#v>f#G%w}o32^!&9A2suj-yWr%l!{I44%|a
zL4<$~0r4859CHsjn1Ia@0;sw`ra%slionW?#uo{nWGL`pFMPtkj$s~}XJN(nVGSA#
z4zOq@#XFE-No&eR!r!V%R>}e1C})ip;CBXV%(E=PadK-*f}<Bi)~=kQK=3&0zR$P?
zFab!z?O}~vR!8z0DHMT0x@`|!-w+nJg$D{euBG5HOG<>AheRU;84oDiwJ=Y7|E^bU
zPf#aaZ~liW_I^`?Z3Zt9R(*qaScJ9SUwv_3Q<kDq+B#R0>fX~?T-W%k-X<>8UgL=j
zd}ekQW|i;f7Vq_tsi(X3-JsT;L+v-o1XsyBp4;T7(%n-D;dAR-CS<UohPl7N`l(K4
zr0Ys`qSe~^Wumbb`*GsCKMuR^K<6vz%<9a-OTmGKYrRRW?O0>w{Q6hxpZ$6M`h_o>
zw~J42>AA|JXZ&N&#J$xHv5k3on>Xg(ukv3b;h>Ln<a)b(XM#LwT?+4icV}nkh0TA6
zo&B4bdNC2MEc==)o>a+U<_nRGCQVoQRnT!plNqI)a{8Ff@)_sAV5(!c!vJ$=_ozlX
z<-P*D&c#bd?4=WAs{s;qIV!309p_*u{7g7Dd~zy26wOR#W081lI2)ci5Q#<(#PJXw
bjz`1MY<74kUf$<AES)45abZx`X{vt#JnkA(

diff --git a/llvm/test/DebugInfo/Generic/inlined-local-type.ll b/llvm/test/DebugInfo/Generic/inlined-local-type.ll
deleted file mode 100644
index a6f7d294de928..0000000000000
--- a/llvm/test/DebugInfo/Generic/inlined-local-type.ll
+++ /dev/null
@@ -1,128 +0,0 @@
-; RUN: %llc_dwarf -O0 -filetype=obj < %s | llvm-dwarfdump -debug-info - | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s
-; REQUIRES: object-emission
-
-; inline __attribute__((always_inline))
-; int removed() { struct A {int i;}; struct A a; return a.i++; }
-;
-; __attribute__((always_inline))
-; int not_removed() { struct B {int i;}; struct B b; return b.i++; }
-;
-; int foo() { return removed() + not_removed(); }}
-
-; Ensure that function-local types have the correct subprogram parent even if
-; those subprograms are inlined.
-
-; CHECK: DW_TAG_compile_unit
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_AT_abstract_origin	({{0x.*}} "not_removed")
-; CHECK:     DW_TAG_variable
-; CHECK:     NULL
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_AT_name	("removed")
-; CHECK: [[A:0x.*]]: DW_TAG_structure_type
-; CHECK:       DW_AT_name	("A")
-; CHECK:       DW_TAG_member
-; CHECK:       NULL
-; CHECK:     DW_TAG_variable
-; CHECK:       DW_AT_type	([[A]] "A")
-; CHECK:     NULL
-; CHECK:   DW_TAG_base_type
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_AT_name	("not_removed")
-; CHECK: [[B:0x.*]]: DW_TAG_structure_type
-; CHECK:       DW_AT_name	("B")
-; CHECK:       DW_TAG_member
-; CHECK:       NULL
-; CHECK:     DW_TAG_variable
-; CHECK:       DW_AT_type	([[B]] "B")
-; CHECK:     NULL
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_TAG_inlined_subroutine
-; CHECK:       DW_TAG_variable
-; CHECK:       NULL
-; CHECK:     DW_TAG_inlined_subroutine
-; CHECK:       DW_TAG_variable
-; CHECK:       NULL
-; CHECK:     NULL
-; CHECK:   NULL
-
-%struct.B = type { i32 }
-%struct.A = type { i32 }
-
-define dso_local i32 @not_removed() !dbg !12 {
-  %1 = alloca %struct.B, align 4
-  call void @llvm.dbg.declare(metadata %struct.B* %1, metadata !18, metadata !DIExpression()), !dbg !22
-  %2 = getelementptr inbounds %struct.B, %struct.B* %1, i32 0, i32 0, !dbg !23
-  %3 = load i32, i32* %2, align 4, !dbg !24
-  %4 = add nsw i32 %3, 1, !dbg !24
-  store i32 %4, i32* %2, align 4, !dbg !24
-  ret i32 %3, !dbg !25
-}
-
-declare void @llvm.dbg.declare(metadata, metadata, metadata)
-
-define dso_local i32 @foo() !dbg !26 {
-  %1 = alloca %struct.A, align 4
-  %2 = alloca %struct.B, align 4
-  call void @llvm.dbg.declare(metadata %struct.A* %1, metadata !27, metadata !DIExpression()), !dbg !32
-  %3 = getelementptr inbounds %struct.A, %struct.A* %1, i32 0, i32 0, !dbg !34
-  %4 = load i32, i32* %3, align 4, !dbg !35
-  %5 = add nsw i32 %4, 1, !dbg !35
-  store i32 %5, i32* %3, align 4, !dbg !35
-  call void @llvm.dbg.declare(metadata %struct.B* %2, metadata !18, metadata !DIExpression()), !dbg !36
-  %6 = getelementptr inbounds %struct.B, %struct.B* %2, i32 0, i32 0, !dbg !38
-  %7 = load i32, i32* %6, align 4, !dbg !39
-  %8 = add nsw i32 %7, 1, !dbg !39
-  store i32 %8, i32* %6, align 4, !dbg !39
-  %9 = add nsw i32 %4, %7, !dbg !40
-  ret i32 %9, !dbg !41
-}
-
-!llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!2, !3, !4, !5, !6, !7, !8, !9, !10}
-!llvm.ident = !{!11}
-
-!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
-!1 = !DIFile(filename: "inlined-local-type.cpp", directory: "")
-!2 = !{i32 7, !"Dwarf Version", i32 4}
-!3 = !{i32 2, !"Debug Info Version", i32 3}
-!4 = !{i32 1, !"wchar_size", i32 4}
-!5 = !{i32 1, !"branch-target-enforcement", i32 0}
-!6 = !{i32 1, !"sign-return-address", i32 0}
-!7 = !{i32 1, !"sign-return-address-all", i32 0}
-!8 = !{i32 1, !"sign-return-address-with-bkey", i32 0}
-!9 = !{i32 7, !"uwtable", i32 1}
-!10 = !{i32 7, !"frame-pointer", i32 1}
-!11 = !{!"clang version 14.0.0"}
-!12 = distinct !DISubprogram(name: "not_removed", scope: !13, file: !13, line: 5, type: !14, scopeLine: 5, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !43)
-!13 = !DIFile(filename: "inlined-local-type.cpp", directory: "")
-!14 = !DISubroutineType(types: !15)
-!15 = !{!16}
-!16 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-!17 = !{}
-!18 = !DILocalVariable(name: "b", scope: !12, file: !13, line: 5, type: !19)
-!19 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "B", scope: !12, file: !13, line: 5, size: 32, elements: !20)
-!20 = !{!21}
-!21 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !19, file: !13, line: 5, baseType: !16, size: 32)
-!22 = !DILocation(line: 5, column: 49, scope: !12)
-!23 = !DILocation(line: 5, column: 61, scope: !12)
-!24 = !DILocation(line: 5, column: 62, scope: !12)
-!25 = !DILocation(line: 5, column: 52, scope: !12)
-!26 = distinct !DISubprogram(name: "foo", scope: !13, file: !13, line: 7, type: !14, scopeLine: 7, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !17)
-!27 = !DILocalVariable(name: "a", scope: !28, file: !13, line: 2, type: !29)
-!28 = distinct !DISubprogram(name: "removed", scope: !13, file: !13, line: 2, type: !14, scopeLine: 2, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !42)
-!29 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A", scope: !28, file: !13, line: 2, size: 32, elements: !30)
-!30 = !{!31}
-!31 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !29, file: !13, line: 2, baseType: !16, size: 32)
-!32 = !DILocation(line: 2, column: 45, scope: !28, inlinedAt: !33)
-!33 = distinct !DILocation(line: 7, column: 20, scope: !26)
-!34 = !DILocation(line: 2, column: 57, scope: !28, inlinedAt: !33)
-!35 = !DILocation(line: 2, column: 58, scope: !28, inlinedAt: !33)
-!36 = !DILocation(line: 5, column: 49, scope: !12, inlinedAt: !37)
-!37 = distinct !DILocation(line: 7, column: 32, scope: !26)
-!38 = !DILocation(line: 5, column: 61, scope: !12, inlinedAt: !37)
-!39 = !DILocation(line: 5, column: 62, scope: !12, inlinedAt: !37)
-!40 = !DILocation(line: 7, column: 30, scope: !26)
-!41 = !DILocation(line: 7, column: 13, scope: !26)
-!42 = !{!29}
-!43 = !{!19}
diff --git a/llvm/test/DebugInfo/Generic/lexical-block-retained-types.ll b/llvm/test/DebugInfo/Generic/lexical-block-retained-types.ll
deleted file mode 100644
index 64de1323c98c6..0000000000000
--- a/llvm/test/DebugInfo/Generic/lexical-block-retained-types.ll
+++ /dev/null
@@ -1,55 +0,0 @@
-; RUN: %llc_dwarf -O0 -filetype=obj < %s | llvm-dwarfdump - | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s
-
-; Test that retained unused (unreferenced) types emission.
-
-; Compiled from
-; $ clang -cc1 -debug-info-kind=unused-types test.cpp -emit-llvm
-
-; void test_unused() {
-;   struct Y {};
-;   {
-;     struct X {};
-;   }
-; }
-
-; CHECK: DW_TAG_compile_unit
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_AT_name	("test_unused")
-; CHECK:     DW_TAG_structure_type
-; CHECK:       DW_AT_name	("Y")
-
-; FIXME: here should be DW_TAG_lexical_block as a parent of structure 'X'.
-; But it's not possible to reliably emit a lexical block for which a LexicalScope
-; wasn't created, so we just fallback to the most close parent DIE
-; (see DwarfCompileUnit::getOrCreateLexicalBlockDIE() for details).
-
-; CHECK:     DW_TAG_structure_type
-; CHECK:       DW_AT_name	("X")
-; CHECK:     NULL
-; CHECK:   NULL
-
-define dso_local void @_Z11test_unusedv() !dbg !5 {
-entry:
-  ret void, !dbg !16
-}
-
-!llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!13, !14}
-!llvm.ident = !{!15}
-
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 15.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, retainedTypes: !2, splitDebugInlining: false, nameTableKind: None)
-!1 = !DIFile(filename: "<stdin>", directory: "/")
-!2 = !{!3, !10}
-!3 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "Y", scope: !5, file: !4, line: 2, size: 8, flags: DIFlagTypePassByValue, elements: !8)
-!4 = !DIFile(filename: "test.cpp", directory: "/")
-!5 = distinct !DISubprogram(name: "test_unused", linkageName: "_Z11test_unusedv", scope: !4, file: !4, line: 1, type: !6, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !9)
-!6 = !DISubroutineType(types: !7)
-!7 = !{null}
-!8 = !{}
-!9 = !{!3, !10}
-!10 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "X", scope: !11, file: !4, line: 4, size: 8, flags: DIFlagTypePassByValue, elements: !8)
-!11 = distinct !DILexicalBlock(scope: !5, file: !4, line: 3, column: 3)
-!13 = !{i32 2, !"Debug Info Version", i32 3}
-!14 = !{i32 1, !"wchar_size", i32 4}
-!15 = !{!"clang version 15.0.0"}
-!16 = !DILocation(line: 6, column: 1, scope: !5)
diff --git a/llvm/test/DebugInfo/Generic/lexical-block-types.ll b/llvm/test/DebugInfo/Generic/lexical-block-types.ll
deleted file mode 100644
index aef59984351a7..0000000000000
--- a/llvm/test/DebugInfo/Generic/lexical-block-types.ll
+++ /dev/null
@@ -1,425 +0,0 @@
-; RUN: %llc_dwarf -O0 -filetype=obj < %s | llvm-dwarfdump -debug-info - | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s
-; REQUIRES: object-emission
-
-; inline __attribute__((always_inline))
-; void removed() {
-;   struct A1 { int i; };
-;   typedef int Int1;
-;   {
-;     struct I1 { Int1 j; };
-;     struct C1 { typedef char Char1; Char1 c; };
-;     A1 a1; a1.i++;
-;     {
-;       I1 i1; i1.j++;
-;       C1 c1; c1.c++;
-;     }
-;   }
-; }
-;
-; __attribute__((always_inline))
-; void not_removed() {
-;   struct A2 { int i; };
-;   typedef int Int2;
-;   {
-;     struct I2 { Int2 j; };
-;     struct C2 { typedef char Char2; Char2 c; };
-;     A2 a2; a2.i++;
-;     {
-;       I2 i2; i2.j++;
-;       C2 c2; c2.c++;
-;     }
-;   }
-; }
-;
-; void foo() {
-;   struct A3 { int i; };
-;   typedef int Int3;
-;   {
-;     struct I3 { Int3 j; };
-;     {
-;       struct C3 { typedef char Char3; Char3 c; };
-;       A3 a3; a3.i++;
-;       {
-;         I3 i3; i3.j++;
-;         C3 c3; c3.c++;
-;       }
-;     }
-;   }
-;   removed();
-;   not_removed();
-; }
-;
-; CHECK: DW_TAG_compile_unit
-
-; Out-of-line definition of `not_removed()` shouldn't contain any debug info for types.
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_AT_abstract_origin	{{.*}} "_Z11not_removedv"
-; CHECK:     DW_TAG_lexical_block
-; CHECK:       DW_TAG_variable
-; CHECK:         DW_AT_abstract_origin	{{.*}} "a2"
-; CHECK:       DW_TAG_lexical_block
-; CHECK:         DW_TAG_variable
-; CHECK:           DW_AT_abstract_origin	{{.*}} "i2"
-; CHECK:         DW_TAG_variable
-; CHECK:           DW_AT_abstract_origin	{{.*}} "c2"
-; CHECK:         NULL
-; CHECK:       NULL
-; CHECK:     NULL
-
-; Abstract definition of `removed()`.
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_AT_name	("removed")
-; CHECK:     DW_AT_inline	(DW_INL_inlined)
-
-; I1 and C1 defined in the first lexical block, typedef Char1 is a child of C1.
-; CHECK:     DW_TAG_lexical_block
-; CHECK:       DW_TAG_variable
-; CHECK:         DW_AT_name	("a1")
-; CHECK:         DW_AT_type	{{.*}} "A1"
-; CHECK:       DW_TAG_lexical_block
-; CHECK:         DW_TAG_variable
-; CHECK:           DW_AT_type	{{.*}} "I1"
-; CHECK:         DW_TAG_variable
-; CHECK:           DW_AT_type	{{.*}} "C1"
-; CHECK:         NULL
-; CHECK:       DW_TAG_structure_type
-; CHECK:         DW_AT_name	("I1")
-; CHECK:         DW_TAG_member
-; CHECK:           DW_AT_type	{{.*}} "Int1"
-; CHECK:         NULL
-; CHECK:       DW_TAG_structure_type
-; CHECK:         DW_AT_name	("C1")
-; CHECK:         DW_TAG_member
-; CHECK:           DW_AT_type	{{.*}} "Char1"
-; CHECK:         DW_TAG_typedef
-; CHECK:           DW_AT_name	("Char1")
-; CHECK:         NULL
-; CHECK:       NULL
-
-; A1 and typedef Int1 defined in the subprogram scope.
-; CHECK:     DW_TAG_structure_type
-; CHECK:       DW_AT_name	("A1")
-; CHECK:       DW_TAG_member
-; CHECK:       NULL
-; CHECK:     DW_TAG_typedef
-; CHECK:       DW_AT_name	("Int1")
-; CHECK:     NULL
-
-; CHECK:   DW_TAG_base_type
-; CHECK:   DW_TAG_base_type
-
-; Abstract definition of `not_removed()`.
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_AT_name	("not_removed")
-; CHECK:     DW_AT_inline	(DW_INL_inlined)
-
-; I2 and C2 defined in the first lexical block, typedef Char2 is a child of C2.
-; CHECK:     DW_TAG_lexical_block
-; CHECK:       DW_TAG_variable
-; CHECK:         DW_AT_name	("a2")
-; CHECK:         DW_AT_type	{{.*}} "A2"
-; CHECK:       DW_TAG_lexical_block
-; CHECK:         DW_TAG_variable
-; CHECK:           DW_AT_name	("i2")
-; CHECK:           DW_AT_type	{{.*}} "I2"
-; CHECK:         DW_TAG_variable
-; CHECK:           DW_AT_name	("c2")
-; CHECK:           DW_AT_type	{{.*}} "C2"
-; CHECK:         NULL
-; CHECK:       DW_TAG_structure_type
-; CHECK:         DW_AT_name	("I2")
-; CHECK:         DW_TAG_member
-; CHECK:           DW_AT_type	{{.*}} "Int2"
-; CHECK:         NULL
-; CHECK:       DW_TAG_structure_type
-; CHECK:         DW_AT_name	("C2")
-; CHECK:         DW_TAG_member
-; CHECK:           DW_AT_type	{{.*}} "Char2"
-; CHECK:         DW_TAG_typedef
-; CHECK:           DW_AT_name	("Char2")
-; CHECK:         NULL
-; CHECK:       NULL
-
-; A2 and typedef Int2 defined in subprogram scope.
-; CHECK:     DW_TAG_structure_type
-; CHECK:       DW_AT_name	("A2")
-; CHECK:       DW_TAG_member
-; CHECK:       NULL
-; CHECK:     DW_TAG_typedef
-; CHECK:       DW_AT_name	("Int2")
-; CHECK:     NULL
-
-; Definition of `foo()`.
-; CHECK:   DW_TAG_subprogram
-; CHECK:     DW_AT_name	("foo")
-
-; CHECK:     DW_TAG_lexical_block
-; CHECK:       DW_TAG_lexical_block
-; CHECK:         DW_TAG_variable
-; CHECK:           DW_AT_name	("a3")
-; CHECK:           DW_AT_type	{{.*}} "A3"
-; CHECK:         DW_TAG_lexical_block
-; CHECK:           DW_TAG_variable
-; CHECK:             DW_AT_name	("i3")
-; CHECK:             DW_AT_type	{{.*}} "I3"
-; CHECK:           DW_TAG_variable
-; CHECK:             DW_AT_name	("c3")
-; CHECK:             DW_AT_type	{{.*}} "C3"
-; CHECK:           NULL
-
-; C3 has the inner lexical block scope, typedef Char3 is a child of C3.
-; CHECK:         DW_TAG_structure_type
-; CHECK:           DW_AT_name	("C3")
-; CHECK:           DW_TAG_member
-; CHECK:             DW_AT_type	{{.*}} "Char3"
-; CHECK:           DW_TAG_typedef
-; CHECK:             DW_AT_name	("Char3")
-; CHECK:           NULL
-; CHECK:         NULL
-
-; I3 has the outer lexical block scope.
-; CHECK:       DW_TAG_structure_type
-; CHECK:         DW_AT_name	("I3")
-; CHECK:         DW_TAG_member
-; CHECK:           DW_AT_type	{{.*}} "Int3"
-; CHECK:         NULL
-; CHECK:       NULL
-
-; CHECK:     DW_TAG_inlined_subroutine
-; CHECK:       DW_AT_abstract_origin	{{.*}} "_Z7removedv"
-; CHECK:       DW_TAG_lexical_block
-; CHECK:         DW_TAG_variable
-; CHECK:         DW_TAG_lexical_block
-; CHECK:           DW_TAG_variable
-; CHECK:           DW_TAG_variable
-; CHECK:           NULL
-; CHECK:         NULL
-; CHECK:       NULL
-
-; CHECK:     DW_TAG_inlined_subroutine
-; CHECK:       DW_AT_abstract_origin	{{.*}} "_Z11not_removedv"
-; CHECK:       DW_TAG_lexical_block
-; CHECK:         DW_TAG_variable
-; CHECK:         DW_TAG_lexical_block
-; CHECK:           DW_TAG_variable
-; CHECK:           DW_TAG_variable
-; CHECK:           NULL
-; CHECK:         NULL
-; CHECK:       NULL
-
-; A3 and Int3 defined within the subprogam scope.
-; CHECK:     DW_TAG_structure_type
-; CHECK:       DW_AT_name	("A3")
-; CHECK:       DW_TAG_member
-; CHECK:       NULL
-; CHECK:     DW_TAG_typedef
-; CHECK:       DW_AT_name	("Int3")
-; CHECK:     NULL
-; CHECK:   NULL
-
-%struct.A2 = type { i32 }
-%struct.I2 = type { i32 }
-%struct.C2 = type { i8 }
-%struct.A1 = type { i32 }
-%struct.I1 = type { i32 }
-%struct.C1 = type { i8 }
-%struct.A3 = type { i32 }
-%struct.I3 = type { i32 }
-%struct.C3 = type { i8 }
-
-define dso_local void @_Z11not_removedv() !dbg !8 {
-entry:
-  %a2 = alloca %struct.A2, align 4
-  %i2 = alloca %struct.I2, align 4
-  %c2 = alloca %struct.C2, align 1
-  call void @llvm.dbg.declare(metadata %struct.A2* %a2, metadata !12, metadata !DIExpression()), !dbg !18
-  %i = getelementptr inbounds %struct.A2, %struct.A2* %a2, i32 0, i32 0, !dbg !19
-  %0 = load i32, i32* %i, align 4, !dbg !20
-  %inc = add nsw i32 %0, 1, !dbg !20
-  store i32 %inc, i32* %i, align 4, !dbg !20
-  call void @llvm.dbg.declare(metadata %struct.I2* %i2, metadata !21, metadata !DIExpression()), !dbg !27
-  %j = getelementptr inbounds %struct.I2, %struct.I2* %i2, i32 0, i32 0, !dbg !28
-  %1 = load i32, i32* %j, align 4, !dbg !29
-  %inc1 = add nsw i32 %1, 1, !dbg !29
-  store i32 %inc1, i32* %j, align 4, !dbg !29
-  call void @llvm.dbg.declare(metadata %struct.C2* %c2, metadata !30, metadata !DIExpression()), !dbg !36
-  %c = getelementptr inbounds %struct.C2, %struct.C2* %c2, i32 0, i32 0, !dbg !37
-  %2 = load i8, i8* %c, align 1, !dbg !38
-  %inc2 = add i8 %2, 1, !dbg !38
-  store i8 %inc2, i8* %c, align 1, !dbg !38
-  ret void, !dbg !39
-}
-
-declare void @llvm.dbg.declare(metadata, metadata, metadata)
-
-define dso_local void @_Z3foov() !dbg !40 {
-entry:
-  %a1.i = alloca %struct.A1, align 4
-  %i1.i = alloca %struct.I1, align 4
-  %c1.i = alloca %struct.C1, align 1
-  %a2.i = alloca %struct.A2, align 4
-  %i2.i = alloca %struct.I2, align 4
-  %c2.i = alloca %struct.C2, align 1
-  %a3 = alloca %struct.A3, align 4
-  %i3 = alloca %struct.I3, align 4
-  %c3 = alloca %struct.C3, align 1
-  call void @llvm.dbg.declare(metadata %struct.A3* %a3, metadata !41, metadata !DIExpression()), !dbg !47
-  %i = getelementptr inbounds %struct.A3, %struct.A3* %a3, i32 0, i32 0, !dbg !48
-  %0 = load i32, i32* %i, align 4, !dbg !49
-  %inc = add nsw i32 %0, 1, !dbg !49
-  store i32 %inc, i32* %i, align 4, !dbg !49
-  call void @llvm.dbg.declare(metadata %struct.I3* %i3, metadata !50, metadata !DIExpression()), !dbg !56
-  %j = getelementptr inbounds %struct.I3, %struct.I3* %i3, i32 0, i32 0, !dbg !57
-  %1 = load i32, i32* %j, align 4, !dbg !58
-  %inc1 = add nsw i32 %1, 1, !dbg !58
-  store i32 %inc1, i32* %j, align 4, !dbg !58
-  call void @llvm.dbg.declare(metadata %struct.C3* %c3, metadata !59, metadata !DIExpression()), !dbg !64
-  %c = getelementptr inbounds %struct.C3, %struct.C3* %c3, i32 0, i32 0, !dbg !65
-  %2 = load i8, i8* %c, align 1, !dbg !66
-  %inc2 = add i8 %2, 1, !dbg !66
-  store i8 %inc2, i8* %c, align 1, !dbg !66
-  call void @llvm.dbg.declare(metadata %struct.A1* %a1.i, metadata !67, metadata !DIExpression()), !dbg !73
-  %i.i3 = getelementptr inbounds %struct.A1, %struct.A1* %a1.i, i32 0, i32 0, !dbg !75
-  %3 = load i32, i32* %i.i3, align 4, !dbg !76
-  %inc.i4 = add nsw i32 %3, 1, !dbg !76
-  store i32 %inc.i4, i32* %i.i3, align 4, !dbg !76
-  call void @llvm.dbg.declare(metadata %struct.I1* %i1.i, metadata !77, metadata !DIExpression()), !dbg !83
-  %j.i5 = getelementptr inbounds %struct.I1, %struct.I1* %i1.i, i32 0, i32 0, !dbg !84
-  %4 = load i32, i32* %j.i5, align 4, !dbg !85
-  %inc1.i6 = add nsw i32 %4, 1, !dbg !85
-  store i32 %inc1.i6, i32* %j.i5, align 4, !dbg !85
-  call void @llvm.dbg.declare(metadata %struct.C1* %c1.i, metadata !86, metadata !DIExpression()), !dbg !91
-  %c.i7 = getelementptr inbounds %struct.C1, %struct.C1* %c1.i, i32 0, i32 0, !dbg !92
-  %5 = load i8, i8* %c.i7, align 1, !dbg !93
-  %inc2.i8 = add i8 %5, 1, !dbg !93
-  store i8 %inc2.i8, i8* %c.i7, align 1, !dbg !93
-  call void @llvm.dbg.declare(metadata %struct.A2* %a2.i, metadata !12, metadata !DIExpression()), !dbg !94
-  %i.i = getelementptr inbounds %struct.A2, %struct.A2* %a2.i, i32 0, i32 0, !dbg !96
-  %6 = load i32, i32* %i.i, align 4, !dbg !97
-  %inc.i = add nsw i32 %6, 1, !dbg !97
-  store i32 %inc.i, i32* %i.i, align 4, !dbg !97
-  call void @llvm.dbg.declare(metadata %struct.I2* %i2.i, metadata !21, metadata !DIExpression()), !dbg !98
-  %j.i = getelementptr inbounds %struct.I2, %struct.I2* %i2.i, i32 0, i32 0, !dbg !99
-  %7 = load i32, i32* %j.i, align 4, !dbg !100
-  %inc1.i = add nsw i32 %7, 1, !dbg !100
-  store i32 %inc1.i, i32* %j.i, align 4, !dbg !100
-  call void @llvm.dbg.declare(metadata %struct.C2* %c2.i, metadata !30, metadata !DIExpression()), !dbg !101
-  %c.i = getelementptr inbounds %struct.C2, %struct.C2* %c2.i, i32 0, i32 0, !dbg !102
-  %8 = load i8, i8* %c.i, align 1, !dbg !103
-  %inc2.i = add i8 %8, 1, !dbg !103
-  store i8 %inc2.i, i8* %c.i, align 1, !dbg !103
-  ret void, !dbg !104
-}
-
-!llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!2, !3, !4, !5, !6}
-!llvm.ident = !{!7}
-
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 14.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None)
-!1 = !DIFile(filename: "test.cpp", directory: "/")
-!2 = !{i32 7, !"Dwarf Version", i32 4}
-!3 = !{i32 2, !"Debug Info Version", i32 3}
-!4 = !{i32 1, !"wchar_size", i32 4}
-!5 = !{i32 7, !"uwtable", i32 1}
-!6 = !{i32 7, !"frame-pointer", i32 2}
-!7 = !{!"clang version 14.0.0"}
-!8 = distinct !DISubprogram(name: "not_removed", linkageName: "_Z11not_removedv", scope: !1, file: !1, line: 17, type: !9, scopeLine: 17, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !105)
-!9 = !DISubroutineType(types: !10)
-!10 = !{null}
-!11 = !{}
-!12 = !DILocalVariable(name: "a2", scope: !13, file: !1, line: 23, type: !14)
-!13 = distinct !DILexicalBlock(scope: !8, file: !1, line: 20, column: 3)
-!14 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A2", scope: !8, file: !1, line: 18, size: 32, flags: DIFlagTypePassByValue, elements: !15)
-!15 = !{!16}
-!16 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !14, file: !1, line: 18, baseType: !17, size: 32)
-!17 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-!18 = !DILocation(line: 23, column: 8, scope: !13)
-!19 = !DILocation(line: 23, column: 15, scope: !13)
-!20 = !DILocation(line: 23, column: 16, scope: !13)
-!21 = !DILocalVariable(name: "i2", scope: !22, file: !1, line: 25, type: !23)
-!22 = distinct !DILexicalBlock(scope: !13, file: !1, line: 24, column: 5)
-!23 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "I2", scope: !13, file: !1, line: 21, size: 32, flags: DIFlagTypePassByValue, elements: !24)
-!24 = !{!25}
-!25 = !DIDerivedType(tag: DW_TAG_member, name: "j", scope: !23, file: !1, line: 21, baseType: !26, size: 32)
-!26 = !DIDerivedType(tag: DW_TAG_typedef, name: "Int2", scope: !8, file: !1, line: 19, baseType: !17)
-!27 = !DILocation(line: 25, column: 10, scope: !22)
-!28 = !DILocation(line: 25, column: 17, scope: !22)
-!29 = !DILocation(line: 25, column: 18, scope: !22)
-!30 = !DILocalVariable(name: "c2", scope: !22, file: !1, line: 26, type: !31)
-!31 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "C2", scope: !13, file: !1, line: 22, size: 8, flags: DIFlagTypePassByValue, elements: !32)
-!32 = !{!33}
-!33 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !31, file: !1, line: 22, baseType: !34, size: 8)
-!34 = !DIDerivedType(tag: DW_TAG_typedef, name: "Char2", scope: !31, file: !1, line: 22, baseType: !35)
-!35 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
-!36 = !DILocation(line: 26, column: 10, scope: !22)
-!37 = !DILocation(line: 26, column: 17, scope: !22)
-!38 = !DILocation(line: 26, column: 18, scope: !22)
-!39 = !DILocation(line: 29, column: 1, scope: !8)
-!40 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !1, file: !1, line: 31, type: !9, scopeLine: 31, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !107)
-!41 = !DILocalVariable(name: "a3", scope: !42, file: !1, line: 38, type: !44)
-!42 = distinct !DILexicalBlock(scope: !43, file: !1, line: 36, column: 5)
-!43 = distinct !DILexicalBlock(scope: !40, file: !1, line: 34, column: 3)
-!44 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A3", scope: !40, file: !1, line: 32, size: 32, flags: DIFlagTypePassByValue, elements: !45)
-!45 = !{!46}
-!46 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !44, file: !1, line: 32, baseType: !17, size: 32)
-!47 = !DILocation(line: 38, column: 10, scope: !42)
-!48 = !DILocation(line: 38, column: 17, scope: !42)
-!49 = !DILocation(line: 38, column: 18, scope: !42)
-!50 = !DILocalVariable(name: "i3", scope: !51, file: !1, line: 40, type: !52)
-!51 = distinct !DILexicalBlock(scope: !42, file: !1, line: 39, column: 7)
-!52 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "I3", scope: !43, file: !1, line: 35, size: 32, flags: DIFlagTypePassByValue, elements: !53)
-!53 = !{!54}
-!54 = !DIDerivedType(tag: DW_TAG_member, name: "j", scope: !52, file: !1, line: 35, baseType: !55, size: 32)
-!55 = !DIDerivedType(tag: DW_TAG_typedef, name: "Int3", scope: !40, file: !1, line: 33, baseType: !17)
-!56 = !DILocation(line: 40, column: 12, scope: !51)
-!57 = !DILocation(line: 40, column: 19, scope: !51)
-!58 = !DILocation(line: 40, column: 20, scope: !51)
-!59 = !DILocalVariable(name: "c3", scope: !51, file: !1, line: 41, type: !60)
-!60 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "C3", scope: !42, file: !1, line: 37, size: 8, flags: DIFlagTypePassByValue, elements: !61)
-!61 = !{!62}
-!62 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !60, file: !1, line: 37, baseType: !63, size: 8)
-!63 = !DIDerivedType(tag: DW_TAG_typedef, name: "Char3", scope: !60, file: !1, line: 37, baseType: !35)
-!64 = !DILocation(line: 41, column: 12, scope: !51)
-!65 = !DILocation(line: 41, column: 19, scope: !51)
-!66 = !DILocation(line: 41, column: 20, scope: !51)
-!67 = !DILocalVariable(name: "a1", scope: !68, file: !1, line: 8, type: !70)
-!68 = distinct !DILexicalBlock(scope: !69, file: !1, line: 5, column: 3)
-!69 = distinct !DISubprogram(name: "removed", linkageName: "_Z7removedv", scope: !1, file: !1, line: 2, type: !9, scopeLine: 2, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !110)
-!70 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A1", scope: !69, file: !1, line: 3, size: 32, flags: DIFlagTypePassByValue, elements: !71, identifier: "_ZTSZ7removedvE2A1")
-!71 = !{!72}
-!72 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !70, file: !1, line: 3, baseType: !17, size: 32)
-!73 = !DILocation(line: 8, column: 8, scope: !68, inlinedAt: !74)
-!74 = distinct !DILocation(line: 45, column: 3, scope: !40)
-!75 = !DILocation(line: 8, column: 15, scope: !68, inlinedAt: !74)
-!76 = !DILocation(line: 8, column: 16, scope: !68, inlinedAt: !74)
-!77 = !DILocalVariable(name: "i1", scope: !78, file: !1, line: 10, type: !79)
-!78 = distinct !DILexicalBlock(scope: !68, file: !1, line: 9, column: 5)
-!79 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "I1", scope: !68, file: !1, line: 6, size: 32, flags: DIFlagTypePassByValue, elements: !80, identifier: "_ZTSZ7removedvE2I1")
-!80 = !{!81}
-!81 = !DIDerivedType(tag: DW_TAG_member, name: "j", scope: !79, file: !1, line: 6, baseType: !82, size: 32)
-!82 = !DIDerivedType(tag: DW_TAG_typedef, name: "Int1", scope: !69, file: !1, line: 4, baseType: !17)
-!83 = !DILocation(line: 10, column: 10, scope: !78, inlinedAt: !74)
-!84 = !DILocation(line: 10, column: 17, scope: !78, inlinedAt: !74)
-!85 = !DILocation(line: 10, column: 18, scope: !78, inlinedAt: !74)
-!86 = !DILocalVariable(name: "c1", scope: !78, file: !1, line: 11, type: !87)
-!87 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "C1", scope: !68, file: !1, line: 7, size: 8, flags: DIFlagTypePassByValue, elements: !88, identifier: "_ZTSZ7removedvE2C1")
-!88 = !{!89}
-!89 = !DIDerivedType(tag: DW_TAG_member, name: "c", scope: !87, file: !1, line: 7, baseType: !90, size: 8)
-!90 = !DIDerivedType(tag: DW_TAG_typedef, name: "Char1", scope: !87, file: !1, line: 7, baseType: !35)
-!91 = !DILocation(line: 11, column: 10, scope: !78, inlinedAt: !74)
-!92 = !DILocation(line: 11, column: 17, scope: !78, inlinedAt: !74)
-!93 = !DILocation(line: 11, column: 18, scope: !78, inlinedAt: !74)
-!94 = !DILocation(line: 23, column: 8, scope: !13, inlinedAt: !95)
-!95 = distinct !DILocation(line: 46, column: 3, scope: !40)
-!96 = !DILocation(line: 23, column: 15, scope: !13, inlinedAt: !95)
-!97 = !DILocation(line: 23, column: 16, scope: !13, inlinedAt: !95)
-!98 = !DILocation(line: 25, column: 10, scope: !22, inlinedAt: !95)
-!99 = !DILocation(line: 25, column: 17, scope: !22, inlinedAt: !95)
-!100 = !DILocation(line: 25, column: 18, scope: !22, inlinedAt: !95)
-!101 = !DILocation(line: 26, column: 10, scope: !22, inlinedAt: !95)
-!102 = !DILocation(line: 26, column: 17, scope: !22, inlinedAt: !95)
-!103 = !DILocation(line: 26, column: 18, scope: !22, inlinedAt: !95)
-!104 = !DILocation(line: 47, column: 1, scope: !40)
-!105 = !{!14, !23, !26, !31}
-!107 = !{!44, !52, !55, !60}
-!110 = !{!70, !79, !82, !87}
diff --git a/llvm/test/DebugInfo/X86/split-dwarf-local-import.ll b/llvm/test/DebugInfo/Generic/split-dwarf-local-import.ll
similarity index 98%
rename from llvm/test/DebugInfo/X86/split-dwarf-local-import.ll
rename to llvm/test/DebugInfo/Generic/split-dwarf-local-import.ll
index 676ef23b679e8..1a41f3f348ca0 100644
--- a/llvm/test/DebugInfo/X86/split-dwarf-local-import.ll
+++ b/llvm/test/DebugInfo/Generic/split-dwarf-local-import.ll
@@ -1,3 +1,4 @@
+; REQUIRES: x86-registered-target
 ; RUN: %llc_dwarf -O1 -filetype=obj -split-dwarf-file=%t.dwo < %s | llvm-dwarfdump -debug-info - | FileCheck %s --implicit-check-not "{{DW_TAG|NULL}}"
 
 ; CHECK-LABEL: debug_info contents
diff --git a/llvm/test/DebugInfo/X86/split-dwarf-local-import2.ll b/llvm/test/DebugInfo/Generic/split-dwarf-local-import2.ll
similarity index 98%
rename from llvm/test/DebugInfo/X86/split-dwarf-local-import2.ll
rename to llvm/test/DebugInfo/Generic/split-dwarf-local-import2.ll
index 71f80992d1b04..0057f675f9b31 100644
--- a/llvm/test/DebugInfo/X86/split-dwarf-local-import2.ll
+++ b/llvm/test/DebugInfo/Generic/split-dwarf-local-import2.ll
@@ -1,3 +1,4 @@
+; REQUIRES: x86-registered-target
 ; RUN: %llc_dwarf -split-dwarf-file=%t.dwo < %s | FileCheck %s
 
 ; Ensure function-local DW_TAG_imported_declaration get skipped if its parent subprogram was not emitted.
diff --git a/llvm/test/DebugInfo/X86/split-dwarf-local-import3.ll b/llvm/test/DebugInfo/Generic/split-dwarf-local-import3.ll
similarity index 100%
rename from llvm/test/DebugInfo/X86/split-dwarf-local-import3.ll
rename to llvm/test/DebugInfo/Generic/split-dwarf-local-import3.ll
diff --git a/llvm/test/DebugInfo/Generic/verifier-invalid-disubprogram.ll b/llvm/test/DebugInfo/Generic/verifier-invalid-disubprogram.ll
index 54ce1c56c6b30..6d4d0e93d38f9 100644
--- a/llvm/test/DebugInfo/Generic/verifier-invalid-disubprogram.ll
+++ b/llvm/test/DebugInfo/Generic/verifier-invalid-disubprogram.ll
@@ -38,7 +38,7 @@ define void @invalid_subprogram_declaration() !dbg !9 { ret void }
 define void @invalid_retained_nodes_list() !dbg !10 { ret void }
 !10 = distinct !DISubprogram(retainedNodes: !0)
 
-; CHECK: invalid retained nodes, expected DILocalVariable, DILabel, DIImportedEntity or DIType
+; CHECK: invalid retained nodes, expected DILocalVariable, DILabel or DIImportedEntity
 define void @invalid_retained_nodes_expected() !dbg !11 { ret void }
 !11 = distinct !DISubprogram(retainedNodes: !{!0})
 
diff --git a/llvm/test/DebugInfo/X86/local-type-as-template-parameter.ll b/llvm/test/DebugInfo/X86/local-type-as-template-parameter.ll
deleted file mode 100644
index 586574d3e1aaf..0000000000000
--- a/llvm/test/DebugInfo/X86/local-type-as-template-parameter.ll
+++ /dev/null
@@ -1,161 +0,0 @@
-; REQUIERES: system-linux
-; RUN: %llc_dwarf -mtriple=x86_64-linux -O0 -filetype=obj < %s              \
-; RUN:  | llvm-dwarfdump --show-children --name=foo - \
-; RUN:  | FileCheck --implicit-check-not "{{DW_TAG|NULL}}" %s
-
-; The test ensures that AsmPrinter doesn't crashed compiling this.
-; It also demostrates misplacement for a local type (see PR55680 for details).
-
-; The test compiled from:
-
-; template<typename T>
-; struct A {
-;   A(T &in) : a(in) {}
-;   T a;
-; };
-;
-; __attribute__((always_inline))
-; void foo() {
-;   struct B { int i; };
-;   B objB;
-;   A<B> objA(objB);
-; }
-;
-; int main() {
-;   foo();
-; }
-
-; Concrete out-of-line tree of foo().
-; CHECK: DW_TAG_subprogram
-; CHECK:   DW_AT_abstract_origin {{.*}} "_Z3foov"
-
-; FIXME: 'struct B' should be in the abstract tree below, not here.
-; CHECK:   DW_TAG_structure_type
-; CHECK:     DW_AT_name	("B")
-; CHECK:     DW_TAG_member
-; CHECK:     NULL
-;
-; CHECK:   DW_TAG_variable
-; CHECK:     DW_AT_abstract_origin {{.*}} "objB"
-; CHECK:   DW_TAG_variable
-; CHECK:     DW_AT_abstract_origin {{.*}} "objA"
-
-; CHECK:   NULL
-
-; Abstract tree of foo().
-; CHECK: DW_TAG_subprogram
-; CHECK:   DW_AT_name	("foo")
-; CHECK:   DW_AT_inline	(DW_INL_inlined)
-
-; CHECK:   DW_TAG_variable
-; CHECK:     DW_AT_name	("objB")
-; CHECK:   DW_TAG_variable
-; CHECK:     DW_AT_name	("objA")
-
-; CHECK:   NULL
-
-; CHECK: DW_TAG_inlined_subroutine
-; CHECK:   DW_AT_abstract_origin {{.*}} "_Z3foov"
-; CHECK:   DW_TAG_variable
-; CHECK:     DW_AT_abstract_origin {{.*}} "objB"
-; CHECK:   DW_TAG_variable
-; CHECK:     DW_AT_abstract_origin {{.*}} "objA"
-; CHECK:   NULL
-
-%struct.B = type { i32 }
-%struct.A = type { %struct.B }
-
-define dso_local void @_Z3foov() !dbg !7 {
-entry:
-  %objB = alloca %struct.B, align 4
-  %objA = alloca %struct.A, align 4
-  call void @llvm.dbg.declare(metadata ptr %objB, metadata !30, metadata !DIExpression()), !dbg !31
-  call void @llvm.dbg.declare(metadata ptr %objA, metadata !32, metadata !DIExpression()), !dbg !33
-  call void @_ZN1AIZ3foovE1BEC2ERS0_(ptr noundef nonnull align 4 dereferenceable(4) %objA, ptr noundef nonnull align 4 dereferenceable(4) %objB), !dbg !33
-  ret void, !dbg !34
-}
-
-declare void @llvm.dbg.declare(metadata, metadata, metadata)
-
-define internal void @_ZN1AIZ3foovE1BEC2ERS0_(ptr noundef nonnull align 4 dereferenceable(4) %this, ptr noundef nonnull align 4 dereferenceable(4) %in) unnamed_addr align 2 !dbg !35 {
-entry:
-  %this.addr = alloca ptr, align 8
-  %in.addr = alloca ptr, align 8
-  store ptr %this, ptr %this.addr, align 8
-  call void @llvm.dbg.declare(metadata ptr %this.addr, metadata !36, metadata !DIExpression()), !dbg !38
-  store ptr %in, ptr %in.addr, align 8
-  call void @llvm.dbg.declare(metadata ptr %in.addr, metadata !39, metadata !DIExpression()), !dbg !40
-  %this1 = load ptr, ptr %this.addr, align 8
-  %a = getelementptr inbounds %struct.A, ptr %this1, i32 0, i32 0, !dbg !41
-  %0 = load ptr, ptr %in.addr, align 8, !dbg !42
-  call void @llvm.memcpy.p0.p0.i64(ptr align 4 %a, ptr align 4 %0, i64 4, i1 false), !dbg !41
-  ret void, !dbg !43
-}
-
-define dso_local noundef i32 @main() !dbg !44 {
-entry:
-  %objB.i = alloca %struct.B, align 4
-  %objA.i = alloca %struct.A, align 4
-  call void @llvm.dbg.declare(metadata ptr %objB.i, metadata !30, metadata !DIExpression()), !dbg !47
-  call void @llvm.dbg.declare(metadata ptr %objA.i, metadata !32, metadata !DIExpression()), !dbg !49
-  call void @_ZN1AIZ3foovE1BEC2ERS0_(ptr noundef nonnull align 4 dereferenceable(4) %objA.i, ptr noundef nonnull align 4 dereferenceable(4) %objB.i), !dbg !49
-  ret i32 0, !dbg !50
-}
-
-declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg)
-
-!llvm.dbg.cu = !{!0}
-!llvm.module.flags = !{!22, !23, !24, !25, !26, !27, !28}
-!llvm.ident = !{!29}
-
-!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus_14, file: !1, producer: "clang version 15.0.0", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, retainedTypes: !2, splitDebugInlining: false, nameTableKind: None)
-!1 = !DIFile(filename: "test.cpp", directory: "/", checksumkind: CSK_MD5, checksum: "aec7fd397e86f8655ef7f4bb4233b849")
-!2 = !{!3}
-!3 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "A<B>", file: !1, line: 2, size: 32, flags: DIFlagTypePassByValue | DIFlagNonTrivial, elements: !4, templateParams: !20)
-!4 = !{!5, !15}
-!5 = !DIDerivedType(tag: DW_TAG_member, name: "a", scope: !3, file: !1, line: 4, baseType: !6, size: 32)
-!6 = distinct !DICompositeType(tag: DW_TAG_structure_type, name: "B", scope: !7, file: !1, line: 9, size: 32, flags: DIFlagTypePassByValue, elements: !12)
-!7 = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov", scope: !1, file: !1, line: 8, type: !8, scopeLine: 8, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !11)
-!8 = !DISubroutineType(types: !9)
-!9 = !{null}
-!10 = !{}
-!11 = !{!6}
-!12 = !{!13}
-!13 = !DIDerivedType(tag: DW_TAG_member, name: "i", scope: !6, file: !1, line: 9, baseType: !14, size: 32)
-!14 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed)
-!15 = !DISubprogram(name: "A", scope: !3, file: !1, line: 3, type: !16, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagLocalToUnit)
-!16 = !DISubroutineType(types: !17)
-!17 = !{null, !18, !19}
-!18 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !3, size: 64, flags: DIFlagArtificial | DIFlagObjectPointer)
-!19 = !DIDerivedType(tag: DW_TAG_reference_type, baseType: !6, size: 64)
-!20 = !{!21}
-!21 = !DITemplateTypeParameter(name: "T", type: !6)
-!22 = !{i32 7, !"Dwarf Version", i32 5}
-!23 = !{i32 2, !"Debug Info Version", i32 3}
-!24 = !{i32 1, !"wchar_size", i32 4}
-!25 = !{i32 7, !"PIC Level", i32 2}
-!26 = !{i32 7, !"PIE Level", i32 2}
-!27 = !{i32 7, !"uwtable", i32 2}
-!28 = !{i32 7, !"frame-pointer", i32 2}
-!29 = !{!"clang version 15.0.0"}
-!30 = !DILocalVariable(name: "objB", scope: !7, file: !1, line: 10, type: !6)
-!31 = !DILocation(line: 10, column: 5, scope: !7)
-!32 = !DILocalVariable(name: "objA", scope: !7, file: !1, line: 11, type: !3)
-!33 = !DILocation(line: 11, column: 8, scope: !7)
-!34 = !DILocation(line: 12, column: 1, scope: !7)
-!35 = distinct !DISubprogram(name: "A", linkageName: "_ZN1AIZ3foovE1BEC2ERS0_", scope: !3, file: !1, line: 3, type: !16, scopeLine: 3, flags: DIFlagPrototyped, spFlags: DISPFlagLocalToUnit | DISPFlagDefinition, unit: !0, declaration: !15, retainedNodes: !10)
-!36 = !DILocalVariable(name: "this", arg: 1, scope: !35, type: !37, flags: DIFlagArtificial | DIFlagObjectPointer)
-!37 = !DIDerivedType(tag: DW_TAG_pointer_type, baseType: !3, size: 64)
-!38 = !DILocation(line: 0, scope: !35)
-!39 = !DILocalVariable(name: "in", arg: 2, scope: !35, file: !1, line: 3, type: !19)
-!40 = !DILocation(line: 3, column: 8, scope: !35)
-!41 = !DILocation(line: 3, column: 14, scope: !35)
-!42 = !DILocation(line: 3, column: 16, scope: !35)
-!43 = !DILocation(line: 3, column: 21, scope: !35)
-!44 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 14, type: !45, scopeLine: 14, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0, retainedNodes: !10)
-!45 = !DISubroutineType(types: !46)
-!46 = !{!14}
-!47 = !DILocation(line: 10, column: 5, scope: !7, inlinedAt: !48)
-!48 = distinct !DILocation(line: 15, column: 3, scope: !44)
-!49 = !DILocation(line: 11, column: 8, scope: !7, inlinedAt: !48)
-!50 = !DILocation(line: 16, column: 1, scope: !44)
diff --git a/llvm/test/DebugInfo/X86/set.ll b/llvm/test/DebugInfo/X86/set.ll
index 7dbec4520ed6c..292c7c6e4a577 100644
--- a/llvm/test/DebugInfo/X86/set.ll
+++ b/llvm/test/DebugInfo/X86/set.ll
@@ -68,11 +68,11 @@ attributes #1 = { nofree nosync nounwind readnone speculatable willreturn }
 !llvm.module.flags = !{!18, !19, !20}
 
 !0 = !{!"versions- cm3: d5.10.0 llvm: 9.0"}
-!1 = distinct !DICompileUnit(language: DW_LANG_Modula3, file: !2, producer: "cm3", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = distinct !DICompileUnit(language: DW_LANG_Modula3, file: !2, producer: "cm3", isOptimized: false, runtimeVersion: 0, emissionKind: FullDebug, enums: !3)
 !2 = !DIFile(filename: "Main.m3", directory: "/home/cm3/settest/src")
 !3 = !{!4}
 !4 = !DICompositeType(tag: DW_TAG_enumeration_type, name: "Enum", scope: !5, file: !2, line: 11, size: 8, align: 8, elements: !9)
-!5 = distinct !DISubprogram(name: "Test", linkageName: "Main__Test", scope: !2, file: !2, line: 11, type: !6, scopeLine: 11, spFlags: DISPFlagDefinition, unit: !1, retainedNodes: !3)
+!5 = distinct !DISubprogram(name: "Test", linkageName: "Main__Test", scope: !2, file: !2, line: 11, type: !6, scopeLine: 11, spFlags: DISPFlagDefinition, unit: !1, retainedNodes: !8)
 !6 = !DISubroutineType(types: !7)
 !7 = !{null}
 !8 = !{}

From 149f309f50cd981ed320c948f772aa571eaa9afa Mon Sep 17 00:00:00 2001
From: Haojian Wu <hokein.wu@gmail.com>
Date: Tue, 20 Jun 2023 13:10:45 +0200
Subject: [PATCH 117/130] [include-cleaner] Ignore the ParmVarDecl itself in
 WalkAST.cpp

This fixes a false positive where a ParamVarDecl happend to be the
same name of some C standard symbol and has a global namespace.

```
using A = int(int time); // we suggest <ctime> for the `int time`.
```

Differential Revision: https://reviews.llvm.org/D153330
---
 clang-tools-extra/include-cleaner/lib/WalkAST.cpp            | 4 ++++
 clang-tools-extra/include-cleaner/unittests/AnalysisTest.cpp | 4 ++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/clang-tools-extra/include-cleaner/lib/WalkAST.cpp b/clang-tools-extra/include-cleaner/lib/WalkAST.cpp
index fc392fec36865..8cfda506fc254 100644
--- a/clang-tools-extra/include-cleaner/lib/WalkAST.cpp
+++ b/clang-tools-extra/include-cleaner/lib/WalkAST.cpp
@@ -214,6 +214,10 @@ class ASTWalker : public RecursiveASTVisitor<ASTWalker> {
     return true;
   }
   bool VisitVarDecl(VarDecl *VD) {
+    // Ignore the parameter decl itself (its children were handled elsewhere),
+    // as they don't contribute to the main-file #include.
+    if (llvm::isa<ParmVarDecl>(VD))
+      return true;
     // Mark declaration from definition as it needs type-checking.
     if (VD->isThisDeclarationADefinition())
       report(VD->getLocation(), VD);
diff --git a/clang-tools-extra/include-cleaner/unittests/AnalysisTest.cpp b/clang-tools-extra/include-cleaner/unittests/AnalysisTest.cpp
index bad55e1433549..5b5f77b5fdea8 100644
--- a/clang-tools-extra/include-cleaner/unittests/AnalysisTest.cpp
+++ b/clang-tools-extra/include-cleaner/unittests/AnalysisTest.cpp
@@ -91,7 +91,8 @@ TEST_F(WalkUsedTest, Basic) {
   #include "header.h"
   #include "private.h"
 
-  void $bar^bar($private^Private $p^p) {
+  // No reference reported for the Parameter "p".
+  void $bar^bar($private^Private p) {
     $foo^foo();
     std::$vector^vector $vconstructor^$v^v;
     $builtin^__builtin_popcount(1);
@@ -120,7 +121,6 @@ TEST_F(WalkUsedTest, Basic) {
       offsetToProviders(AST, SM),
       UnorderedElementsAre(
           Pair(Code.point("bar"), UnorderedElementsAre(MainFile)),
-          Pair(Code.point("p"), UnorderedElementsAre(MainFile)),
           Pair(Code.point("private"),
                UnorderedElementsAre(PublicFile, PrivateFile)),
           Pair(Code.point("foo"), UnorderedElementsAre(HeaderFile)),

From 87da4b6fa6ead0a0bb823536625c596c75516f2c Mon Sep 17 00:00:00 2001
From: Matthias Springer <me@m-sp.org>
Date: Tue, 20 Jun 2023 14:39:40 +0200
Subject: [PATCH 118/130] [mlir][Pass] Check supported op types before running
 a pass

Add extra error checking to prevent passes from being run on unsupported ops through the pass manager infrastructure.

Differential Revision: https://reviews.llvm.org/D153144
---
 mlir/lib/Pass/Pass.cpp                        |  3 +++
 .../Transform/test-pass-application.mlir      | 19 +++++++++++++++++++
 mlir/test/Pass/pipeline-invalid.mlir          | 12 +++++++++++-
 3 files changed, 33 insertions(+), 1 deletion(-)

diff --git a/mlir/lib/Pass/Pass.cpp b/mlir/lib/Pass/Pass.cpp
index e496a29e9fbe5..c7738a2248a10 100644
--- a/mlir/lib/Pass/Pass.cpp
+++ b/mlir/lib/Pass/Pass.cpp
@@ -438,6 +438,9 @@ LogicalResult OpToOpPassAdaptor::run(Pass *pass, Operation *op,
   if (!opInfo->hasTrait<OpTrait::IsIsolatedFromAbove>())
     return op->emitOpError() << "trying to schedule a pass on an operation not "
                                 "marked as 'IsolatedFromAbove'";
+  if (!pass->canScheduleOn(*op->getName().getRegisteredInfo()))
+    return op->emitOpError()
+           << "trying to schedule a pass on an unsupported operation";
 
   // Initialize the pass state with a callback for the pass to dynamically
   // execute a pipeline on the currently visited operation.
diff --git a/mlir/test/Dialect/Transform/test-pass-application.mlir b/mlir/test/Dialect/Transform/test-pass-application.mlir
index 720f1eb79a1a0..6342657b61280 100644
--- a/mlir/test/Dialect/Transform/test-pass-application.mlir
+++ b/mlir/test/Dialect/Transform/test-pass-application.mlir
@@ -70,3 +70,22 @@ transform.sequence failures(propagate) {
   %1 = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
   transform.apply_registered_pass "canonicalize" to %1 {options = "top-down=false"} : (!transform.any_op) -> !transform.any_op
 }
+
+// -----
+
+module {
+  // expected-error @below {{trying to schedule a pass on an unsupported operation}}
+  // expected-note @below {{target op}}
+  func.func @invalid_target_op_type() {
+    return
+  }
+
+  transform.sequence failures(propagate) {
+  ^bb1(%arg1: !transform.any_op):
+    %1 = transform.structured.match ops{["func.func"]} in %arg1 : (!transform.any_op) -> !transform.any_op
+
+    // func-bufferize can be applied only to ModuleOps.
+    // expected-error @below {{pass pipeline failed}}
+    transform.apply_registered_pass "func-bufferize" to %1 : (!transform.any_op) -> !transform.any_op
+  }
+}
diff --git a/mlir/test/Pass/pipeline-invalid.mlir b/mlir/test/Pass/pipeline-invalid.mlir
index dcc4f0a20c9dc..f9dd4c29dd7f0 100644
--- a/mlir/test/Pass/pipeline-invalid.mlir
+++ b/mlir/test/Pass/pipeline-invalid.mlir
@@ -1,4 +1,9 @@
-// RUN: mlir-opt --no-implicit-module --canonicalize --verify-diagnostics --split-input-file %s
+// RUN: mlir-opt --no-implicit-module \
+// RUN:     --pass-pipeline='any(buffer-deallocation)' --verify-diagnostics \
+// RUN:     --split-input-file %s
+
+// Note: "buffer-deallocation" is a function pass. Any other function pass could
+// be used for this test.
 
 // expected-error@below {{trying to schedule a pass on an operation not marked as 'IsolatedFromAbove'}}
 arith.constant 0
@@ -7,3 +12,8 @@ arith.constant 0
 
 // expected-error@below {{trying to schedule a pass on an unregistered operation}}
 "test.op"() : () -> ()
+
+// -----
+
+// expected-error@below {{trying to schedule a pass on an unsupported operation}}
+module {}

From 8784b6a8540f4a333690e7233587859f1d82f620 Mon Sep 17 00:00:00 2001
From: Joseph Huber <jhuber6@vols.utk.edu>
Date: Wed, 7 Jun 2023 13:11:19 -0500
Subject: [PATCH 119/130] [Clang] Allow bitcode linking when the input is
 LLVM-IR

Clang provides the `-mlink-bitcode-file` and `-mlink-builtin-bitcode`
options to insert LLVM-IR into the current TU. These are usefuly
primarily for including LLVM-IR files that require special handling to
be correct and cannot be linked normally, such as GPU vendor libraries
like `libdevice.10.bc`. Currently these options can only be used if the
source input goes through the AST consumer path. This patch makes the
changes necessary to also support this when the input is LLVM-IR. This
will allow the following operation:

```
clang in.bc -Xclang -mlink-builtin-bitcode -Xclang libdevice.10.bc
```

Reviewed By: yaxunl

Differential Revision: https://reviews.llvm.org/D152391
---
 clang/include/clang/CodeGen/CodeGenAction.h |   3 +
 clang/lib/CodeGen/CGCall.cpp                | 100 +++++++++++++-------
 clang/lib/CodeGen/CGCall.h                  |   9 ++
 clang/lib/CodeGen/CodeGenAction.cpp         |  79 ++++++++++------
 clang/test/CodeGen/link-bitcode-file.c      |   8 ++
 clang/test/CodeGen/link-builtin-bitcode.c   |  42 ++++++++
 6 files changed, 178 insertions(+), 63 deletions(-)
 create mode 100644 clang/test/CodeGen/link-builtin-bitcode.c

diff --git a/clang/include/clang/CodeGen/CodeGenAction.h b/clang/include/clang/CodeGen/CodeGenAction.h
index b5721344046d0..821e80919fc84 100644
--- a/clang/include/clang/CodeGen/CodeGenAction.h
+++ b/clang/include/clang/CodeGen/CodeGenAction.h
@@ -53,6 +53,9 @@ class CodeGenAction : public ASTFrontendAction {
 
   std::unique_ptr<llvm::Module> loadModule(llvm::MemoryBufferRef MBRef);
 
+  /// Load bitcode modules to link into our module from the options.
+  bool loadLinkModules(CompilerInstance &CI);
+
 protected:
   /// Create a new code generation action.  If the optional \p _VMContext
   /// parameter is supplied, the action uses it without taking ownership,
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index fc77566c790c1..61029c9226d7b 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -1845,8 +1845,9 @@ addMergableDefaultFunctionAttributes(const CodeGenOptions &CodeGenOpts,
                        FuncAttrs);
 }
 
-void CodeGenModule::getTrivialDefaultFunctionAttributes(
-    StringRef Name, bool HasOptnone, bool AttrOnCallSite,
+static void getTrivialDefaultFunctionAttributes(
+    StringRef Name, bool HasOptnone, const CodeGenOptions &CodeGenOpts,
+    const LangOptions &LangOpts, bool AttrOnCallSite,
     llvm::AttrBuilder &FuncAttrs) {
   // OptimizeNoneAttr takes precedence over -Os or -Oz. No warning needed.
   if (!HasOptnone) {
@@ -1967,7 +1968,7 @@ void CodeGenModule::getTrivialDefaultFunctionAttributes(
     }
   }
 
-  if (getLangOpts().assumeFunctionsAreConvergent()) {
+  if (LangOpts.assumeFunctionsAreConvergent()) {
     // Conservatively, mark all functions and calls in CUDA and OpenCL as
     // convergent (meaning, they may call an intrinsically convergent op, such
     // as __syncthreads() / barrier(), and so can't have certain optimizations
@@ -1978,8 +1979,8 @@ void CodeGenModule::getTrivialDefaultFunctionAttributes(
 
   // TODO: NoUnwind attribute should be added for other GPU modes HIP,
   // OpenMP offload. AFAIK, neither of them support exceptions in device code.
-  if ((getLangOpts().CUDA && getLangOpts().CUDAIsDevice) ||
-      getLangOpts().OpenCL || getLangOpts().SYCLIsDevice) {
+  if ((LangOpts.CUDA && LangOpts.CUDAIsDevice) || LangOpts.OpenCL ||
+      LangOpts.SYCLIsDevice) {
     FuncAttrs.addAttribute(llvm::Attribute::NoUnwind);
   }
 
@@ -1990,36 +1991,25 @@ void CodeGenModule::getTrivialDefaultFunctionAttributes(
   }
 }
 
-void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
-                                                 bool HasOptnone,
-                                                 bool AttrOnCallSite,
-                                                 llvm::AttrBuilder &FuncAttrs) {
-  getTrivialDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite,
-                                      FuncAttrs);
-  if (!AttrOnCallSite) {
-    // If we're just getting the default, get the default values for mergeable
-    // attributes.
-    addMergableDefaultFunctionAttributes(CodeGenOpts, FuncAttrs);
-  }
-}
+/// Adds attributes to \p F according to our \p CodeGenOpts and \p LangOpts, as
+/// though we had emitted it ourselves. We remove any attributes on F that
+/// conflict with the attributes we add here.
+static void mergeDefaultFunctionDefinitionAttributes(
+    llvm::Function &F, const CodeGenOptions CodeGenOpts,
+    const LangOptions &LangOpts, const TargetOptions &TargetOpts,
+    bool WillInternalize) {
 
-void CodeGenModule::addDefaultFunctionDefinitionAttributes(llvm::Function &F) {
   llvm::AttrBuilder FuncAttrs(F.getContext());
-  getDefaultFunctionAttributes(F.getName(), F.hasOptNone(),
-                               /* AttrOnCallSite = */ false, FuncAttrs);
-  // TODO: call GetCPUAndFeaturesAttributes?
-  F.addFnAttrs(FuncAttrs);
-}
+  // Here we only extract the options that are relevant compared to the version
+  // from GetCPUAndFeaturesAttributes.
+  if (!TargetOpts.CPU.empty())
+    FuncAttrs.addAttribute("target-cpu", TargetOpts.CPU);
+  if (!TargetOpts.TuneCPU.empty())
+    FuncAttrs.addAttribute("tune-cpu", TargetOpts.TuneCPU);
 
-/// Apply default attributes to \p F, accounting for merge semantics of
-/// attributes that should not overwrite existing attributes.
-void CodeGenModule::mergeDefaultFunctionDefinitionAttributes(
-    llvm::Function &F, bool WillInternalize) {
-  llvm::AttrBuilder FuncAttrs(F.getContext());
-  getTrivialDefaultFunctionAttributes(F.getName(), F.hasOptNone(),
-                                      /*AttrOnCallSite=*/false, FuncAttrs);
-  GetCPUAndFeaturesAttributes(GlobalDecl(), FuncAttrs,
-                              /*AddTargetFeatures=*/false);
+  ::getTrivialDefaultFunctionAttributes(F.getName(), F.hasOptNone(),
+                                        CodeGenOpts, LangOpts,
+                                        /*AttrOnCallSite=*/false, FuncAttrs);
 
   if (!WillInternalize && F.isInterposable()) {
     // Do not promote "dynamic" denormal-fp-math to this translation unit's
@@ -2064,6 +2054,52 @@ void CodeGenModule::mergeDefaultFunctionDefinitionAttributes(
   F.addFnAttrs(FuncAttrs);
 }
 
+void clang::CodeGen::mergeDefaultFunctionDefinitionAttributes(
+    llvm::Function &F, const CodeGenOptions CodeGenOpts,
+    const LangOptions &LangOpts, const TargetOptions &TargetOpts,
+    bool WillInternalize) {
+
+  ::mergeDefaultFunctionDefinitionAttributes(F, CodeGenOpts, LangOpts,
+                                             TargetOpts, WillInternalize);
+}
+
+void CodeGenModule::getTrivialDefaultFunctionAttributes(
+    StringRef Name, bool HasOptnone, bool AttrOnCallSite,
+    llvm::AttrBuilder &FuncAttrs) {
+  ::getTrivialDefaultFunctionAttributes(Name, HasOptnone, getCodeGenOpts(),
+                                        getLangOpts(), AttrOnCallSite,
+                                        FuncAttrs);
+}
+
+void CodeGenModule::getDefaultFunctionAttributes(StringRef Name,
+                                                 bool HasOptnone,
+                                                 bool AttrOnCallSite,
+                                                 llvm::AttrBuilder &FuncAttrs) {
+  getTrivialDefaultFunctionAttributes(Name, HasOptnone, AttrOnCallSite,
+                                      FuncAttrs);
+  // If we're just getting the default, get the default values for mergeable
+  // attributes.
+  if (!AttrOnCallSite)
+    addMergableDefaultFunctionAttributes(CodeGenOpts, FuncAttrs);
+}
+
+void CodeGenModule::addDefaultFunctionDefinitionAttributes(llvm::Function &F) {
+  llvm::AttrBuilder FuncAttrs(F.getContext());
+  getDefaultFunctionAttributes(F.getName(), F.hasOptNone(),
+                               /* AttrOnCallSite = */ false, FuncAttrs);
+  // TODO: call GetCPUAndFeaturesAttributes?
+  F.addFnAttrs(FuncAttrs);
+}
+
+/// Apply default attributes to \p F, accounting for merge semantics of
+/// attributes that should not overwrite existing attributes.
+void CodeGenModule::mergeDefaultFunctionDefinitionAttributes(
+    llvm::Function &F, bool WillInternalize) {
+  ::mergeDefaultFunctionDefinitionAttributes(F, getCodeGenOpts(), getLangOpts(),
+                                             getTarget().getTargetOpts(),
+                                             WillInternalize);
+}
+
 void CodeGenModule::addDefaultFunctionDefinitionAttributes(
     llvm::AttrBuilder &attrs) {
   getDefaultFunctionAttributes(/*function name*/ "", /*optnone*/ false,
diff --git a/clang/lib/CodeGen/CGCall.h b/clang/lib/CodeGen/CGCall.h
index 59c3f304f59b9..824f0a9a88299 100644
--- a/clang/lib/CodeGen/CGCall.h
+++ b/clang/lib/CodeGen/CGCall.h
@@ -30,6 +30,7 @@ class Value;
 namespace clang {
 class Decl;
 class FunctionDecl;
+class TargetOptions;
 class VarDecl;
 
 namespace CodeGen {
@@ -377,6 +378,14 @@ class ReturnValueSlot {
   bool isExternallyDestructed() const { return IsExternallyDestructed; }
 };
 
+/// Helper to add attributes to \p F according to the CodeGenOptions and
+/// LangOptions without requiring a CodeGenModule to be constructed.
+void mergeDefaultFunctionDefinitionAttributes(llvm::Function &F,
+                                              const CodeGenOptions CodeGenOpts,
+                                              const LangOptions &LangOpts,
+                                              const TargetOptions &TargetOpts,
+                                              bool WillInternalize);
+
 } // end namespace CodeGen
 } // end namespace clang
 
diff --git a/clang/lib/CodeGen/CodeGenAction.cpp b/clang/lib/CodeGen/CodeGenAction.cpp
index 4aa51e956655f..4879bcd6a42a5 100644
--- a/clang/lib/CodeGen/CodeGenAction.cpp
+++ b/clang/lib/CodeGen/CodeGenAction.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/CodeGen/CodeGenAction.h"
+#include "CGCall.h"
 #include "CodeGenModule.h"
 #include "CoverageMappingGen.h"
 #include "MacroPPCallbacks.h"
@@ -262,7 +263,7 @@ namespace clang {
     }
 
     // Links each entry in LinkModules into our module.  Returns true on error.
-    bool LinkInModules() {
+    bool LinkInModules(llvm::Module *M) {
       for (auto &LM : LinkModules) {
         assert(LM.Module && "LinkModule does not actually have a module");
         if (LM.PropagateAttrs)
@@ -271,8 +272,8 @@ namespace clang {
             // in LLVM IR.
             if (F.isIntrinsic())
               continue;
-            Gen->CGM().mergeDefaultFunctionDefinitionAttributes(F,
-                                                                LM.Internalize);
+            CodeGen::mergeDefaultFunctionDefinitionAttributes(
+                F, CodeGenOpts, LangOpts, TargetOpts, LM.Internalize);
           }
 
         CurLinkModule = LM.Module.get();
@@ -280,15 +281,14 @@ namespace clang {
         bool Err;
         if (LM.Internalize) {
           Err = Linker::linkModules(
-              *getModule(), std::move(LM.Module), LM.LinkFlags,
+              *M, std::move(LM.Module), LM.LinkFlags,
               [](llvm::Module &M, const llvm::StringSet<> &GVS) {
                 internalizeModule(M, [&GVS](const llvm::GlobalValue &GV) {
                   return !GV.hasName() || (GVS.count(GV.getName()) == 0);
                 });
               });
         } else {
-          Err = Linker::linkModules(*getModule(), std::move(LM.Module),
-                                    LM.LinkFlags);
+          Err = Linker::linkModules(*M, std::move(LM.Module), LM.LinkFlags);
         }
 
         if (Err)
@@ -357,7 +357,7 @@ namespace clang {
       }
 
       // Link each LinkModule into our module.
-      if (LinkInModules())
+      if (LinkInModules(getModule()))
         return;
 
       for (auto &F : getModule()->functions()) {
@@ -993,6 +993,36 @@ CodeGenAction::~CodeGenAction() {
     delete VMContext;
 }
 
+bool CodeGenAction::loadLinkModules(CompilerInstance &CI) {
+  if (!LinkModules.empty())
+    return false;
+
+  for (const CodeGenOptions::BitcodeFileToLink &F :
+       CI.getCodeGenOpts().LinkBitcodeFiles) {
+    auto BCBuf = CI.getFileManager().getBufferForFile(F.Filename);
+    if (!BCBuf) {
+      CI.getDiagnostics().Report(diag::err_cannot_open_file)
+          << F.Filename << BCBuf.getError().message();
+      LinkModules.clear();
+      return true;
+    }
+
+    Expected<std::unique_ptr<llvm::Module>> ModuleOrErr =
+        getOwningLazyBitcodeModule(std::move(*BCBuf), *VMContext);
+    if (!ModuleOrErr) {
+      handleAllErrors(ModuleOrErr.takeError(), [&](ErrorInfoBase &EIB) {
+        CI.getDiagnostics().Report(diag::err_cannot_open_file)
+            << F.Filename << EIB.message();
+      });
+      LinkModules.clear();
+      return true;
+    }
+    LinkModules.push_back({std::move(ModuleOrErr.get()), F.PropagateAttrs,
+                           F.Internalize, F.LinkFlags});
+  }
+  return false;
+}
+
 bool CodeGenAction::hasIRSupport() const { return true; }
 
 void CodeGenAction::EndSourceFileAction() {
@@ -1048,30 +1078,8 @@ CodeGenAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
     return nullptr;
 
   // Load bitcode modules to link with, if we need to.
-  if (LinkModules.empty())
-    for (const CodeGenOptions::BitcodeFileToLink &F :
-         CI.getCodeGenOpts().LinkBitcodeFiles) {
-      auto BCBuf = CI.getFileManager().getBufferForFile(F.Filename);
-      if (!BCBuf) {
-        CI.getDiagnostics().Report(diag::err_cannot_open_file)
-            << F.Filename << BCBuf.getError().message();
-        LinkModules.clear();
-        return nullptr;
-      }
-
-      Expected<std::unique_ptr<llvm::Module>> ModuleOrErr =
-          getOwningLazyBitcodeModule(std::move(*BCBuf), *VMContext);
-      if (!ModuleOrErr) {
-        handleAllErrors(ModuleOrErr.takeError(), [&](ErrorInfoBase &EIB) {
-          CI.getDiagnostics().Report(diag::err_cannot_open_file)
-              << F.Filename << EIB.message();
-        });
-        LinkModules.clear();
-        return nullptr;
-      }
-      LinkModules.push_back({std::move(ModuleOrErr.get()), F.PropagateAttrs,
-                             F.Internalize, F.LinkFlags});
-    }
+  if (loadLinkModules(CI))
+    return nullptr;
 
   CoverageSourceInfo *CoverageInfo = nullptr;
   // Add the preprocessor callback only when the coverage mapping is generated.
@@ -1139,6 +1147,10 @@ CodeGenAction::loadModule(MemoryBufferRef MBRef) {
     return std::move(*MOrErr);
   }
 
+  // Load bitcode modules to link with, if we need to.
+  if (loadLinkModules(CI))
+    return nullptr;
+
   llvm::SMDiagnostic Err;
   if (std::unique_ptr<llvm::Module> M = parseIR(MBRef, Err, *VMContext))
     return M;
@@ -1218,6 +1230,11 @@ void CodeGenAction::ExecuteAction() {
                          CI.getCodeGenOpts(), CI.getTargetOpts(),
                          CI.getLangOpts(), TheModule.get(),
                          std::move(LinkModules), *VMContext, nullptr);
+
+  // Link in each pending link module.
+  if (Result.LinkInModules(&*TheModule))
+    return;
+
   // PR44896: Force DiscardValueNames as false. DiscardValueNames cannot be
   // true here because the valued names are needed for reading textual IR.
   Ctx.setDiscardValueNames(false);
diff --git a/clang/test/CodeGen/link-bitcode-file.c b/clang/test/CodeGen/link-bitcode-file.c
index df04ec2bec0d9..58fee64a95138 100644
--- a/clang/test/CodeGen/link-bitcode-file.c
+++ b/clang/test/CodeGen/link-bitcode-file.c
@@ -11,6 +11,14 @@
 // RUN: not %clang_cc1 -triple i386-pc-linux-gnu -mlink-bitcode-file no-such-file.bc \
 // RUN:    -emit-llvm -o - %s 2>&1 | FileCheck -check-prefix=CHECK-NO-FILE %s
 
+// Make sure we can perform the same options if the input is LLVM-IR
+// RUN: %clang_cc1 -triple i386-pc-linux-gnu -emit-llvm-bc -o %t-in.bc %s
+// RUN: %clang_cc1 -triple i386-pc-linux-gnu -mlink-bitcode-file %t.bc \
+// RUN:     -O3 -emit-llvm -o - %t-in.bc | FileCheck -check-prefix=CHECK-NO-BC %s
+// RUN: %clang_cc1 -triple i386-pc-linux-gnu -O3 -emit-llvm -o - \
+// RUN:     -mlink-bitcode-file %t.bc -mlink-bitcode-file %t-2.bc %t-in.bc \
+// RUN:     | FileCheck -check-prefix=CHECK-NO-BC -check-prefix=CHECK-NO-BC2 %s
+
 int f(void);
 
 #ifdef BITCODE
diff --git a/clang/test/CodeGen/link-builtin-bitcode.c b/clang/test/CodeGen/link-builtin-bitcode.c
new file mode 100644
index 0000000000000..b3b54badf3f82
--- /dev/null
+++ b/clang/test/CodeGen/link-builtin-bitcode.c
@@ -0,0 +1,42 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --check-attributes --check-globals --include-generated-funcs --version 2
+// RUN: %clang_cc1 -triple amdgcn-- -target-cpu gfx803 -DBITCODE -emit-llvm-bc -o %t-lib.bc %s
+// RUN: %clang_cc1 -triple amdgcn-- -target-cpu gfx90a -emit-llvm-bc -o %t.bc %s
+// RUN: %clang_cc1 -triple amdgcn-- -target-cpu gfx90a -emit-llvm \
+// RUN:   -mlink-builtin-bitcode %t-lib.bc -o - %t.bc | FileCheck %s
+
+#ifdef BITCODE
+int foo(void) { return 42; }
+int x = 12;
+#endif
+
+extern int foo(void);
+extern int x;
+
+int bar() { return foo() + x; }
+//.
+// CHECK: @x = internal addrspace(1) global i32 12, align 4
+//.
+// CHECK: Function Attrs: noinline nounwind optnone
+// CHECK-LABEL: define dso_local i32 @bar
+// CHECK-SAME: () #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
+// CHECK-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// CHECK-NEXT:    [[CALL:%.*]] = call i32 @foo()
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspacecast (ptr addrspace(1) @x to ptr), align 4
+// CHECK-NEXT:    [[ADD:%.*]] = add nsw i32 [[CALL]], [[TMP0]]
+// CHECK-NEXT:    ret i32 [[ADD]]
+//
+//
+// CHECK: Function Attrs: convergent noinline nounwind optnone
+// CHECK-LABEL: define internal i32 @foo
+// CHECK-SAME: () #[[ATTR1:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[RETVAL:%.*]] = alloca i32, align 4, addrspace(5)
+// CHECK-NEXT:    [[RETVAL_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[RETVAL]] to ptr
+// CHECK-NEXT:    ret i32 42
+//
+//.
+// CHECK: attributes #0 = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="gfx90a" "target-features"="+16-bit-insts,+atomic-buffer-global-pk-add-f16-insts,+atomic-fadd-rtn-insts,+ci-insts,+dl-insts,+dot1-insts,+dot10-insts,+dot2-insts,+dot3-insts,+dot4-insts,+dot5-insts,+dot6-insts,+dot7-insts,+dpp,+gfx8-insts,+gfx9-insts,+gfx90a-insts,+mai-insts,+s-memrealtime,+s-memtime-inst,+wavefrontsize64" }
+// CHECK: attributes #1 = { convergent noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="gfx90a" "target-features"="+16-bit-insts,+ci-insts,+dpp,+gfx8-insts,+s-memrealtime,+s-memtime-inst,+wavefrontsize64" }
+//.

From efacdfc235e327341d2b8a733d9963fb526cf17b Mon Sep 17 00:00:00 2001
From: Joseph Huber <jhuber6@vols.utk.edu>
Date: Thu, 8 Jun 2023 08:28:07 -0500
Subject: [PATCH 120/130] [LinkerWrapper] Support linking vendor bitcode late

The GPU vendors currently provide bitcode files for their device
runtime. These files need to be handled specially as they are not built
to be linked in with a standard `llvm-link` call or through LTO linking.
This patch adds an alternative to use the existing clang handling of
these libraries that does the necessary magic to make this work.

We do this by causing the LTO backend to emit bitcode before running the
backend. We then pass this through to clang which uses the existing
support which has been fixed to support this by D152391. The backend
will then be run with the merged module.

This patch adds the `--builtin-bitcode=<triple>=file.bc` to specify a single
file, or just `--clang-backend` to let the toolchain handle its defaults
(currently nothing for NVPTX and the ROCm device libs for AMDGPU). This may have
a performance impact due to running the optimizations again, we could
potentially disable optimizations in LTO and only do the linking if this is an
issue.

This should allow us to resolve issues when relying on the `linker-wrapper` to
do a late linking that may depend on vendor libraries.

Depends on D152391

Reviewed By: JonChesterfield

Differential Revision: https://reviews.llvm.org/D152442
---
 clang/test/Driver/linker-wrapper.c            |  9 +++++++
 .../ClangLinkerWrapper.cpp                    | 25 ++++++++++++++++---
 .../clang-linker-wrapper/LinkerWrapperOpts.td |  7 ++++++
 3 files changed, 37 insertions(+), 4 deletions(-)

diff --git a/clang/test/Driver/linker-wrapper.c b/clang/test/Driver/linker-wrapper.c
index 97058a96465f8..140cc72886692 100644
--- a/clang/test/Driver/linker-wrapper.c
+++ b/clang/test/Driver/linker-wrapper.c
@@ -130,3 +130,12 @@
 // RUN:   -o a.out 2>&1 | FileCheck %s --check-prefix=MISSING-LIBRARY
 
 // MISSING-LIBRARY: error: unable to find library -ldummy
+
+// RUN: clang-offload-packager -o %t.out \
+// RUN:   --image=file=%t.amdgpu.bc,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx908 \
+// RUN:   --image=file=%t.amdgpu.bc,kind=openmp,triple=amdgcn-amd-amdhsa,arch=gfx908
+// RUN: %clang -cc1 %s -triple x86_64-unknown-linux-gnu -emit-obj -o %t.o -fembed-offload-object=%t.out
+// RUN: clang-linker-wrapper --host-triple=x86_64-unknown-linux-gnu --dry-run --clang-backend \
+// RUN:   --linker-path=/usr/bin/ld -- %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=CLANG-BACKEND
+
+// CLANG-BACKEND: clang{{.*}} -o {{.*}}.img --target=amdgcn-amd-amdhsa -mcpu=gfx908 -O2 -Wl,--no-undefined {{.*}}.bc
diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
index 0af0f2e371b18..c553cf86da8e3 100644
--- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
+++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
@@ -427,6 +427,17 @@ Expected<StringRef> clang(ArrayRef<StringRef> InputFiles, const ArgList &Args) {
   for (StringRef Arg : Args.getAllArgValues(OPT_linker_arg_EQ))
     CmdArgs.push_back(Args.MakeArgString("-Wl," + Arg));
 
+  for (StringRef Arg : Args.getAllArgValues(OPT_builtin_bitcode_EQ)) {
+    if (llvm::Triple(Arg.split('=').first) == Triple)
+      CmdArgs.append({"-Xclang", "-mlink-builtin-bitcode", "-Xclang",
+                      Args.MakeArgString(Arg.split('=').second)});
+  }
+
+  // The OpenMPOpt pass can introduce new calls and is expensive, we do not want
+  // this when running CodeGen through clang.
+  if (Args.hasArg(OPT_clang_backend) || Args.hasArg(OPT_builtin_bitcode_EQ))
+    CmdArgs.append({"-mllvm", "-openmp-opt-disable"});
+
   if (Error Err = executeCommands(*ClangPath, CmdArgs))
     return std::move(Err);
 
@@ -629,7 +640,7 @@ Error linkBitcodeFiles(SmallVectorImpl<OffloadFile> &InputFiles,
   llvm::erase_if(InputFiles, [](OffloadFile &F) { return !F.getBinary(); });
 
   // LTO Module hook to output bitcode without running the backend.
-  SmallVector<StringRef, 4> BitcodeOutput;
+  SmallVector<StringRef> BitcodeOutput;
   auto OutputBitcode = [&](size_t, const Module &M) {
     auto TempFileOrErr = createOutputFile(sys::path::filename(ExecutableName) +
                                               "-jit-" + Triple.getTriple(),
@@ -648,7 +659,9 @@ Error linkBitcodeFiles(SmallVectorImpl<OffloadFile> &InputFiles,
 
   // We assume visibility of the whole program if every input file was bitcode.
   auto Features = getTargetFeatures(BitcodeInputFiles);
-  auto LTOBackend = Args.hasArg(OPT_embed_bitcode)
+  auto LTOBackend = Args.hasArg(OPT_embed_bitcode) ||
+                            Args.hasArg(OPT_builtin_bitcode_EQ) ||
+                            Args.hasArg(OPT_clang_backend)
                         ? createLTO(Args, Features, OutputBitcode)
                         : createLTO(Args, Features);
 
@@ -757,8 +770,12 @@ Error linkBitcodeFiles(SmallVectorImpl<OffloadFile> &InputFiles,
     return Error::success();
   }
 
-  // Append the new inputs to the device linker input.
-  for (StringRef File : Files)
+  // Append the new inputs to the device linker input. If the user requested an
+  // internalizing link we need to pass the bitcode to clang.
+  for (StringRef File :
+       Args.hasArg(OPT_clang_backend) || Args.hasArg(OPT_builtin_bitcode_EQ)
+           ? BitcodeOutput
+           : Files)
     OutputFiles.push_back(File);
 
   return Error::success();
diff --git a/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td b/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td
index 5dace9766e0c1..abab4d0b39b90 100644
--- a/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td
+++ b/clang/tools/clang-linker-wrapper/LinkerWrapperOpts.td
@@ -25,9 +25,16 @@ def opt_level : Joined<["--"], "opt-level=">,
 def bitcode_library_EQ : Joined<["--"], "bitcode-library=">,
   Flags<[WrapperOnlyOption]>, MetaVarName<"<kind>-<triple>-<arch>=<path>">,
   HelpText<"Extra bitcode library to link">;
+def builtin_bitcode_EQ : Joined<["--"], "builtin-bitcode=">,
+  Flags<[WrapperOnlyOption]>, MetaVarName<"<triple>=<path>">,
+  HelpText<"Perform a special internalizing link on the bitcode file. "
+           "This is necessary for some vendor libraries to be linked correctly">;
 def device_linker_args_EQ : Joined<["--"], "device-linker=">,
   Flags<[WrapperOnlyOption]>, MetaVarName<"<value> or <triple>=<value>">,
   HelpText<"Arguments to pass to the device linker invocation">;
+def clang_backend : Flag<["--"], "clang-backend">,
+  Flags<[WrapperOnlyOption]>,
+  HelpText<"Run the backend using clang rather than the LTO backend">;
 def dry_run : Flag<["--"], "dry-run">,
   Flags<[WrapperOnlyOption]>,
   HelpText<"Print program arguments without running">;

From 8cf8956897ce9bca3176c6339077b1ca17b27abc Mon Sep 17 00:00:00 2001
From: Simi Pallipurath <simi.pallipurath@arm.com>
Date: Thu, 18 May 2023 12:29:07 +0100
Subject: [PATCH 121/130] [lld][Arm] Big Endian - Byte invariant support.

Arm has BE8 big endian configuration called a byte-invariant(every byte has the same address on little and big-endian systems).

When in BE8 mode:
  1. Instructions are big-endian in relocatable objects but
     little-endian in executables and shared objects.
  2. Data is big-endian.
  3. The data encoding of the ELF file is ELFDATA2MSB.

To support BE8 without an ABI break for relocatable objects,the linker takes on the responsibility of changing the endianness of instructions. At a high level the only difference between BE32 and BE8 in the linker is that for BE8:
  1. The linker sets the flag EF_ARM_BE8 in the ELF header.
  2. The linker endian reverses the instructions, but not data.

This patch adds BE8 big endian support for Arm. To endian reverse the instructions we'll need access to the mapping symbols. Code sections can contain a mix of Arm, Thumb and literal data. We need to endian reverse Arm instructions as words, Thumb instructions
as half-words and ignore literal data.The only way to find these transitions precisely is by using mapping symbols. The instruction reversal will need to take place after relocation. For Arm BE8 code sections (Section has SHF_EXECINSTR flag ) we inserted a step after relocation to endian reverse the instructions. The implementation strategy i have used here is to write all sections BE32  including SyntheticSections then endian reverse all code in InputSections via mapping symbols.

Reviewed By: peter.smith

Differential Revision: https://reviews.llvm.org/D150870
---
 lld/ELF/Arch/ARM.cpp                          | 122 +++++++++++-
 lld/ELF/Config.h                              |   1 +
 lld/ELF/Driver.cpp                            |   6 +-
 lld/ELF/Options.td                            |   2 +
 lld/ELF/OutputSections.cpp                    |   6 +
 lld/ELF/SyntheticSections.cpp                 |   7 +
 lld/ELF/Target.h                              |   4 +
 lld/ELF/Thunks.cpp                            |  30 ++-
 lld/ELF/Writer.cpp                            |   5 +
 lld/docs/ld.lld.1                             |   2 +
 lld/test/ELF/arm-bl-v6.s                      |   6 +
 lld/test/ELF/arm-data-relocs.s                |   3 +
 lld/test/ELF/arm-exidx-emit-relocs.s          |   4 +
 lld/test/ELF/arm-exidx-relocatable.s          |   2 +
 .../ELF/arm-exidx-sentinel-norelocatable.s    |   2 +
 lld/test/ELF/arm-header.s                     |  24 +++
 lld/test/ELF/arm-mov-relocs.s                 |   5 +
 lld/test/ELF/arm-plt-reloc.s                  | 179 ++++++++++++------
 lld/test/ELF/arm-thumb-plt-reloc.s            |   6 +
 lld/test/ELF/arm-thunk-arm-thumb-reuse.s      |   2 +
 lld/test/ELF/arm-thunk-edgecase.s             |   5 +
 lld/test/ELF/arm-thunk-largesection.s         |   7 +
 lld/test/ELF/arm-thunk-linkerscript-dotexpr.s |   3 +
 lld/test/ELF/arm-thunk-linkerscript.s         |   2 +
 lld/test/ELF/arm-thunk-nosuitable.s           |   2 +
 lld/test/ELF/arm-thunk-re-add.s               |   7 +
 lld/test/ELF/arm-thunk-reuse.s                |  23 +++
 lld/test/ELF/arm-thunk-section-too-large.s    |   1 +
 lld/test/ELF/arm-thunk-toolargesection.s      |   1 +
 lld/test/ELF/arm-v5-reloc-error.s             |   1 +
 lld/test/ELF/emulation-arm.s                  |  27 +++
 lld/test/ELF/target-specific-options.s        |   3 +
 32 files changed, 432 insertions(+), 68 deletions(-)

diff --git a/lld/ELF/Arch/ARM.cpp b/lld/ELF/Arch/ARM.cpp
index 79635a22df4ec..cdab2a40e2183 100644
--- a/lld/ELF/Arch/ARM.cpp
+++ b/lld/ELF/Arch/ARM.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "InputFiles.h"
 #include "Symbols.h"
 #include "SyntheticSections.h"
 #include "Target.h"
@@ -44,8 +45,11 @@ class ARM final : public TargetInfo {
   void relocate(uint8_t *loc, const Relocation &rel,
                 uint64_t val) const override;
 };
+enum class CodeState { Data = 0, Thumb = 2, Arm = 4 };
 } // namespace
 
+static DenseMap<InputSection *, SmallVector<const Defined *, 0>> sectionMap;
+
 ARM::ARM() {
   copyRel = R_ARM_COPY;
   relativeRel = R_ARM_RELATIVE;
@@ -68,16 +72,24 @@ uint32_t ARM::calcEFlags() const {
   // The ABIFloatType is used by loaders to detect the floating point calling
   // convention.
   uint32_t abiFloatType = 0;
+
+  // Set the EF_ARM_BE8 flag in the ELF header, if ELF file is big-endian
+  // with BE-8 code.
+  uint32_t armBE8 = 0;
+
   if (config->armVFPArgs == ARMVFPArgKind::Base ||
       config->armVFPArgs == ARMVFPArgKind::Default)
     abiFloatType = EF_ARM_ABI_FLOAT_SOFT;
   else if (config->armVFPArgs == ARMVFPArgKind::VFP)
     abiFloatType = EF_ARM_ABI_FLOAT_HARD;
 
+  if (!config->isLE && config->armBe8)
+    armBE8 = EF_ARM_BE8;
+
   // We don't currently use any features incompatible with EF_ARM_EABI_VER5,
   // but we don't have any firm guarantees of conformance. Linux AArch64
   // kernels (as of 2016) require an EABI version to be set.
-  return EF_ARM_EABI_VER5 | abiFloatType;
+  return EF_ARM_EABI_VER5 | abiFloatType | armBE8;
 }
 
 RelExpr ARM::getRelExpr(RelType type, const Symbol &s,
@@ -910,6 +922,114 @@ int64_t ARM::getImplicitAddend(const uint8_t *buf, RelType type) const {
   }
 }
 
+static bool isArmMapSymbol(const Symbol *b) {
+  return b->getName() == "$a" || b->getName().startswith("$a.");
+}
+
+static bool isThumbMapSymbol(const Symbol *s) {
+  return s->getName() == "$t" || s->getName().startswith("$t.");
+}
+
+static bool isDataMapSymbol(const Symbol *b) {
+  return b->getName() == "$d" || b->getName().startswith("$d.");
+}
+
+void elf::sortArmMappingSymbols() {
+  // For each input section make sure the mapping symbols are sorted in
+  // ascending order.
+  for (auto &kv : sectionMap) {
+    SmallVector<const Defined *, 0> &mapSyms = kv.second;
+    llvm::stable_sort(mapSyms, [](const Defined *a, const Defined *b) {
+      return a->value < b->value;
+    });
+  }
+}
+
+void elf::addArmInputSectionMappingSymbols() {
+  // Collect mapping symbols for every executable input sections.
+  // The linker generated mapping symbols for all the synthetic
+  // sections are adding into the sectionmap through the function
+  // addArmSyntheitcSectionMappingSymbol.
+  for (ELFFileBase *file : ctx.objectFiles) {
+    for (Symbol *sym : file->getLocalSymbols()) {
+      auto *def = dyn_cast<Defined>(sym);
+      if (!def)
+        continue;
+      if (!isArmMapSymbol(def) && !isDataMapSymbol(def) &&
+          !isThumbMapSymbol(def))
+        continue;
+      if (auto *sec = cast_if_present<InputSection>(def->section))
+        if (sec->flags & SHF_EXECINSTR)
+          sectionMap[sec].push_back(def);
+    }
+  }
+}
+
+// Synthetic sections are not backed by an ELF file where we can access the
+// symbol table, instead mapping symbols added to synthetic sections are stored
+// in the synthetic symbol table. Due to the presence of strip (--strip-all),
+// we can not rely on the synthetic symbol table retaining the mapping symbols.
+// Instead we record the mapping symbols locally.
+void elf::addArmSyntheticSectionMappingSymbol(Defined *sym) {
+  if (!isArmMapSymbol(sym) && !isDataMapSymbol(sym) && !isThumbMapSymbol(sym))
+    return;
+  if (auto *sec = cast_if_present<InputSection>(sym->section))
+    if (sec->flags & SHF_EXECINSTR)
+      sectionMap[sec].push_back(sym);
+}
+
+static void toLittleEndianInstructions(uint8_t *buf, uint64_t start,
+                                       uint64_t end, uint64_t width) {
+  CodeState curState = static_cast<CodeState>(width);
+  if (curState == CodeState::Arm)
+    for (uint64_t i = start; i < end; i += width)
+      write32le(buf + i, read32(buf + i));
+
+  if (curState == CodeState::Thumb)
+    for (uint64_t i = start; i < end; i += width)
+      write16le(buf + i, read16(buf + i));
+}
+
+// Arm BE8 big endian format requires instructions to be little endian, with
+// the initial contents big-endian. Convert the big-endian instructions to
+// little endian leaving literal data untouched. We use mapping symbols to
+// identify half open intervals of Arm code [$a, non $a) and Thumb code
+// [$t, non $t) and convert these to little endian a word or half word at a
+// time respectively.
+void elf::convertArmInstructionstoBE8(InputSection *sec, uint8_t *buf) {
+  SmallVector<const Defined *, 0> &mapSyms = sectionMap[sec];
+
+  if (mapSyms.empty())
+    return;
+
+  CodeState curState = CodeState::Data;
+  uint64_t start = 0, width = 0, size = sec->getSize();
+  for (auto &msym : mapSyms) {
+    CodeState newState = CodeState::Data;
+    if (isThumbMapSymbol(msym))
+      newState = CodeState::Thumb;
+    else if (isArmMapSymbol(msym))
+      newState = CodeState::Arm;
+
+    if (newState == curState)
+      continue;
+
+    if (curState != CodeState::Data) {
+      width = static_cast<uint64_t>(curState);
+      toLittleEndianInstructions(buf, start, msym->value, width);
+    }
+    start = msym->value;
+    curState = newState;
+  }
+
+  // Passed last mapping symbol, may need to reverse
+  // up to end of section.
+  if (curState != CodeState::Data) {
+    width = static_cast<uint64_t>(curState);
+    toLittleEndianInstructions(buf, start, size, width);
+  }
+}
+
 TargetInfo *elf::getARMTargetInfo() {
   static ARM target;
   return &target;
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h
index 956530098c0f1..813102a9bd83a 100644
--- a/lld/ELF/Config.h
+++ b/lld/ELF/Config.h
@@ -201,6 +201,7 @@ struct Config {
   bool armHasMovtMovw = false;
   bool armJ1J2BranchEncoding = false;
   bool asNeeded = false;
+  bool armBe8 = false;
   BsymbolicKind bsymbolic = BsymbolicKind::None;
   bool callGraphProfileSort;
   bool checkSections;
diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp
index 4eb10fc3c91e1..9c813f97bf57c 100644
--- a/lld/ELF/Driver.cpp
+++ b/lld/ELF/Driver.cpp
@@ -351,9 +351,12 @@ static void checkOptions() {
   if (config->fixCortexA8 && config->emachine != EM_ARM)
     error("--fix-cortex-a8 is only supported on ARM targets");
 
+  if (config->armBe8 && config->emachine != EM_ARM)
+    error("--be8 is only supported on ARM targets");
+
   if (config->fixCortexA8 && !config->isLE)
     error("--fix-cortex-a8 is not supported on big endian targets");
-  
+
   if (config->tocOptimize && config->emachine != EM_PPC64)
     error("--toc-optimize is only supported on PowerPC64 targets");
 
@@ -1115,6 +1118,7 @@ static void readConfigs(opt::InputArgList &args) {
                                             OPT_no_android_memtag_stack, false);
   config->androidMemtagMode = getMemtagMode(args);
   config->auxiliaryList = args::getStrings(args, OPT_auxiliary);
+  config->armBe8 = args.hasArg(OPT_be8);
   if (opt::Arg *arg =
           args.getLastArg(OPT_Bno_symbolic, OPT_Bsymbolic_non_weak_functions,
                           OPT_Bsymbolic_functions, OPT_Bsymbolic)) {
diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td
index 0c6884afe52d4..d878ae313fa2c 100644
--- a/lld/ELF/Options.td
+++ b/lld/ELF/Options.td
@@ -36,6 +36,8 @@ multiclass B<string name, string help1, string help2> {
 
 defm auxiliary: Eq<"auxiliary", "Set DT_AUXILIARY field to the specified name">;
 
+def be8: F<"be8">, HelpText<"write a Big Endian ELF file using BE8 format (AArch32 only)">;
+
 def Bno_symbolic: F<"Bno-symbolic">, HelpText<"Don't bind default visibility defined symbols locally for -shared (default)">;
 
 def Bsymbolic: F<"Bsymbolic">, HelpText<"Bind default visibility defined symbols locally for -shared">;
diff --git a/lld/ELF/OutputSections.cpp b/lld/ELF/OutputSections.cpp
index 05f9cdc20fd29..0b9b76e5b832f 100644
--- a/lld/ELF/OutputSections.cpp
+++ b/lld/ELF/OutputSections.cpp
@@ -496,6 +496,12 @@ void OutputSection::writeTo(uint8_t *buf, parallel::TaskGroup &tg) {
       else
         isec->writeTo<ELFT>(buf + isec->outSecOff);
 
+      // When in Arm BE8 mode, the linker has to convert the big-endian
+      // instructions to little-endian, leaving the data big-endian.
+      if (config->emachine == EM_ARM && !config->isLE && config->armBe8 &&
+          (flags & SHF_EXECINSTR))
+        convertArmInstructionstoBE8(isec, buf + isec->outSecOff);
+
       // Fill gaps between sections.
       if (nonZeroFiller) {
         uint8_t *start = buf + isec->outSecOff + isec->getSize();
diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp
index 48f30123dcc00..0a52ad31ede05 100644
--- a/lld/ELF/SyntheticSections.cpp
+++ b/lld/ELF/SyntheticSections.cpp
@@ -271,6 +271,13 @@ Defined *elf::addSyntheticLocal(StringRef name, uint8_t type, uint64_t value,
                            value, size, &section);
   if (in.symTab)
     in.symTab->addSymbol(s);
+
+  if (config->emachine == EM_ARM && !config->isLE && config->armBe8 &&
+      (section.flags & SHF_EXECINSTR))
+    // Adding Linker generated mapping symbols to the arm specific mapping
+    // symbols list.
+    addArmSyntheticSectionMappingSymbol(s);
+
   return s;
 }
 
diff --git a/lld/ELF/Target.h b/lld/ELF/Target.h
index 59534ab3256df..dd0ae392deab7 100644
--- a/lld/ELF/Target.h
+++ b/lld/ELF/Target.h
@@ -223,6 +223,10 @@ uint64_t getPPC64TocBase();
 uint64_t getAArch64Page(uint64_t expr);
 void riscvFinalizeRelax(int passes);
 void mergeRISCVAttributesSections();
+void addArmInputSectionMappingSymbols();
+void addArmSyntheticSectionMappingSymbol(Defined *);
+void sortArmMappingSymbols();
+void convertArmInstructionstoBE8(InputSection *sec, uint8_t *buf);
 
 LLVM_LIBRARY_VISIBILITY extern const TargetInfo *target;
 TargetInfo *getTarget();
diff --git a/lld/ELF/Thunks.cpp b/lld/ELF/Thunks.cpp
index 331e28e8bf4e1..81140686cef2c 100644
--- a/lld/ELF/Thunks.cpp
+++ b/lld/ELF/Thunks.cpp
@@ -744,7 +744,8 @@ void ThumbV6MABSLongThunk::addSymbols(ThunkSection &isec) {
   addSymbol(saver().save("__Thumbv6MABSLongThunk_" + destination.getName()),
             STT_FUNC, 1, isec);
   addSymbol("$t", STT_NOTYPE, 0, isec);
-  addSymbol("$d", STT_NOTYPE, 8, isec);
+  if (!getMayUseShortThunk())
+    addSymbol("$d", STT_NOTYPE, 8, isec);
 }
 
 void ThumbV6MPILongThunk::writeLong(uint8_t *buf) {
@@ -767,7 +768,8 @@ void ThumbV6MPILongThunk::addSymbols(ThunkSection &isec) {
   addSymbol(saver().save("__Thumbv6MPILongThunk_" + destination.getName()),
             STT_FUNC, 1, isec);
   addSymbol("$t", STT_NOTYPE, 0, isec);
-  addSymbol("$d", STT_NOTYPE, 12, isec);
+  if (!getMayUseShortThunk())
+    addSymbol("$d", STT_NOTYPE, 12, isec);
 }
 
 void ARMV5LongLdrPcThunk::writeLong(uint8_t *buf) {
@@ -780,7 +782,8 @@ void ARMV5LongLdrPcThunk::addSymbols(ThunkSection &isec) {
   addSymbol(saver().save("__ARMv5LongLdrPcThunk_" + destination.getName()),
             STT_FUNC, 0, isec);
   addSymbol("$a", STT_NOTYPE, 0, isec);
-  addSymbol("$d", STT_NOTYPE, 4, isec);
+  if (!getMayUseShortThunk())
+    addSymbol("$d", STT_NOTYPE, 4, isec);
 }
 
 void ARMV4ABSLongBXThunk::writeLong(uint8_t *buf) {
@@ -794,7 +797,8 @@ void ARMV4ABSLongBXThunk::addSymbols(ThunkSection &isec) {
   addSymbol(saver().save("__ARMv4ABSLongBXThunk_" + destination.getName()),
             STT_FUNC, 0, isec);
   addSymbol("$a", STT_NOTYPE, 0, isec);
-  addSymbol("$d", STT_NOTYPE, 8, isec);
+  if (!getMayUseShortThunk())
+    addSymbol("$d", STT_NOTYPE, 8, isec);
 }
 
 void ThumbV4ABSLongBXThunk::writeLong(uint8_t *buf) {
@@ -810,7 +814,8 @@ void ThumbV4ABSLongBXThunk::addSymbols(ThunkSection &isec) {
             STT_FUNC, 1, isec);
   addSymbol("$t", STT_NOTYPE, 0, isec);
   addSymbol("$a", STT_NOTYPE, 4, isec);
-  addSymbol("$d", STT_NOTYPE, 8, isec);
+  if (!getMayUseShortThunk())
+    addSymbol("$d", STT_NOTYPE, 8, isec);
 }
 
 void ThumbV4ABSLongThunk::writeLong(uint8_t *buf) {
@@ -827,7 +832,8 @@ void ThumbV4ABSLongThunk::addSymbols(ThunkSection &isec) {
             STT_FUNC, 1, isec);
   addSymbol("$t", STT_NOTYPE, 0, isec);
   addSymbol("$a", STT_NOTYPE, 4, isec);
-  addSymbol("$d", STT_NOTYPE, 12, isec);
+  if (!getMayUseShortThunk())
+    addSymbol("$d", STT_NOTYPE, 12, isec);
 }
 
 void ARMV4PILongBXThunk::writeLong(uint8_t *buf) {
@@ -844,7 +850,8 @@ void ARMV4PILongBXThunk::addSymbols(ThunkSection &isec) {
   addSymbol(saver().save("__ARMv4PILongBXThunk_" + destination.getName()),
             STT_FUNC, 0, isec);
   addSymbol("$a", STT_NOTYPE, 0, isec);
-  addSymbol("$d", STT_NOTYPE, 12, isec);
+  if (!getMayUseShortThunk())
+    addSymbol("$d", STT_NOTYPE, 12, isec);
 }
 
 void ARMV4PILongThunk::writeLong(uint8_t *buf) {
@@ -860,7 +867,8 @@ void ARMV4PILongThunk::addSymbols(ThunkSection &isec) {
   addSymbol(saver().save("__ARMv4PILongThunk_" + destination.getName()),
             STT_FUNC, 0, isec);
   addSymbol("$a", STT_NOTYPE, 0, isec);
-  addSymbol("$d", STT_NOTYPE, 8, isec);
+  if (!getMayUseShortThunk())
+    addSymbol("$d", STT_NOTYPE, 8, isec);
 }
 
 void ThumbV4PILongBXThunk::writeLong(uint8_t *buf) {
@@ -879,7 +887,8 @@ void ThumbV4PILongBXThunk::addSymbols(ThunkSection &isec) {
             STT_FUNC, 1, isec);
   addSymbol("$t", STT_NOTYPE, 0, isec);
   addSymbol("$a", STT_NOTYPE, 4, isec);
-  addSymbol("$d", STT_NOTYPE, 12, isec);
+  if (!getMayUseShortThunk())
+    addSymbol("$d", STT_NOTYPE, 12, isec);
 }
 
 void ThumbV4PILongThunk::writeLong(uint8_t *buf) {
@@ -899,7 +908,8 @@ void ThumbV4PILongThunk::addSymbols(ThunkSection &isec) {
             STT_FUNC, 1, isec);
   addSymbol("$t", STT_NOTYPE, 0, isec);
   addSymbol("$a", STT_NOTYPE, 4, isec);
-  addSymbol("$d", STT_NOTYPE, 16, isec);
+  if (!getMayUseShortThunk())
+    addSymbol("$d", STT_NOTYPE, 16, isec);
 }
 
 // Use the long jump which covers a range up to 8MiB.
diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp
index 3630b199714da..4396cf33c226d 100644
--- a/lld/ELF/Writer.cpp
+++ b/lld/ELF/Writer.cpp
@@ -2149,6 +2149,11 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() {
     sec->finalize();
 
   script->checkMemoryRegions();
+
+  if (config->emachine == EM_ARM && !config->isLE && config->armBe8) {
+    addArmInputSectionMappingSymbols();
+    sortArmMappingSymbols();
+  }
 }
 
 // Ensure data sections are not mixed with executable sections when
diff --git a/lld/docs/ld.lld.1 b/lld/docs/ld.lld.1
index e3db3a20678d9..898ec30dfe6f3 100644
--- a/lld/docs/ld.lld.1
+++ b/lld/docs/ld.lld.1
@@ -88,6 +88,8 @@ Bind default visibility defined function symbols locally for
 .It Fl Bsymbolic-non-weak-functions
 Bind default visibility defined STB_GLOBAL function symbols locally for
 .Fl shared.
+.It Fl --be8
+Write a Big Endian ELF File using BE8 format(AArch32 only)
 .It Fl -build-id Ns = Ns Ar value
 Generate a build ID note.
 .Ar value
diff --git a/lld/test/ELF/arm-bl-v6.s b/lld/test/ELF/arm-bl-v6.s
index 7666d6fb9e506..65f06207303c7 100644
--- a/lld/test/ELF/arm-bl-v6.s
+++ b/lld/test/ELF/arm-bl-v6.s
@@ -13,6 +13,12 @@
 // RUN: llvm-objdump --no-print-imm-hex -d --triple=armv6eb-none-linux-gnueabi --start-address=0x22100c --stop-address=0x221014 %t2 | FileCheck --check-prefix=CHECK-ARM2-EB %s
 // RUN: llvm-objdump --no-print-imm-hex -d --triple=thumbv6eb-none-linux-gnueabi %t2 --start-address=0x622000 --stop-address=0x622002 | FileCheck --check-prefix=CHECK-THUMB2 %s
 
+// RUN: ld.lld --be8 %t -o %t2
+// RUN: llvm-objdump --no-print-imm-hex -d --triple=armv6eb-none-linux-gnueabi --start-address=0x21000 --stop-address=0x21008 %t2 | FileCheck --check-prefix=CHECK-ARM1 %s
+// RUN: llvm-objdump --no-print-imm-hex -d --triple=thumbv6eb-none-linux-gnueabi %t2 --start-address=0x21008 --stop-address=0x2100c | FileCheck --check-prefix=CHECK-THUMB1 %s
+// RUN: llvm-objdump --no-print-imm-hex -d --triple=armv6eb-none-linux-gnueabi --start-address=0x22100c --stop-address=0x221014 %t2 | FileCheck --check-prefix=CHECK-ARM2-EB %s
+// RUN: llvm-objdump --no-print-imm-hex -d --triple=thumbv6eb-none-linux-gnueabi %t2 --start-address=0x622000 --stop-address=0x622002 | FileCheck --check-prefix=CHECK-THUMB2 %s
+
 /// On Arm v6 the range of a Thumb BL instruction is only 4 megabytes as the
 /// extended range encoding is not supported. The following example has a Thumb
 /// BL that is out of range on ARM v6 and requires a range extension thunk.
diff --git a/lld/test/ELF/arm-data-relocs.s b/lld/test/ELF/arm-data-relocs.s
index 70335aa6d51ae..91037c8cebf9b 100644
--- a/lld/test/ELF/arm-data-relocs.s
+++ b/lld/test/ELF/arm-data-relocs.s
@@ -9,6 +9,9 @@
 // RUN: ld.lld %t.be.o %t256.be.o -o %t.be
 // RUN: llvm-objdump -s %t.be | FileCheck %s --check-prefixes=CHECK,BE
 
+// RUN: ld.lld --be8 %t.be.o %t256.be.o -o %t.be8
+// RUN: llvm-objdump -s %t.be8 | FileCheck %s --check-prefixes=CHECK,BE
+
 .globl _start
 _start:
 .section .R_ARM_ABS, "ax","progbits"
diff --git a/lld/test/ELF/arm-exidx-emit-relocs.s b/lld/test/ELF/arm-exidx-emit-relocs.s
index a641200946b93..d3b609fa75073 100644
--- a/lld/test/ELF/arm-exidx-emit-relocs.s
+++ b/lld/test/ELF/arm-exidx-emit-relocs.s
@@ -9,6 +9,10 @@
 // RUN: llvm-objdump -s --triple=armv7aeb-none-linux-gnueabi %t2 | FileCheck -check-prefix=CHECK-EB %s
 // RUN: llvm-readelf --relocs %t2 | FileCheck -check-prefix=CHECK-RELOCS %s
 
+// RUN: ld.lld --be8 --emit-relocs %t -o %t2
+// RUN: llvm-objdump -s --triple=armv7aeb-none-linux-gnueabi %t2 | FileCheck -check-prefix=CHECK-EB %s
+// RUN: llvm-readelf --relocs %t2 | FileCheck -check-prefix=CHECK-RELOCS %s
+
 /// LLD does not support --emit-relocs for .ARM.exidx sections as the relocations
 /// from synthetic table entries won't be represented. Given the known use cases
 /// of --emit-relocs, relocating kernels, and binary analysis, the former doesn't
diff --git a/lld/test/ELF/arm-exidx-relocatable.s b/lld/test/ELF/arm-exidx-relocatable.s
index e750c0a4e749b..447a276d76110 100644
--- a/lld/test/ELF/arm-exidx-relocatable.s
+++ b/lld/test/ELF/arm-exidx-relocatable.s
@@ -10,6 +10,8 @@
 // Check that relocatable link maintains SHF_LINK_ORDER for big endian
 // RUN: ld.lld -r %t %tcantunwind -o %t4
 // RUN: llvm-readobj -S %t4 | FileCheck %s
+// RUN: ld.lld --be8 -r %t %tcantunwind -o %t4
+// RUN: llvm-readobj -S %t4 | FileCheck %s
 
 // Each assembler created .ARM.exidx section has the SHF_LINK_ORDER flag set
 // with the sh_link containing the section index of the executable section
diff --git a/lld/test/ELF/arm-exidx-sentinel-norelocatable.s b/lld/test/ELF/arm-exidx-sentinel-norelocatable.s
index 4c44b00708c48..696f0dcd847ca 100644
--- a/lld/test/ELF/arm-exidx-sentinel-norelocatable.s
+++ b/lld/test/ELF/arm-exidx-sentinel-norelocatable.s
@@ -6,6 +6,8 @@
 // RUN: llvm-mc %s -triple=armv7eb-unknown-linux-gnueabi -mcpu=cortex-a8 -filetype=obj -o %t.o
 // RUN: ld.lld -r %t.o -o %t
 // RUN: llvm-readobj -S %t | FileCheck %s
+// RUN: ld.lld --be8 -r %t.o -o %t
+// RUN: llvm-readobj -S %t | FileCheck %s
 
 // Check that when doing a relocatable link we don't add a terminating entry
 // to the .ARM.exidx section
diff --git a/lld/test/ELF/arm-header.s b/lld/test/ELF/arm-header.s
index 6712410d59f1f..bb20250a46ede 100644
--- a/lld/test/ELF/arm-header.s
+++ b/lld/test/ELF/arm-header.s
@@ -3,6 +3,10 @@
 # RUN: ld.lld %t.o -o %t1
 # RUN: llvm-readobj -h %t1 | FileCheck %s
 
+# RUN: llvm-mc -filetype=obj -triple=armv7aeb-none-linux-gnueabi -mcpu=cortex-a8 %s -o %t.o
+# RUN: ld.lld --be8 %t.o -o %t1
+# RUN: llvm-readobj -h %t1 | FileCheck -check-prefix=CHECK-EB %s
+
 # CHECK: Format: elf32-bigarm
 
 # CHECK: ElfHeader {
@@ -24,3 +28,23 @@
 
 # CHECK-NOT: 0x800000
 
+# CHECK-EB: Format: elf32-bigarm
+
+# CHECK-EB: ElfHeader {
+# CHECK-EB-NEXT:   Ident {
+# CHECK-EB-NEXT:     Magic: (7F 45 4C 46)
+# CHECK-EB-NEXT:     Class: 32-bit (0x1)
+# CHECK-EB-NEXT:     DataEncoding: BigEndian (0x2)
+# CHECK-EB-NEXT:     FileVersion: 1
+# CHECK-EB-NEXT:     OS/ABI: SystemV (0x0)
+# CHECK-EB-NEXT:     ABIVersion: 0
+# CHECK-EB-NEXT:     Unused: (00 00 00 00 00 00 00)
+# CHECK-EB-NEXT:   }
+
+# CHECK-EB:   Flags [ (0x5800200)
+# CHECK-EB-NEXT:     0x200
+# CHECK-EB-NEXT:     0x800000
+# CHECK-EB-NEXT:     0x1000000
+# CHECK-EB-NEXT:     0x4000000
+# CHECK-EB-NEXT:   ]
+
diff --git a/lld/test/ELF/arm-mov-relocs.s b/lld/test/ELF/arm-mov-relocs.s
index 559f546f7b493..2964a37f4071c 100644
--- a/lld/test/ELF/arm-mov-relocs.s
+++ b/lld/test/ELF/arm-mov-relocs.s
@@ -13,6 +13,11 @@
 // RUN: ld.lld %t3 -o %t4
 // RUN: llvm-objdump --no-print-imm-hex -d %t4 --triple=thumbv7aeb-unknown-linux-gnueabi --no-show-raw-insn | FileCheck %s
 
+// RUN: ld.lld --be8 %t -o %t2
+// RUN: llvm-objdump --no-print-imm-hex -d %t2 --triple=armv7aeb-unknown-linux-gnueabi --no-show-raw-insn | FileCheck %s
+// RUN: ld.lld --be8 %t3 -o %t4
+// RUN: llvm-objdump --no-print-imm-hex -d %t4 --triple=thumbv7aeb-unknown-linux-gnueabi --no-show-raw-insn | FileCheck %s
+
 /// Test the following relocation pairs:
 ///  * R_ARM_MOVW_ABS_NC and R_ARM_MOVT_ABS
 ///  * R_ARM_MOVW_PREL_NC and R_ARM_MOVT_PREL
diff --git a/lld/test/ELF/arm-plt-reloc.s b/lld/test/ELF/arm-plt-reloc.s
index e2438a64cc573..177eaf4b1f12b 100644
--- a/lld/test/ELF/arm-plt-reloc.s
+++ b/lld/test/ELF/arm-plt-reloc.s
@@ -6,6 +6,20 @@
 // RUN: ld.lld -shared %t1 %t2 -o %t3
 // RUN: llvm-objdump --no-print-imm-hex --triple=armv7a-none-linux-gnueabi -d --no-show-raw-insn %t3 | FileCheck --check-prefix=DSO %s
 // RUN: llvm-readobj -S -r %t3 | FileCheck -check-prefix=DSOREL %s
+
+// RUN: llvm-mc -filetype=obj -triple=armv7aeb-none-linux-gnueabi %p/Inputs/arm-plt-reloc.s -o %t1.be
+// RUN: llvm-mc -filetype=obj -triple=armv7aeb-none-linux-gnueabi %s -o %t2.be
+// RUN: ld.lld %t1.be %t2.be -o %t.be
+// RUN: llvm-objdump --no-print-imm-hex --triple=armv7aeb-none-linux-gnueabi -d --no-show-raw-insn %t.be | FileCheck %s
+// RUN: ld.lld -shared %t1.be %t2.be -o %t3.be
+// RUN: llvm-objdump --no-print-imm-hex --triple=armv7aeb-none-linux-gnueabi -d --no-show-raw-insn %t3.be | FileCheck --check-prefix=DSO %s
+// RUN: llvm-readobj -S -r %t3.be | FileCheck -check-prefix=DSOREL %s
+
+// RUN: ld.lld --be8 %t1.be %t2.be -o %t.be
+// RUN: llvm-objdump --no-print-imm-hex --triple=armv7aeb-none-linux-gnueabi -d --no-show-raw-insn %t.be | FileCheck %s
+// RUN: ld.lld --be8 -shared %t1.be %t2.be -o %t3.be
+// RUN: llvm-objdump --no-print-imm-hex --triple=armv7aeb-none-linux-gnueabi -d --no-show-raw-insn %t3.be | FileCheck --check-prefix=DSO %s
+// RUN: llvm-readobj -S -r %t3.be | FileCheck -check-prefix=DSOREL %s
 //
 // Test PLT entry generation
  .syntax unified
@@ -114,6 +128,13 @@ _start:
 // RUN: llvm-objdump --no-print-imm-hex --triple=armv7a-none-linux-gnueabi -d --no-show-raw-insn %t4 | FileCheck --check-prefix=CHECKHIGH %s
 // RUN: llvm-readobj -S -r %t4 | FileCheck --check-prefix=DSORELHIGH %s
 
+// RUN: ld.lld --hash-style=sysv --script %t.script -shared %t1.be %t2.be -o %t4.be
+// RUN: llvm-objdump --no-print-imm-hex --triple=armv7aeb-none-linux-gnueabi -d --no-show-raw-insn %t4.be | FileCheck --check-prefix=CHECKHIGH %s
+// RUN: llvm-readobj -S -r %t4.be | FileCheck --check-prefix=DSORELHIGH %s
+// RUN: ld.lld --be8 --hash-style=sysv --script %t.script -shared %t1.be %t2.be -o %t4.be
+// RUN: llvm-objdump --no-print-imm-hex --triple=armv7aeb-none-linux-gnueabi -d --no-show-raw-insn %t4.be | FileCheck --check-prefix=CHECKHIGH %s
+// RUN: llvm-readobj -S -r %t4.be | FileCheck --check-prefix=DSORELHIGH %s
+
 // CHECKHIGH: Disassembly of section .text:
 // CHECKHIGH-EMPTY:
 // CHECKHIGH-NEXT: <func1>:
@@ -182,6 +203,13 @@ _start:
 // RUN: llvm-objdump --no-print-imm-hex --triple=armv7a-none-linux-gnueabi -d --no-show-raw-insn %t5 | FileCheck --check-prefix=CHECKLONG %s
 // RUN: llvm-readobj -S -r %t5 | FileCheck --check-prefix=DSORELLONG %s
 
+// RUN: ld.lld --hash-style=sysv --script %t2.script -shared %t1.be %t2.be -o %t5.be
+// RUN: llvm-objdump --no-print-imm-hex --triple=armv7aeb-none-linux-gnueabi -d --no-show-raw-insn %t5.be | FileCheck --check-prefix=CHECKLONG-EB %s
+// RUN: llvm-readobj -S -r %t5.be | FileCheck --check-prefix=DSORELLONG %s
+// RUN: ld.lld --be8 --hash-style=sysv --script %t2.script -shared %t1.be %t2.be -o %t5.be
+// RUN: llvm-objdump --no-print-imm-hex --triple=armv7aeb-none-linux-gnueabi -d --no-show-raw-insn %t5.be | FileCheck --check-prefix=CHECKLONG-EB %s
+// RUN: llvm-readobj -S -r %t5.be | FileCheck --check-prefix=DSORELLONG %s
+
 // CHECKLONG: Disassembly of section .text:
 // CHECKLONG-EMPTY:
 // CHECKLONG-NEXT: <func1>:
@@ -226,6 +254,50 @@ _start:
 // CHECKLONG: <$d>:
 // CHECKLONG-NEXT:     204c:       c8 f0 10 11     .word   0x1110f0c8
 
+// CHECKLONG-EB: Disassembly of section .text:
+// CHECKLONG-EB-EMPTY:
+// CHECKLONG-EB-NEXT: <func1>:
+// CHECKLONG-EB-NEXT:     1000:       bx      lr
+// CHECKLONG-EB: <func2>:
+// CHECKLONG-EB-NEXT:                 bx      lr
+// CHECKLONG-EB: <func3>:
+// CHECKLONG-EB-NEXT:                 bx      lr
+// CHECKLONG-EB: <_start>:
+// CHECKLONG-EB-NEXT:                 b       0x2020
+// CHECKLONG-EB-NEXT:                 bl      0x2030
+// CHECKLONG-EB-NEXT:                 beq     0x2040
+// CHECKLONG-EB-EMPTY:
+// CHECKLONG-EB-NEXT: Disassembly of section .plt:
+// CHECKLONG-EB-EMPTY:
+// CHECKLONG-EB-NEXT: <$a>:
+// CHECKLONG-EB-NEXT:     2000:       str     lr, [sp, #-4]!
+// CHECKLONG-EB-NEXT:                 ldr     lr, [pc, #4]
+// CHECKLONG-EB-NEXT:                 add     lr, pc, lr
+// CHECKLONG-EB-NEXT:                 ldr     pc, [lr, #8]!
+// CHECKLONG-EB: <$d>:
+// CHECKLONG-EB-NEXT:                 11 10 f0 f0     .word   0x1110f0f0
+// CHECKLONG-EB-NEXT:                 d4 d4 d4 d4     .word   0xd4d4d4d4
+// CHECKLONG-EB-NEXT:                 d4 d4 d4 d4     .word   0xd4d4d4d4
+// CHECKLONG-EB-NEXT:                 d4 d4 d4 d4     .word   0xd4d4d4d4
+// CHECKLONG-EB: <$a>:
+// CHECKLONG-EB-NEXT:     2020:       ldr     r12, [pc, #4]
+// CHECKLONG-EB-NEXT:                 add     r12, r12, pc
+// CHECKLONG-EB-NEXT:                 ldr     pc, [r12]
+// CHECKLONG-EB: <$d>:
+// CHECKLONG-EB-NEXT:                 11 10 f0 e0     .word   0x1110f0e0
+// CHECKLONG-EB: <$a>:
+// CHECKLONG-EB-NEXT:     2030:       ldr     r12, [pc, #4]
+// CHECKLONG-EB-NEXT:                 add     r12, r12, pc
+// CHECKLONG-EB-NEXT:                 ldr     pc, [r12]
+// CHECKLONG-EB: <$d>:
+// CHECKLONG-EB-NEXT:                 11 10 f0 d4     .word   0x1110f0d4
+// CHECKLONG-EB: <$a>:
+// CHECKLONG-EB-NEXT:     2040:       ldr     r12, [pc, #4]
+// CHECKLONG-EB-NEXT:                 add     r12, r12, pc
+// CHECKLONG-EB-NEXT:                 ldr     pc, [r12]
+// CHECKLONG-EB: <$d>:
+// CHECKLONG-EB-NEXT:                11 10 f0 c8     .word   0x1110f0c8
+
 // DSORELLONG: Name: .got.plt
 // DSORELLONG-NEXT:     Type: SHT_PROGBITS
 // DSORELLONG-NEXT:     Flags [
@@ -251,6 +323,13 @@ _start:
 // RUN: llvm-objdump --no-print-imm-hex --triple=armv7a-none-linux-gnueabi -d --no-show-raw-insn %t6 | FileCheck --check-prefix=CHECKMIX %s
 // RUN: llvm-readobj -S -r %t6 | FileCheck --check-prefix=DSORELMIX %s
 
+// RUN: ld.lld --hash-style=sysv --script %t3.script -shared %t1.be %t2.be -o %t6.be
+// RUN: llvm-objdump --no-print-imm-hex --triple=armv7aeb-none-linux-gnueabi -d --no-show-raw-insn %t6.be | FileCheck --check-prefix=CHECKMIX-EB %s
+// RUN: llvm-readobj -S -r %t6.be | FileCheck --check-prefix=DSORELMIX %s
+// RUN: ld.lld --be8 --hash-style=sysv --script %t3.script -shared %t1.be %t2.be -o %t6.be
+// RUN: llvm-objdump --no-print-imm-hex --triple=armv7a-none-linux-gnueabi -d --no-show-raw-insn %t6.be | FileCheck --check-prefix=CHECKMIX-EB %s
+// RUN: llvm-readobj -S -r %t6.be | FileCheck --check-prefix=DSORELMIX %s
+
 // CHECKMIX: Disassembly of section .text:
 // CHECKMIX-EMPTY:
 // CHECKMIX-NEXT: <func1>:
@@ -295,6 +374,50 @@ _start:
 // CHECKMIX: <$d>:
 // CHECKMIX-NEXT:     204c:     d4 d4 d4 d4     .word   0xd4d4d4d4
 
+// CHECKMIX-EB: Disassembly of section .text:
+// CHECKMIX-EB-EMPTY:
+// CHECKMIX-EB-NEXT: <func1>:
+// CHECKMIX-EB-NEXT:     1000:       bx      lr
+// CHECKMIX-EB: <func2>:
+// CHECKMIX-EB-NEXT:                 bx      lr
+// CHECKMIX-EB: <func3>:
+// CHECKMIX-EB-NEXT:                 bx      lr
+// CHECKMIX-EB: <_start>:
+// CHECKMIX-EB-NEXT:                 b       0x2020
+// CHECKMIX-EB-NEXT:                 bl      0x2030
+// CHECKMIX-EB-NEXT:                 beq     0x2040
+// CHECKMIX-EB-EMPTY:
+// CHECKMIX-EB-NEXT: Disassembly of section .plt:
+// CHECKMIX-EB-EMPTY:
+// CHECKMIX-EB-NEXT: <$a>:
+// CHECKMIX-EB-NEXT:     2000:       str     lr, [sp, #-4]!
+// CHECKMIX-EB-NEXT:                 ldr     lr, [pc, #4]
+// CHECKMIX-EB-NEXT:                 add     lr, pc, lr
+// CHECKMIX-EB-NEXT:                 ldr     pc, [lr, #8]!
+// CHECKMIX-EB: <$d>:
+// CHECKMIX-EB-NEXT:                 08 00 00 10     .word   0x08000010
+// CHECKMIX-EB-NEXT:                 d4 d4 d4 d4     .word   0xd4d4d4d4
+// CHECKMIX-EB-NEXT:                 d4 d4 d4 d4     .word   0xd4d4d4d4
+// CHECKMIX-EB-NEXT:                 d4 d4 d4 d4     .word   0xd4d4d4d4
+// CHECKMIX-EB: <$a>:
+// CHECKMIX-EB-NEXT:     2020:       ldr     r12, [pc, #4]
+// CHECKMIX-EB-NEXT:                 add     r12, r12, pc
+// CHECKMIX-EB-NEXT:                 ldr     pc, [r12]
+// CHECKMIX-EB: <$d>:
+// CHECKMIX-EB-NEXT:                 08 00 00 00     .word   0x08000000
+// CHECKMIX-EB: <$a>:
+// CHECKMIX-EB-NEXT:     2030:       add     r12, pc, #133169152
+// CHECKMIX-EB-NEXT:                 add     r12, r12, #1044480
+// CHECKMIX-EB-NEXT:                 ldr     pc, [r12, #4088]!
+// CHECKMIX-EB: <$d>:
+// CHECKMIX-EB-NEXT:                 d4 d4 d4 d4     .word   0xd4d4d4d4
+// CHECKMIX-EB: <$a>:
+// CHECKMIX-EB-NEXT:     2040:       add     r12, pc, #133169152
+// CHECKMIX-EB-NEXT:                 add     r12, r12, #1044480
+// CHECKMIX-EB-NEXT:                 ldr     pc, [r12, #4076]!
+// CHECKMIX-EB: <$d>:
+// CHECKMIX-EB-NEXT:                 d4 d4 d4 d4     .word   0xd4d4d4d4
+
 // DSORELMIX:    Name: .got.plt
 // DSORELMIX-NEXT:     Type: SHT_PROGBITS
 // DSORELMIX-NEXT:     Flags [
@@ -307,59 +430,3 @@ _start:
 // DSORELMIX-NEXT:     0x8002030 R_ARM_JUMP_SLOT func2
 // DSORELMIX-NEXT:     0x8002034 R_ARM_JUMP_SLOT func3
 
-// RUN: llvm-mc -filetype=obj -triple=armv7aeb-none-linux-gnueabi -mcpu=cortex-a8 %p/Inputs/arm-plt-reloc.s -o %t1
-// RUN: llvm-mc -filetype=obj -triple=armv7aeb-none-linux-gnueabi -mcpu=cortex-a8 %s -o %t2
-// RUN: ld.lld %t1 %t2 -o %t
-// RUN: llvm-objdump --no-print-imm-hex --triple=armv7aeb-none-linux-gnueabi -d --no-show-raw-insn %t | FileCheck %s
-// RUN: ld.lld -shared %t1 %t2 -o %t3
-// RUN: llvm-objdump --no-print-imm-hex --triple=armv7aeb-none-linux-gnueabi -d --no-show-raw-insn %t3 | FileCheck --check-prefix=DSO %s
-// RUN: llvm-readobj -S -r %t3 | FileCheck -check-prefix=DSOREL %s
-
-// RUN: ld.lld --hash-style=sysv --script %t2.script -shared %t1 %t2 -o %t5
-// RUN: llvm-objdump --no-print-imm-hex --triple=armv7aeb-none-linux-gnueabi -d --no-show-raw-insn %t5 | FileCheck --check-prefix=CHECKLONG-EB %s
-// RUN: llvm-readobj -S -r %t5 | FileCheck --check-prefix=DSORELLONG %s
-
-// CHECKLONG-EB: Disassembly of section .text:
-// CHECKLONG-EB-EMPTY:
-// CHECKLONG-EB-NEXT: <func1>:
-// CHECKLONG-EB-NEXT:     1000:       bx      lr
-// CHECKLONG-EB: <func2>:
-// CHECKLONG-EB-NEXT:     1004:       bx      lr
-// CHECKLONG-EB: <func3>:
-// CHECKLONG-EB-NEXT:     1008:       bx      lr
-// CHECKLONG-EB: <_start>:
-// CHECKLONG-EB-NEXT:     100c:       b       0x2020
-// CHECKLONG-EB-NEXT:     1010:       bl      0x2030
-// CHECKLONG-EB-NEXT:     1014:       beq     0x2040
-// CHECKLONG-EB-EMPTY:
-// CHECKLONG-EB-NEXT: Disassembly of section .plt:
-// CHECKLONG-EB-EMPTY:
-// CHECKLONG-EB-NEXT: <$a>:
-// CHECKLONG-EB-NEXT:     2000:       str     lr, [sp, #-4]!
-// CHECKLONG-EB-NEXT:     2004:       ldr     lr, [pc, #4]
-// CHECKLONG-EB-NEXT:     2008:       add     lr, pc, lr
-// CHECKLONG-EB-NEXT:     200c:       ldr     pc, [lr, #8]!
-// CHECKLONG-EB: <$d>:
-// CHECKLONG-EB-NEXT:     2010:	      11 10 f0 f0     .word   0x1110f0f0
-// CHECKLONG-EB-NEXT:     2014:       d4 d4 d4 d4     .word   0xd4d4d4d4
-// CHECKLONG-EB-NEXT:     2018:       d4 d4 d4 d4     .word   0xd4d4d4d4
-// CHECKLONG-EB-NEXT:     201c:       d4 d4 d4 d4     .word   0xd4d4d4d4
-// CHECKLONG-EB: <$a>:
-// CHECKLONG-EB-NEXT:     2020:       ldr     r12, [pc, #4]
-// CHECKLONG-EB-NEXT:     2024:       add     r12, r12, pc
-// CHECKLONG-EB-NEXT:     2028:       ldr     pc, [r12]
-// CHECKLONG-EB: <$d>:
-// CHECKLONG-EB-NEXT:     202c:       11 10 f0 e0     .word   0x1110f0e0
-// CHECKLONG-EB: <$a>:
-// CHECKLONG-EB-NEXT:     2030:       ldr     r12, [pc, #4]
-// CHECKLONG-EB-NEXT:     2034:       add     r12, r12, pc
-// CHECKLONG-EB-NEXT:     2038:       ldr     pc, [r12]
-// CHECKLONG-EB: <$d>:
-// CHECKLONG-EB-NEXT:     203c:       11 10 f0 d4     .word 0x1110f0d4
-// CHECKLONG-EB: <$a>:
-// CHECKLONG-EB-NEXT:     2040:       ldr     r12, [pc, #4]
-// CHECKLONG-EB-NEXT:     2044:       add     r12, r12, pc
-// CHECKLONG-EB-NEXT:     2048:       ldr     pc, [r12]
-// CHECKLONG-EB: <$d>:
-// CHECKLONG-EB-NEXT:     204c:       11 10 f0 c8     .word   0x1110f0c8
-
diff --git a/lld/test/ELF/arm-thumb-plt-reloc.s b/lld/test/ELF/arm-thumb-plt-reloc.s
index 580c06d6cfaea..a0a90893318ff 100644
--- a/lld/test/ELF/arm-thumb-plt-reloc.s
+++ b/lld/test/ELF/arm-thumb-plt-reloc.s
@@ -15,6 +15,12 @@
 // RUN: llvm-objdump --no-print-imm-hex --triple=thumbv7aeb-none-linux-gnueabi -d %t.so | FileCheck --check-prefix=DSO %s
 // RUN: llvm-readobj -S -r %t.so | FileCheck -check-prefix=DSOREL %s
 
+// RUN: ld.lld --be8 %t1 %t2 -o %t
+// RUN: llvm-objdump --no-print-imm-hex --triple=thumbv7aeb-none-linux-gnueabi -d %t | FileCheck %s
+// RUN: ld.lld --be8 -shared %t1 %t2 -o %t.so
+// RUN: llvm-objdump --no-print-imm-hex --triple=thumbv7aeb-none-linux-gnueabi -d %t.so | FileCheck --check-prefix=DSO %s
+// RUN: llvm-readobj -S -r %t.so | FileCheck -check-prefix=DSOREL %s
+
 // Test PLT entry generation
  .syntax unified
  .text
diff --git a/lld/test/ELF/arm-thunk-arm-thumb-reuse.s b/lld/test/ELF/arm-thunk-arm-thumb-reuse.s
index 1618e7eebb931..12538312f0e09 100644
--- a/lld/test/ELF/arm-thunk-arm-thumb-reuse.s
+++ b/lld/test/ELF/arm-thunk-arm-thumb-reuse.s
@@ -7,6 +7,8 @@
 // RUN: llvm-mc -arm-add-build-attributes -filetype=obj -triple=thumbv7aeb-none-linux-gnueabi -mcpu=cortex-a8 %t/test.s -o %t.o
 // RUN: ld.lld --script %t/script %t.o -o %t2
 // RUN: llvm-objdump --no-print-imm-hex --no-show-raw-insn -d %t2 | FileCheck %s
+// RUN: ld.lld --be8 --script %t/script %t.o -o %t2
+// RUN: llvm-objdump --no-print-imm-hex --no-show-raw-insn -d %t2 | FileCheck %s
 
 /// Test that we can reuse thunks between Arm and Thumb callers
 /// using a BL. Expect two thunks, one for far, one for far2.
diff --git a/lld/test/ELF/arm-thunk-edgecase.s b/lld/test/ELF/arm-thunk-edgecase.s
index a1c17b859c2f5..46b0335236eea 100644
--- a/lld/test/ELF/arm-thunk-edgecase.s
+++ b/lld/test/ELF/arm-thunk-edgecase.s
@@ -19,6 +19,11 @@
 // RUN: llvm-objdump --no-print-imm-hex --triple=armv7aeb-none-linux-gnueabi -d %tarm_to_thumb.so | FileCheck --check-prefix=ARM-TO-THUMB %s
 // RUN: llvm-objdump --no-print-imm-hex -d %tthumb_to_arm.so | FileCheck --check-prefix=THUMB-TO-ARM %s
 
+// RUN: ld.lld --be8 -shared -Bsymbolic -script %tarm_to_thumb.script %t.o -o %tarm_to_thumb.so
+// RUN: ld.lld --be8 -shared -Bsymbolic -script %tthumb_to_arm.script %t.o -o %tthumb_to_arm.so
+// RUN: llvm-objdump --no-print-imm-hex --triple=armv7aeb-none-linux-gnueabi -d %tarm_to_thumb.so | FileCheck --check-prefix=ARM-TO-THUMB %s
+// RUN: llvm-objdump --no-print-imm-hex -d %tthumb_to_arm.so | FileCheck --check-prefix=THUMB-TO-ARM %s
+
 .syntax unified
 
 .arm
diff --git a/lld/test/ELF/arm-thunk-largesection.s b/lld/test/ELF/arm-thunk-largesection.s
index 80927adb73bf9..be16393664b6f 100644
--- a/lld/test/ELF/arm-thunk-largesection.s
+++ b/lld/test/ELF/arm-thunk-largesection.s
@@ -13,6 +13,13 @@
 // RUN: llvm-objdump --no-print-imm-hex -d --no-show-raw-insn --start-address=0x22004 --stop-address=0x22008 %t2 | FileCheck --check-prefix=CHECK2 %s
 // RUN: llvm-objdump --no-print-imm-hex -d --no-show-raw-insn --start-address=0x1021ff8 --stop-address=0x1021ffc %t2 | FileCheck --check-prefix=CHECK3 %s
 // RUN: llvm-objdump --no-print-imm-hex -d --no-show-raw-insn --start-address=0x2012ff8 --stop-address=0x2021ffc %t2 | FileCheck --check-prefix=CHECK4 %s
+// RUN: llvm-objdump --no-print-imm-hex -d --no-show-raw-insn --start-address=0x3021fec --stop-address=0x3021ff6 %t2 | FileCheck --check-prefix=CHECK5 %s
+
+// RUN: ld.lld --be8 %t -o %t2
+// RUN: llvm-objdump --no-print-imm-hex -d --no-show-raw-insn --start-address=0x21000 --stop-address=0x21006 %t2 | FileCheck --check-prefix=CHECK1 %s
+// RUN: llvm-objdump --no-print-imm-hex -d --no-show-raw-insn --start-address=0x22004 --stop-address=0x22008 %t2 | FileCheck --check-prefix=CHECK2 %s
+// RUN: llvm-objdump --no-print-imm-hex -d --no-show-raw-insn --start-address=0x1021ff8 --stop-address=0x1021ffc %t2 | FileCheck --check-prefix=CHECK3 %s
+// RUN: llvm-objdump --no-print-imm-hex -d --no-show-raw-insn --start-address=0x2012ff8 --stop-address=0x2021ffc %t2 | FileCheck --check-prefix=CHECK4 %s
 // RUN: llvm-objdump --no-print-imm-hex -d --no-show-raw-insn --start-address=0x3021fec --stop-address=0x3021ff6 %t2 | FileCheck --check-prefix=CHECK5 %s
 
  .syntax unified
diff --git a/lld/test/ELF/arm-thunk-linkerscript-dotexpr.s b/lld/test/ELF/arm-thunk-linkerscript-dotexpr.s
index 9514560446b06..21849aa3e9e29 100644
--- a/lld/test/ELF/arm-thunk-linkerscript-dotexpr.s
+++ b/lld/test/ELF/arm-thunk-linkerscript-dotexpr.s
@@ -12,6 +12,9 @@
 // RUN: ld.lld --no-rosegment --script %t.script %t -o %t2
 // RUN: llvm-objdump --no-print-imm-hex -d %t2 --start-address=0x94 --stop-address=0xbc | FileCheck --check-prefix=CHECK1 %s
 // RUN: llvm-objdump --no-print-imm-hex -d %t2 --start-address=0x20000bc --stop-address=0x20000de | FileCheck --check-prefix=CHECK2 %s
+// RUN: ld.lld --be8 --no-rosegment --script %t.script %t -o %t2
+// RUN: llvm-objdump --no-print-imm-hex -d %t2 --start-address=0x94 --stop-address=0xbc | FileCheck --check-prefix=CHECK1 %s
+// RUN: llvm-objdump --no-print-imm-hex -d %t2 --start-address=0x20000bc --stop-address=0x20000de | FileCheck --check-prefix=CHECK2 %s
 
 // Test that range extension thunks can handle location expressions within
 // a Section Description
diff --git a/lld/test/ELF/arm-thunk-linkerscript.s b/lld/test/ELF/arm-thunk-linkerscript.s
index f18861b33ab31..131ab41a40a0c 100644
--- a/lld/test/ELF/arm-thunk-linkerscript.s
+++ b/lld/test/ELF/arm-thunk-linkerscript.s
@@ -11,6 +11,8 @@
 // RUN: llvm-mc -arm-add-build-attributes -filetype=obj -triple=armv7aeb-none-linux-gnueabi -mcpu=cortex-a8 %s -o %t
 // RUN: ld.lld --no-rosegment --script %t.script %t -o %t2
 // RUN: llvm-objdump --no-print-imm-hex -d %t2 | FileCheck %s
+// RUN: ld.lld --be8 --no-rosegment --script %t.script %t -o %t2
+// RUN: llvm-objdump --no-print-imm-hex -d %t2 | FileCheck %s
 
 // Simple test that we can support range extension thunks with linker scripts
  .syntax unified
diff --git a/lld/test/ELF/arm-thunk-nosuitable.s b/lld/test/ELF/arm-thunk-nosuitable.s
index cf19d0c135d92..dbd8334a2db17 100644
--- a/lld/test/ELF/arm-thunk-nosuitable.s
+++ b/lld/test/ELF/arm-thunk-nosuitable.s
@@ -5,6 +5,8 @@
 
 // RUN: llvm-mc %s --arm-add-build-attributes --triple=armv7aeb-linux-gnueabihf --filetype=obj -mcpu=cortex-a8 -o %t.o
 // RUN: ld.lld %t.o -o %t
+// RUN: llvm-objdump -d --start-address=0x2200b4 --stop-address=0x2200be %t | FileCheck %s
+// RUN: ld.lld --be8 %t.o -o %t
 // RUN: llvm-objdump -d --start-address=0x2200b4 --stop-address=0x2200be %t | FileCheck %s
 
         /// Create a conditional branch too far away from a precreated thunk
diff --git a/lld/test/ELF/arm-thunk-re-add.s b/lld/test/ELF/arm-thunk-re-add.s
index 996552bfa0b02..3fa876082c398 100644
--- a/lld/test/ELF/arm-thunk-re-add.s
+++ b/lld/test/ELF/arm-thunk-re-add.s
@@ -131,3 +131,10 @@ callers:
 // RUN: llvm-objdump --no-print-imm-hex -d %t.so --start-address=0x1100008 --stop-address=0x1100022 | FileCheck --check-prefix=CHECK2 %s
 // RUN: llvm-objdump --no-print-imm-hex -d %t.so --start-address=0x1100020 --stop-address=0x1100064 --triple=armv7aeb-linux-gnueabihf | FileCheck --check-prefix=CHECK3 %s
 
+// RUN: ld.lld --be8 %t --shared -o %t.so
+// The output file is large, most of it zeroes. We dissassemble only the
+// parts we need to speed up the test and avoid a large output file
+// RUN: llvm-objdump --no-print-imm-hex -d %t.so --start-address=0x1000004 --stop-address=0x100001c | FileCheck --check-prefix=CHECK1 %s
+// RUN: llvm-objdump --no-print-imm-hex -d %t.so --start-address=0x1100008 --stop-address=0x1100022 | FileCheck --check-prefix=CHECK2 %s
+// RUN: llvm-objdump --no-print-imm-hex -d %t.so --start-address=0x1100020 --stop-address=0x1100064 --triple=armv7aeb-linux-gnueabihf | FileCheck --check-prefix=CHECK3 %s
+
diff --git a/lld/test/ELF/arm-thunk-reuse.s b/lld/test/ELF/arm-thunk-reuse.s
index 67993a97e6326..f6815da7af496 100644
--- a/lld/test/ELF/arm-thunk-reuse.s
+++ b/lld/test/ELF/arm-thunk-reuse.s
@@ -3,10 +3,15 @@
 # RUN: llvm-mc -filetype=obj -triple=armv7-a-none-eabi --arm-add-build-attributes %t/a.s -o %t/a.o
 # RUN: ld.lld -pie -T %t/lds %t/a.o -o %t/a
 # RUN: llvm-objdump --no-print-imm-hex -d --no-show-raw-insn %t/a | FileCheck %s
+# RUN: llvm-objdump -s --triple=armv7a-none-linux-gnueabi %t/a | FileCheck --check-prefix=CHECK-LE %s
 
 # RUN: llvm-mc -filetype=obj -triple=armv7eb-a-none-eabi --arm-add-build-attributes -mcpu=cortex-a8 %t/a.s -o %t/a.o
 # RUN: ld.lld -pie -T %t/lds %t/a.o -o %t/a
 # RUN: llvm-objdump --no-print-imm-hex -d --no-show-raw-insn %t/a | FileCheck %s
+# RUN: llvm-objdump -s --triple=armv7eb-a-none-eabi %t/a| FileCheck --check-prefix=CHECK-EB %s
+# RUN: ld.lld --be8 -pie -T %t/lds %t/a.o -o %t/a
+# RUN: llvm-objdump --no-print-imm-hex -d --no-show-raw-insn %t/a | FileCheck %s
+# RUN: llvm-objdump -s --triple=armv7eb-a-none-eabi %t/a| FileCheck --check-prefix=CHECK-LE %s
 
 ## We create a thunk for dest.
 # CHECK-LABEL: <mid>:
@@ -30,6 +35,24 @@
 # CHECK-NEXT:                 add     r12, r12, pc
 # CHECK-NEXT:                 bx      r12
 
+# CHECK-EB: Contents of section .text_low:
+# CHECK-EB-NEXT: 10000 e320f000 e12fff1e
+# CHECK-EB: Contents of section .text_mid:
+# CHECK-EB-NEXT: 2010004 eaffffff e30fcfec e34fcdff e08cc00f
+# CHECK-EB-NEXT: 2010014 e12fff1c
+# CHECK-EB: Contents of section .text_high:
+# CHECK-EB-NEXT: 4010000 eb800000 eaffffff e30fcfec e34fcbff
+# CHECK-EB-NEXT: 4010010 e08cc00f e12fff1c
+
+# CHECK-LE: Contents of section .text_low:
+# CHECK-LE-NEXT: 10000 00f020e3 1eff2fe1
+# CHECK-LE: Contents of section .text_mid:
+# CHECK-LE-NEXT: 2010004 ffffffea eccf0fe3 ffcd4fe3 0fc08ce0
+# CHECK-LE-NEXT: 2010014 1cff2fe1
+# CHECK-LE: Contents of section .text_high:
+# CHECK-LE-NEXT: 4010000 000080eb ffffffea eccf0fe3 ffcb4fe3
+# CHECK-LE-NEXT: 4010010 0fc08ce0 1cff2fe1
+
 #--- a.s
 .section .text_low, "ax", %progbits
 
diff --git a/lld/test/ELF/arm-thunk-section-too-large.s b/lld/test/ELF/arm-thunk-section-too-large.s
index 227f16485e443..c6c058085968d 100644
--- a/lld/test/ELF/arm-thunk-section-too-large.s
+++ b/lld/test/ELF/arm-thunk-section-too-large.s
@@ -4,6 +4,7 @@
 
 // RUN: llvm-mc %s -triple=armv7aeb-linux-gnueabihf -arm-add-build-attributes -filetype=obj -mcpu=cortex-a8 -o %t.o
 // RUN: not ld.lld %t.o -o /dev/null 2>&1 | FileCheck %s
+// RUN: not ld.lld --be8 %t.o -o /dev/null 2>&1 | FileCheck %s
 
 // CHECK: InputSection too large for range extension thunk
         .syntax unified
diff --git a/lld/test/ELF/arm-thunk-toolargesection.s b/lld/test/ELF/arm-thunk-toolargesection.s
index e0383572e3ffa..f91c4a4959315 100644
--- a/lld/test/ELF/arm-thunk-toolargesection.s
+++ b/lld/test/ELF/arm-thunk-toolargesection.s
@@ -4,6 +4,7 @@
 
 // RUN: llvm-mc -filetype=obj -triple=thumbv7aeb-none-linux-gnueabi %s -o %t
 // RUN: not ld.lld %t -o /dev/null 2>&1 | FileCheck %s
+// RUN: not ld.lld --be8 %t -o /dev/null 2>&1 | FileCheck %s
 
  .syntax unified
  .balign 0x1000
diff --git a/lld/test/ELF/arm-v5-reloc-error.s b/lld/test/ELF/arm-v5-reloc-error.s
index 85a675db27e71..bd4b9ad68d10a 100644
--- a/lld/test/ELF/arm-v5-reloc-error.s
+++ b/lld/test/ELF/arm-v5-reloc-error.s
@@ -9,6 +9,7 @@
 
 // RUN: llvm-mc -filetype=obj -triple=armv7aeb-linux-gnueabi -mcpu=cortex-a8 %s -o %t
 // RUN: not ld.lld --script %t.script %t -o /dev/null 2>&1 | FileCheck %s
+// RUN: not ld.lld -be8 --script %t.script %t -o /dev/null 2>&1 | FileCheck %s
 
 // CHECK: error: relocation R_ARM_THM_JUMP24 to far not supported for Armv5 or Armv6 targets
 
diff --git a/lld/test/ELF/emulation-arm.s b/lld/test/ELF/emulation-arm.s
index 3ab1033279dd7..b061bbb9f3f5b 100644
--- a/lld/test/ELF/emulation-arm.s
+++ b/lld/test/ELF/emulation-arm.s
@@ -30,6 +30,11 @@
 # RUN: llvm-readobj -h %t1 | FileCheck %s --check-prefix=ARMEB
 # RUN: llvm-readobj -h %t1 | FileCheck %s --check-prefix=BE8
 
+# RUN: ld.lld --be8 -marmelfb_linux_eabi %t.o -o %t1
+# echo 'OUTPUT_FORMAT(elf32-bigarm)' > %t6arm.script
+# ld.lld --be8 %tarm -o %t6arm
+# RUN: llvm-readobj -h %t1 | FileCheck %s --check-prefix=ARMEB8
+
 # ARMEB: ElfHeader {
 # ARMEB-NEXT:   Ident {
 # ARMEB-NEXT:     Magic: (7F 45 4C 46)
@@ -54,5 +59,27 @@
 
 # BE8-NOT: 0x800000
 
+# ARMEB8: ElfHeader {
+# ARMEB8-NEXT:   Ident {
+# ARMEB8-NEXT:     Magic: (7F 45 4C 46)
+# ARMEB8-NEXT:     Class: 32-bit (0x1)
+# ARMEB8-NEXT:     DataEncoding: BigEndian (0x2)
+# ARMEB8-NEXT:     FileVersion: 1
+# ARMEB8-NEXT:     OS/ABI: SystemV (0x0)
+# ARMEB8-NEXT:     ABIVersion: 0
+# ARMEB8-NEXT:     Unused: (00 00 00 00 00 00 00)
+# ARMEB8-NEXT:   }
+# ARMEB8-NEXT:   Type: Executable (0x2)
+# ARMEB8-NEXT:   Machine: EM_ARM (0x28)
+# ARMEB8-NEXT:   Version: 1
+
+## Ensure that the EF_ARM_BE8 flag is set for be8
+# ARMEB8:   Flags [ (0x5800200)
+# ARMEB8-NEXT:     0x200
+# ARMEB8-NEXT:     0x800000
+# ARMEB8-NEXT:     0x1000000
+# ARMEB8-NEXT:     0x4000000
+# ARMEB8-NEXT:   ]
+
 .globl _start
 _start:
diff --git a/lld/test/ELF/target-specific-options.s b/lld/test/ELF/target-specific-options.s
index 76227803f9350..94dccd9e4a0c1 100644
--- a/lld/test/ELF/target-specific-options.s
+++ b/lld/test/ELF/target-specific-options.s
@@ -4,6 +4,9 @@
 # RUN: not ld.lld %t --fix-cortex-a53-843419 -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR-843419
 # ERR-843419: error: --fix-cortex-a53-843419 is only supported on AArch64 targets
 
+# RUN: not ld.lld %t --be8 -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR-BE8
+# ERR-BE8: error: --be8 is only supported on ARM targets
+
 # RUN: not ld.lld %t --pcrel-optimize -o /dev/null 2>&1 | FileCheck %s --check-prefix=ERR-PCREL
 # ERR-PCREL: error: --pcrel-optimize is only supported on PowerPC64 targets
 

From 1d27ad20774435bb97760df124fba99d4c780ccb Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Tue, 20 Jun 2023 14:10:25 +0100
Subject: [PATCH 122/130] [AArch64] Add tablegen patterns for fp16 fcvtn2.

Similar to the existing f32 pattern, this adds a tablegen pattern for the fp16
fcvtn2.
---
 llvm/lib/Target/AArch64/AArch64InstrInfo.td   |   8 +-
 .../CodeGen/AArch64/fp16-v16-instructions.ll  |  68 +++++-----
 .../CodeGen/AArch64/fp16-v8-instructions.ll   | 123 ++++++++----------
 llvm/test/CodeGen/AArch64/vector-fcopysign.ll |   3 +-
 4 files changed, 92 insertions(+), 110 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index e6af51629c5a0..daf8aed458096 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -4673,10 +4673,14 @@ def : Pat<(v4i16 (int_aarch64_neon_vcvtfp2hf (v4f32 V128:$Rn))),
 def : Pat<(concat_vectors V64:$Rd,
                           (v4i16 (int_aarch64_neon_vcvtfp2hf (v4f32 V128:$Rn)))),
           (FCVTNv8i16 (INSERT_SUBREG (IMPLICIT_DEF), V64:$Rd, dsub), V128:$Rn)>;
-def : Pat<(v2f32 (any_fpround (v2f64 V128:$Rn))), (FCVTNv2i32 V128:$Rn)>;
-def : Pat<(v4f16 (any_fpround (v4f32 V128:$Rn))), (FCVTNv4i16 V128:$Rn)>;
+def : Pat<(v2f32 (any_fpround (v2f64 V128:$Rn))),
+          (FCVTNv2i32 V128:$Rn)>;
+def : Pat<(v4f16 (any_fpround (v4f32 V128:$Rn))),
+          (FCVTNv4i16 V128:$Rn)>;
 def : Pat<(concat_vectors V64:$Rd, (v2f32 (any_fpround (v2f64 V128:$Rn)))),
           (FCVTNv4i32 (INSERT_SUBREG (IMPLICIT_DEF), V64:$Rd, dsub), V128:$Rn)>;
+def : Pat<(concat_vectors V64:$Rd, (v4f16 (any_fpround (v4f32 V128:$Rn)))),
+          (FCVTNv8i16 (INSERT_SUBREG (IMPLICIT_DEF), V64:$Rd, dsub), V128:$Rn)>;
 defm FCVTPS : SIMDTwoVectorFPToInt<0,1,0b11010, "fcvtps",int_aarch64_neon_fcvtps>;
 defm FCVTPU : SIMDTwoVectorFPToInt<1,1,0b11010, "fcvtpu",int_aarch64_neon_fcvtpu>;
 defm FCVTXN : SIMDFPInexactCvtTwoVector<1, 0, 0b10110, "fcvtxn",
diff --git a/llvm/test/CodeGen/AArch64/fp16-v16-instructions.ll b/llvm/test/CodeGen/AArch64/fp16-v16-instructions.ll
index af3b52f66de77..74552735179c3 100644
--- a/llvm/test/CodeGen/AArch64/fp16-v16-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/fp16-v16-instructions.ll
@@ -5,16 +5,14 @@
 define <16 x half> @sitofp_i32(<16 x i32> %a) #0 {
 ; CHECK-LABEL: sitofp_i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    scvtf v1.4s, v1.4s
-; CHECK-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-NEXT:    scvtf v3.4s, v3.4s
 ; CHECK-NEXT:    scvtf v2.4s, v2.4s
-; CHECK-NEXT:    fcvtn v4.4h, v1.4s
+; CHECK-NEXT:    scvtf v0.4s, v0.4s
+; CHECK-NEXT:    scvtf v4.4s, v1.4s
 ; CHECK-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-NEXT:    fcvtn v3.4h, v3.4s
 ; CHECK-NEXT:    fcvtn v1.4h, v2.4s
-; CHECK-NEXT:    mov v0.d[1], v4.d[0]
-; CHECK-NEXT:    mov v1.d[1], v3.d[0]
+; CHECK-NEXT:    scvtf v2.4s, v3.4s
+; CHECK-NEXT:    fcvtn2 v0.8h, v4.4s
+; CHECK-NEXT:    fcvtn2 v1.8h, v2.4s
 ; CHECK-NEXT:    ret
 
   %1 = sitofp <16 x i32> %a to <16 x half>
@@ -25,28 +23,26 @@ define <16 x half> @sitofp_i32(<16 x i32> %a) #0 {
 define <16 x half> @sitofp_i64(<16 x i64> %a) #0 {
 ; CHECK-LABEL: sitofp_i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    scvtf v2.2d, v2.2d
 ; CHECK-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-NEXT:    scvtf v6.2d, v6.2d
 ; CHECK-NEXT:    scvtf v4.2d, v4.2d
-; CHECK-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-NEXT:    scvtf v2.2d, v2.2d
 ; CHECK-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-NEXT:    scvtf v7.2d, v7.2d
+; CHECK-NEXT:    scvtf v6.2d, v6.2d
 ; CHECK-NEXT:    scvtf v5.2d, v5.2d
-; CHECK-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-NEXT:    scvtf v7.2d, v7.2d
 ; CHECK-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-NEXT:    fcvtn v6.2s, v6.2d
 ; CHECK-NEXT:    fcvtn v4.2s, v4.2d
-; CHECK-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-NEXT:    fcvtn v6.2s, v6.2d
 ; CHECK-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-NEXT:    fcvtn2 v6.4s, v7.2d
 ; CHECK-NEXT:    fcvtn2 v4.4s, v5.2d
-; CHECK-NEXT:    fcvtn v2.4h, v2.4s
+; CHECK-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-NEXT:    fcvtn2 v6.4s, v7.2d
 ; CHECK-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-NEXT:    fcvtn v3.4h, v6.4s
 ; CHECK-NEXT:    fcvtn v1.4h, v4.4s
-; CHECK-NEXT:    mov v0.d[1], v2.d[0]
-; CHECK-NEXT:    mov v1.d[1], v3.d[0]
+; CHECK-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-NEXT:    fcvtn2 v1.8h, v6.4s
 ; CHECK-NEXT:    ret
 
 
@@ -60,16 +56,14 @@ define <16 x half> @sitofp_i64(<16 x i64> %a) #0 {
 define <16 x half> @uitofp_i32(<16 x i32> %a) #0 {
 ; CHECK-LABEL: uitofp_i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ucvtf v1.4s, v1.4s
-; CHECK-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-NEXT:    ucvtf v3.4s, v3.4s
 ; CHECK-NEXT:    ucvtf v2.4s, v2.4s
-; CHECK-NEXT:    fcvtn v4.4h, v1.4s
+; CHECK-NEXT:    ucvtf v0.4s, v0.4s
+; CHECK-NEXT:    ucvtf v4.4s, v1.4s
 ; CHECK-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-NEXT:    fcvtn v3.4h, v3.4s
 ; CHECK-NEXT:    fcvtn v1.4h, v2.4s
-; CHECK-NEXT:    mov v0.d[1], v4.d[0]
-; CHECK-NEXT:    mov v1.d[1], v3.d[0]
+; CHECK-NEXT:    ucvtf v2.4s, v3.4s
+; CHECK-NEXT:    fcvtn2 v0.8h, v4.4s
+; CHECK-NEXT:    fcvtn2 v1.8h, v2.4s
 ; CHECK-NEXT:    ret
 
   %1 = uitofp <16 x i32> %a to <16 x half>
@@ -80,28 +74,26 @@ define <16 x half> @uitofp_i32(<16 x i32> %a) #0 {
 define <16 x half> @uitofp_i64(<16 x i64> %a) #0 {
 ; CHECK-LABEL: uitofp_i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ucvtf v2.2d, v2.2d
 ; CHECK-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-NEXT:    ucvtf v6.2d, v6.2d
 ; CHECK-NEXT:    ucvtf v4.2d, v4.2d
-; CHECK-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-NEXT:    ucvtf v2.2d, v2.2d
 ; CHECK-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-NEXT:    ucvtf v7.2d, v7.2d
+; CHECK-NEXT:    ucvtf v6.2d, v6.2d
 ; CHECK-NEXT:    ucvtf v5.2d, v5.2d
-; CHECK-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-NEXT:    ucvtf v7.2d, v7.2d
 ; CHECK-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-NEXT:    fcvtn v6.2s, v6.2d
 ; CHECK-NEXT:    fcvtn v4.2s, v4.2d
-; CHECK-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-NEXT:    fcvtn v6.2s, v6.2d
 ; CHECK-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-NEXT:    fcvtn2 v6.4s, v7.2d
 ; CHECK-NEXT:    fcvtn2 v4.4s, v5.2d
-; CHECK-NEXT:    fcvtn v2.4h, v2.4s
+; CHECK-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-NEXT:    fcvtn2 v6.4s, v7.2d
 ; CHECK-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-NEXT:    fcvtn v3.4h, v6.4s
 ; CHECK-NEXT:    fcvtn v1.4h, v4.4s
-; CHECK-NEXT:    mov v0.d[1], v2.d[0]
-; CHECK-NEXT:    mov v1.d[1], v3.d[0]
+; CHECK-NEXT:    fcvtn2 v0.8h, v2.4s
+; CHECK-NEXT:    fcvtn2 v1.8h, v6.4s
 ; CHECK-NEXT:    ret
 
 
diff --git a/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll b/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
index 92d99ead70168..d46d31e297490 100644
--- a/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/fp16-v8-instructions.ll
@@ -302,9 +302,8 @@ entry:
 define <8 x half> @s_to_h(<8 x float> %a) {
 ; CHECK-LABEL: s_to_h:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    fcvtn v1.4h, v1.4s
 ; CHECK-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-NEXT:    ret
   %1 = fptrunc <8 x float> %a to <8 x half>
   ret <8 x half> %1
@@ -417,13 +416,12 @@ define <8 x half> @sitofp_v8i8(<8 x i8> %a) #0 {
 ; CHECK-CVT-LABEL: sitofp_v8i8:
 ; CHECK-CVT:       // %bb.0:
 ; CHECK-CVT-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-CVT-NEXT:    sshll2 v1.4s, v0.8h, #0
-; CHECK-CVT-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-CVT-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-CVT-NEXT:    sshll2 v2.4s, v0.8h, #0
 ; CHECK-CVT-NEXT:    scvtf v1.4s, v1.4s
-; CHECK-CVT-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-CVT-NEXT:    fcvtn v1.4h, v1.4s
-; CHECK-CVT-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-CVT-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-CVT-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-CVT-NEXT:    scvtf v1.4s, v2.4s
+; CHECK-CVT-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: sitofp_v8i8:
@@ -440,20 +438,18 @@ define <16 x half> @sitofp_v16i8(<16 x i8> %a) #0 {
 ; CHECK-CVT:       // %bb.0:
 ; CHECK-CVT-NEXT:    sshll2 v1.8h, v0.16b, #0
 ; CHECK-CVT-NEXT:    sshll v0.8h, v0.8b, #0
-; CHECK-CVT-NEXT:    sshll2 v2.4s, v1.8h, #0
-; CHECK-CVT-NEXT:    sshll v1.4s, v1.4h, #0
-; CHECK-CVT-NEXT:    sshll2 v3.4s, v0.8h, #0
-; CHECK-CVT-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-CVT-NEXT:    sshll v2.4s, v1.4h, #0
+; CHECK-CVT-NEXT:    sshll v3.4s, v0.4h, #0
 ; CHECK-CVT-NEXT:    scvtf v2.4s, v2.4s
-; CHECK-CVT-NEXT:    scvtf v1.4s, v1.4s
 ; CHECK-CVT-NEXT:    scvtf v3.4s, v3.4s
-; CHECK-CVT-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-CVT-NEXT:    fcvtn v2.4h, v2.4s
-; CHECK-CVT-NEXT:    fcvtn v1.4h, v1.4s
-; CHECK-CVT-NEXT:    fcvtn v3.4h, v3.4s
-; CHECK-CVT-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-CVT-NEXT:    mov v1.d[1], v2.d[0]
-; CHECK-CVT-NEXT:    mov v0.d[1], v3.d[0]
+; CHECK-CVT-NEXT:    sshll2 v1.4s, v1.8h, #0
+; CHECK-CVT-NEXT:    sshll2 v4.4s, v0.8h, #0
+; CHECK-CVT-NEXT:    scvtf v5.4s, v1.4s
+; CHECK-CVT-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-CVT-NEXT:    fcvtn v0.4h, v3.4s
+; CHECK-CVT-NEXT:    scvtf v2.4s, v4.4s
+; CHECK-CVT-NEXT:    fcvtn2 v1.8h, v5.4s
+; CHECK-CVT-NEXT:    fcvtn2 v0.8h, v2.4s
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: sitofp_v16i8:
@@ -470,13 +466,12 @@ define <16 x half> @sitofp_v16i8(<16 x i8> %a) #0 {
 define <8 x half> @sitofp_i16(<8 x i16> %a) #0 {
 ; CHECK-CVT-LABEL: sitofp_i16:
 ; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    sshll2 v1.4s, v0.8h, #0
-; CHECK-CVT-NEXT:    sshll v0.4s, v0.4h, #0
+; CHECK-CVT-NEXT:    sshll v1.4s, v0.4h, #0
+; CHECK-CVT-NEXT:    sshll2 v2.4s, v0.8h, #0
 ; CHECK-CVT-NEXT:    scvtf v1.4s, v1.4s
-; CHECK-CVT-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-CVT-NEXT:    fcvtn v1.4h, v1.4s
-; CHECK-CVT-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-CVT-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-CVT-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-CVT-NEXT:    scvtf v1.4s, v2.4s
+; CHECK-CVT-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: sitofp_i16:
@@ -490,11 +485,10 @@ define <8 x half> @sitofp_i16(<8 x i16> %a) #0 {
 define <8 x half> @sitofp_i32(<8 x i32> %a) #0 {
 ; CHECK-LABEL: sitofp_i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    scvtf v1.4s, v1.4s
 ; CHECK-NEXT:    scvtf v0.4s, v0.4s
-; CHECK-NEXT:    fcvtn v1.4h, v1.4s
+; CHECK-NEXT:    scvtf v1.4s, v1.4s
 ; CHECK-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-NEXT:    ret
   %1 = sitofp <8 x i32> %a to <8 x half>
   ret <8 x half> %1
@@ -504,17 +498,16 @@ define <8 x half> @sitofp_i32(<8 x i32> %a) #0 {
 define <8 x half> @sitofp_i64(<8 x i64> %a) #0 {
 ; CHECK-LABEL: sitofp_i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    scvtf v2.2d, v2.2d
 ; CHECK-NEXT:    scvtf v0.2d, v0.2d
-; CHECK-NEXT:    scvtf v3.2d, v3.2d
+; CHECK-NEXT:    scvtf v2.2d, v2.2d
 ; CHECK-NEXT:    scvtf v1.2d, v1.2d
-; CHECK-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-NEXT:    scvtf v3.2d, v3.2d
 ; CHECK-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-NEXT:    fcvtn v2.2s, v2.2d
 ; CHECK-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-NEXT:    fcvtn2 v2.4s, v3.2d
 ; CHECK-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    fcvtn2 v0.8h, v2.4s
 ; CHECK-NEXT:    ret
   %1 = sitofp <8 x i64> %a to <8 x half>
   ret <8 x half> %1
@@ -542,13 +535,12 @@ define <8 x half> @uitofp_v8i8(<8 x i8> %a) #0 {
 ; CHECK-CVT-LABEL: uitofp_v8i8:
 ; CHECK-CVT:       // %bb.0:
 ; CHECK-CVT-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-CVT-NEXT:    ushll2 v1.4s, v0.8h, #0
-; CHECK-CVT-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-CVT-NEXT:    ushll v1.4s, v0.4h, #0
+; CHECK-CVT-NEXT:    ushll2 v2.4s, v0.8h, #0
 ; CHECK-CVT-NEXT:    ucvtf v1.4s, v1.4s
-; CHECK-CVT-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-CVT-NEXT:    fcvtn v1.4h, v1.4s
-; CHECK-CVT-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-CVT-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-CVT-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-CVT-NEXT:    ucvtf v1.4s, v2.4s
+; CHECK-CVT-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: uitofp_v8i8:
@@ -565,20 +557,18 @@ define <16 x half> @uitofp_v16i8(<16 x i8> %a) #0 {
 ; CHECK-CVT:       // %bb.0:
 ; CHECK-CVT-NEXT:    ushll2 v1.8h, v0.16b, #0
 ; CHECK-CVT-NEXT:    ushll v0.8h, v0.8b, #0
-; CHECK-CVT-NEXT:    ushll2 v2.4s, v1.8h, #0
-; CHECK-CVT-NEXT:    ushll v1.4s, v1.4h, #0
-; CHECK-CVT-NEXT:    ushll2 v3.4s, v0.8h, #0
-; CHECK-CVT-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-CVT-NEXT:    ushll v2.4s, v1.4h, #0
+; CHECK-CVT-NEXT:    ushll v3.4s, v0.4h, #0
 ; CHECK-CVT-NEXT:    ucvtf v2.4s, v2.4s
-; CHECK-CVT-NEXT:    ucvtf v1.4s, v1.4s
 ; CHECK-CVT-NEXT:    ucvtf v3.4s, v3.4s
-; CHECK-CVT-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-CVT-NEXT:    fcvtn v2.4h, v2.4s
-; CHECK-CVT-NEXT:    fcvtn v1.4h, v1.4s
-; CHECK-CVT-NEXT:    fcvtn v3.4h, v3.4s
-; CHECK-CVT-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-CVT-NEXT:    mov v1.d[1], v2.d[0]
-; CHECK-CVT-NEXT:    mov v0.d[1], v3.d[0]
+; CHECK-CVT-NEXT:    ushll2 v1.4s, v1.8h, #0
+; CHECK-CVT-NEXT:    ushll2 v4.4s, v0.8h, #0
+; CHECK-CVT-NEXT:    ucvtf v5.4s, v1.4s
+; CHECK-CVT-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-CVT-NEXT:    fcvtn v0.4h, v3.4s
+; CHECK-CVT-NEXT:    ucvtf v2.4s, v4.4s
+; CHECK-CVT-NEXT:    fcvtn2 v1.8h, v5.4s
+; CHECK-CVT-NEXT:    fcvtn2 v0.8h, v2.4s
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: uitofp_v16i8:
@@ -596,13 +586,12 @@ define <16 x half> @uitofp_v16i8(<16 x i8> %a) #0 {
 define <8 x half> @uitofp_i16(<8 x i16> %a) #0 {
 ; CHECK-CVT-LABEL: uitofp_i16:
 ; CHECK-CVT:       // %bb.0:
-; CHECK-CVT-NEXT:    ushll2 v1.4s, v0.8h, #0
-; CHECK-CVT-NEXT:    ushll v0.4s, v0.4h, #0
+; CHECK-CVT-NEXT:    ushll v1.4s, v0.4h, #0
+; CHECK-CVT-NEXT:    ushll2 v2.4s, v0.8h, #0
 ; CHECK-CVT-NEXT:    ucvtf v1.4s, v1.4s
-; CHECK-CVT-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-CVT-NEXT:    fcvtn v1.4h, v1.4s
-; CHECK-CVT-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-CVT-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-CVT-NEXT:    fcvtn v0.4h, v1.4s
+; CHECK-CVT-NEXT:    ucvtf v1.4s, v2.4s
+; CHECK-CVT-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-CVT-NEXT:    ret
 ;
 ; CHECK-FP16-LABEL: uitofp_i16:
@@ -617,11 +606,10 @@ define <8 x half> @uitofp_i16(<8 x i16> %a) #0 {
 define <8 x half> @uitofp_i32(<8 x i32> %a) #0 {
 ; CHECK-LABEL: uitofp_i32:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ucvtf v1.4s, v1.4s
 ; CHECK-NEXT:    ucvtf v0.4s, v0.4s
-; CHECK-NEXT:    fcvtn v1.4h, v1.4s
+; CHECK-NEXT:    ucvtf v1.4s, v1.4s
 ; CHECK-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    fcvtn2 v0.8h, v1.4s
 ; CHECK-NEXT:    ret
   %1 = uitofp <8 x i32> %a to <8 x half>
   ret <8 x half> %1
@@ -631,17 +619,16 @@ define <8 x half> @uitofp_i32(<8 x i32> %a) #0 {
 define <8 x half> @uitofp_i64(<8 x i64> %a) #0 {
 ; CHECK-LABEL: uitofp_i64:
 ; CHECK:       // %bb.0:
-; CHECK-NEXT:    ucvtf v2.2d, v2.2d
 ; CHECK-NEXT:    ucvtf v0.2d, v0.2d
-; CHECK-NEXT:    ucvtf v3.2d, v3.2d
+; CHECK-NEXT:    ucvtf v2.2d, v2.2d
 ; CHECK-NEXT:    ucvtf v1.2d, v1.2d
-; CHECK-NEXT:    fcvtn v2.2s, v2.2d
+; CHECK-NEXT:    ucvtf v3.2d, v3.2d
 ; CHECK-NEXT:    fcvtn v0.2s, v0.2d
-; CHECK-NEXT:    fcvtn2 v2.4s, v3.2d
+; CHECK-NEXT:    fcvtn v2.2s, v2.2d
 ; CHECK-NEXT:    fcvtn2 v0.4s, v1.2d
-; CHECK-NEXT:    fcvtn v1.4h, v2.4s
+; CHECK-NEXT:    fcvtn2 v2.4s, v3.2d
 ; CHECK-NEXT:    fcvtn v0.4h, v0.4s
-; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    fcvtn2 v0.8h, v2.4s
 ; CHECK-NEXT:    ret
   %1 = uitofp <8 x i64> %a to <8 x half>
   ret <8 x half> %1
diff --git a/llvm/test/CodeGen/AArch64/vector-fcopysign.ll b/llvm/test/CodeGen/AArch64/vector-fcopysign.ll
index 754d9e8eb7eda..b69b41da9ee72 100644
--- a/llvm/test/CodeGen/AArch64/vector-fcopysign.ll
+++ b/llvm/test/CodeGen/AArch64/vector-fcopysign.ll
@@ -462,9 +462,8 @@ define <8 x half> @test_copysign_v8f16_v8f32(<8 x half> %a, <8 x float> %b) #0 {
 ;
 ; FP16-LABEL: test_copysign_v8f16_v8f32:
 ; FP16:       ; %bb.0:
-; FP16-NEXT:    fcvtn v2.4h, v2.4s
 ; FP16-NEXT:    fcvtn v1.4h, v1.4s
-; FP16-NEXT:    mov.d v1[1], v2[0]
+; FP16-NEXT:    fcvtn2 v1.8h, v2.4s
 ; FP16-NEXT:    mvni.8h v2, #128, lsl #8
 ; FP16-NEXT:    bif.16b v0, v1, v2
 ; FP16-NEXT:    ret

From 2efdacf74c54a14ff72e9ab4b2e0707e5b3b86a8 Mon Sep 17 00:00:00 2001
From: Weining Lu <luweining@loongson.cn>
Date: Tue, 20 Jun 2023 17:21:31 +0800
Subject: [PATCH 123/130] [LoongArch] Add missing chains and remove unnecessary
 `SDNPSideEffect` property for some intrinsic nodes

---
 .../LoongArch/LoongArchISelLowering.cpp       | 100 ++++++++----------
 .../Target/LoongArch/LoongArchInstrInfo.td    |  31 +++---
 llvm/test/CodeGen/LoongArch/intrinsic-la64.ll |  92 ++++++++++++++++
 llvm/test/CodeGen/LoongArch/intrinsic.ll      |  56 ++++++++++
 4 files changed, 209 insertions(+), 70 deletions(-)

diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
index 5361856fc9f62..74a16bf3f8208 100644
--- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
+++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp
@@ -633,11 +633,8 @@ LoongArchTargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op,
     unsigned Imm = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
     return !isUInt<14>(Imm)
                ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsgOOR, DAG)
-               : DAG.getMergeValues(
-                     {DAG.getNode(LoongArchISD::CSRRD, DL, GRLenVT, Chain,
-                                  DAG.getConstant(Imm, DL, GRLenVT)),
-                      Chain},
-                     DL);
+               : DAG.getNode(LoongArchISD::CSRRD, DL, {GRLenVT, MVT::Other},
+                             {Chain, DAG.getConstant(Imm, DL, GRLenVT)});
   }
   case Intrinsic::loongarch_csrwr_w:
   case Intrinsic::loongarch_csrwr_d: {
@@ -658,29 +655,22 @@ LoongArchTargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op,
                               DAG.getConstant(Imm, DL, GRLenVT)});
   }
   case Intrinsic::loongarch_iocsrrd_d: {
-    return DAG.getMergeValues(
-        {DAG.getNode(
-             LoongArchISD::IOCSRRD_D, DL, GRLenVT, Chain,
-             DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(2))),
-         Chain},
-        DL);
+    return DAG.getNode(
+        LoongArchISD::IOCSRRD_D, DL, {GRLenVT, MVT::Other},
+        {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(2))});
   }
 #define IOCSRRD_CASE(NAME, NODE)                                               \
   case Intrinsic::loongarch_##NAME: {                                          \
-    return DAG.getMergeValues({DAG.getNode(LoongArchISD::NODE, DL, GRLenVT,    \
-                                           Chain, Op.getOperand(2)),           \
-                               Chain},                                         \
-                              DL);                                             \
+    return DAG.getNode(LoongArchISD::NODE, DL, {GRLenVT, MVT::Other},          \
+                       {Chain, Op.getOperand(2)});                             \
   }
     IOCSRRD_CASE(iocsrrd_b, IOCSRRD_B);
     IOCSRRD_CASE(iocsrrd_h, IOCSRRD_H);
     IOCSRRD_CASE(iocsrrd_w, IOCSRRD_W);
 #undef IOCSRRD_CASE
   case Intrinsic::loongarch_cpucfg: {
-    return DAG.getMergeValues({DAG.getNode(LoongArchISD::CPUCFG, DL, GRLenVT,
-                                           Chain, Op.getOperand(2)),
-                               Chain},
-                              DL);
+    return DAG.getNode(LoongArchISD::CPUCFG, DL, {GRLenVT, MVT::Other},
+                       {Chain, Op.getOperand(2)});
   }
   case Intrinsic::loongarch_lddir_d: {
     unsigned Imm = cast<ConstantSDNode>(Op.getOperand(3))->getZExtValue();
@@ -694,11 +684,8 @@ LoongArchTargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op,
     unsigned Imm = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
     return !isUInt<2>(Imm)
                ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsgOOR, DAG)
-               : DAG.getMergeValues(
-                     {DAG.getNode(LoongArchISD::MOVFCSR2GR, DL, VT,
-                                  DAG.getConstant(Imm, DL, GRLenVT)),
-                      Chain},
-                     DL);
+               : DAG.getNode(LoongArchISD::MOVFCSR2GR, DL, {VT, MVT::Other},
+                             {Chain, DAG.getConstant(Imm, DL, GRLenVT)});
   }
   }
 }
@@ -1121,22 +1108,22 @@ void LoongArchTargetLowering::ReplaceNodeResults(
                                                      ErrorMsgOOR);
         return;
       }
+      SDValue MOVFCSR2GRResults = DAG.getNode(
+          LoongArchISD::MOVFCSR2GR, SDLoc(N), {MVT::i64, MVT::Other},
+          {Chain, DAG.getConstant(Imm, DL, GRLenVT)});
       Results.push_back(
-          DAG.getNode(ISD::TRUNCATE, DL, VT,
-                      DAG.getNode(LoongArchISD::MOVFCSR2GR, SDLoc(N), MVT::i64,
-                                  DAG.getConstant(Imm, DL, GRLenVT))));
-      Results.push_back(Chain);
+          DAG.getNode(ISD::TRUNCATE, DL, VT, MOVFCSR2GRResults.getValue(0)));
+      Results.push_back(MOVFCSR2GRResults.getValue(1));
       break;
     }
 #define CRC_CASE_EXT_BINARYOP(NAME, NODE)                                      \
   case Intrinsic::loongarch_##NAME: {                                          \
-    Results.push_back(DAG.getNode(                                             \
-        ISD::TRUNCATE, DL, VT,                                                 \
-        DAG.getNode(                                                           \
-            LoongArchISD::NODE, DL, MVT::i64,                                  \
-            DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2),                   \
-            DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3)))));   \
-    Results.push_back(Chain);                                                  \
+    SDValue NODE = DAG.getNode(                                                \
+        LoongArchISD::NODE, DL, {MVT::i64, MVT::Other},                        \
+        {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2),               \
+         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3))});       \
+    Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NODE.getValue(0)));   \
+    Results.push_back(NODE.getValue(1));                                       \
     break;                                                                     \
   }
       CRC_CASE_EXT_BINARYOP(crc_w_b_w, CRC_W_B_W)
@@ -1149,12 +1136,12 @@ void LoongArchTargetLowering::ReplaceNodeResults(
 
 #define CRC_CASE_EXT_UNARYOP(NAME, NODE)                                       \
   case Intrinsic::loongarch_##NAME: {                                          \
-    Results.push_back(                                                         \
-        DAG.getNode(ISD::TRUNCATE, DL, VT,                                     \
-                    DAG.getNode(LoongArchISD::NODE, DL, MVT::i64, Op2,         \
-                                DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64,     \
-                                            N->getOperand(3)))));              \
-    Results.push_back(Chain);                                                  \
+    SDValue NODE = DAG.getNode(                                                \
+        LoongArchISD::NODE, DL, {MVT::i64, MVT::Other},                        \
+        {Chain, Op2,                                                           \
+         DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3))});       \
+    Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NODE.getValue(0)));   \
+    Results.push_back(NODE.getValue(1));                                                  \
     break;                                                                     \
   }
       CRC_CASE_EXT_UNARYOP(crc_w_d_w, CRC_W_D_W)
@@ -1179,11 +1166,12 @@ void LoongArchTargetLowering::ReplaceNodeResults(
                                                      ErrorMsgOOR);
         return;
       }
+      SDValue CSRRDResults =
+          DAG.getNode(LoongArchISD::CSRRD, DL, {GRLenVT, MVT::Other},
+                      {Chain, DAG.getConstant(Imm, DL, GRLenVT)});
       Results.push_back(
-          DAG.getNode(ISD::TRUNCATE, DL, VT,
-                      DAG.getNode(LoongArchISD::CSRRD, DL, GRLenVT, Chain,
-                                  DAG.getConstant(Imm, DL, GRLenVT))));
-      Results.push_back(Chain);
+          DAG.getNode(ISD::TRUNCATE, DL, VT, CSRRDResults.getValue(0)));
+      Results.push_back(CSRRDResults.getValue(1));
       break;
     }
     case Intrinsic::loongarch_csrwr_w: {
@@ -1221,11 +1209,12 @@ void LoongArchTargetLowering::ReplaceNodeResults(
     }
 #define IOCSRRD_CASE(NAME, NODE)                                               \
   case Intrinsic::loongarch_##NAME: {                                          \
-    Results.push_back(DAG.getNode(                                             \
-        ISD::TRUNCATE, DL, VT,                                                 \
-        DAG.getNode(LoongArchISD::NODE, DL, MVT::i64, Chain,                   \
-                    DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2))));        \
-    Results.push_back(Chain);                                                  \
+    SDValue IOCSRRDResults =                                                   \
+        DAG.getNode(LoongArchISD::NODE, DL, {MVT::i64, MVT::Other},            \
+                    {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2)}); \
+    Results.push_back(                                                         \
+        DAG.getNode(ISD::TRUNCATE, DL, VT, IOCSRRDResults.getValue(0)));       \
+    Results.push_back(IOCSRRDResults.getValue(1));                             \
     break;                                                                     \
   }
       IOCSRRD_CASE(iocsrrd_b, IOCSRRD_B);
@@ -1233,11 +1222,12 @@ void LoongArchTargetLowering::ReplaceNodeResults(
       IOCSRRD_CASE(iocsrrd_w, IOCSRRD_W);
 #undef IOCSRRD_CASE
     case Intrinsic::loongarch_cpucfg: {
-      Results.push_back(DAG.getNode(
-          ISD::TRUNCATE, DL, VT,
-          DAG.getNode(LoongArchISD::CPUCFG, DL, GRLenVT, Chain,
-                      DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2))));
-      Results.push_back(Chain);
+      SDValue CPUCFGResults =
+          DAG.getNode(LoongArchISD::CPUCFG, DL, {GRLenVT, MVT::Other},
+                      {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2)});
+      Results.push_back(
+          DAG.getNode(ISD::TRUNCATE, DL, VT, CPUCFGResults.getValue(0)));
+      Results.push_back(CPUCFGResults.getValue(1));
       break;
     }
     case Intrinsic::loongarch_lddir_d: {
diff --git a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
index 8ea8b76f6797c..3fbcc3e05a996 100644
--- a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
+++ b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
@@ -75,21 +75,21 @@ def loongarch_srl_w : SDNode<"LoongArchISD::SRL_W", SDT_LoongArchIntBinOpW>;
 def loongarch_rotr_w : SDNode<"LoongArchISD::ROTR_W", SDT_LoongArchIntBinOpW>;
 def loongarch_rotl_w : SDNode<"LoongArchISD::ROTL_W", SDT_LoongArchIntBinOpW>;
 def loongarch_crc_w_b_w
-    : SDNode<"LoongArchISD::CRC_W_B_W", SDT_LoongArchIntBinOpW>;
+    : SDNode<"LoongArchISD::CRC_W_B_W", SDT_LoongArchIntBinOpW, [SDNPHasChain]>;
 def loongarch_crc_w_h_w
-    : SDNode<"LoongArchISD::CRC_W_H_W", SDT_LoongArchIntBinOpW>;
+    : SDNode<"LoongArchISD::CRC_W_H_W", SDT_LoongArchIntBinOpW, [SDNPHasChain]>;
 def loongarch_crc_w_w_w
-    : SDNode<"LoongArchISD::CRC_W_W_W", SDT_LoongArchIntBinOpW>;
+    : SDNode<"LoongArchISD::CRC_W_W_W", SDT_LoongArchIntBinOpW, [SDNPHasChain]>;
 def loongarch_crc_w_d_w
-    : SDNode<"LoongArchISD::CRC_W_D_W", SDT_LoongArchIntBinOpW>;
+    : SDNode<"LoongArchISD::CRC_W_D_W", SDT_LoongArchIntBinOpW, [SDNPHasChain]>;
 def loongarch_crcc_w_b_w
-    : SDNode<"LoongArchISD::CRCC_W_B_W", SDT_LoongArchIntBinOpW>;
+    : SDNode<"LoongArchISD::CRCC_W_B_W", SDT_LoongArchIntBinOpW, [SDNPHasChain]>;
 def loongarch_crcc_w_h_w
-    : SDNode<"LoongArchISD::CRCC_W_H_W", SDT_LoongArchIntBinOpW>;
+    : SDNode<"LoongArchISD::CRCC_W_H_W", SDT_LoongArchIntBinOpW, [SDNPHasChain]>;
 def loongarch_crcc_w_w_w
-    : SDNode<"LoongArchISD::CRCC_W_W_W", SDT_LoongArchIntBinOpW>;
+    : SDNode<"LoongArchISD::CRCC_W_W_W", SDT_LoongArchIntBinOpW, [SDNPHasChain]>;
 def loongarch_crcc_w_d_w
-    : SDNode<"LoongArchISD::CRCC_W_D_W", SDT_LoongArchIntBinOpW>;
+    : SDNode<"LoongArchISD::CRCC_W_D_W", SDT_LoongArchIntBinOpW, [SDNPHasChain]>;
 def loongarch_bstrins
     : SDNode<"LoongArchISD::BSTRINS", SDT_LoongArchBStrIns>;
 def loongarch_bstrpick
@@ -106,26 +106,27 @@ def loongarch_ibar : SDNode<"LoongArchISD::IBAR", SDT_LoongArchVI,
                              [SDNPHasChain, SDNPSideEffect]>;
 def loongarch_break : SDNode<"LoongArchISD::BREAK", SDT_LoongArchVI,
                               [SDNPHasChain, SDNPSideEffect]>;
-def loongarch_movfcsr2gr : SDNode<"LoongArchISD::MOVFCSR2GR", SDT_LoongArchMovfcsr2gr>;
+def loongarch_movfcsr2gr : SDNode<"LoongArchISD::MOVFCSR2GR", SDT_LoongArchMovfcsr2gr,
+                                  [SDNPHasChain]>;
 def loongarch_movgr2fcsr : SDNode<"LoongArchISD::MOVGR2FCSR", SDT_LoongArchMovgr2fcsr,
                                    [SDNPHasChain, SDNPSideEffect]>;
 def loongarch_syscall : SDNode<"LoongArchISD::SYSCALL", SDT_LoongArchVI,
                                 [SDNPHasChain, SDNPSideEffect]>;
 def loongarch_csrrd : SDNode<"LoongArchISD::CSRRD", SDT_LoongArchCsrrd,
-                              [SDNPHasChain, SDNPSideEffect]>;
+                              [SDNPHasChain]>;
 def loongarch_csrwr : SDNode<"LoongArchISD::CSRWR", SDT_LoongArchCsrwr,
                               [SDNPHasChain, SDNPSideEffect]>;
 def loongarch_csrxchg : SDNode<"LoongArchISD::CSRXCHG",
                                 SDT_LoongArchCsrxchg,
                                 [SDNPHasChain, SDNPSideEffect]>;
 def loongarch_iocsrrd_b : SDNode<"LoongArchISD::IOCSRRD_B", SDTUnaryOp,
-                                  [SDNPHasChain, SDNPSideEffect]>;
+                                  [SDNPHasChain]>;
 def loongarch_iocsrrd_h : SDNode<"LoongArchISD::IOCSRRD_H", SDTUnaryOp,
-                                  [SDNPHasChain, SDNPSideEffect]>;
+                                  [SDNPHasChain]>;
 def loongarch_iocsrrd_w : SDNode<"LoongArchISD::IOCSRRD_W", SDTUnaryOp,
-                                  [SDNPHasChain, SDNPSideEffect]>;
+                                  [SDNPHasChain]>;
 def loongarch_iocsrrd_d : SDNode<"LoongArchISD::IOCSRRD_D", SDTUnaryOp,
-                                  [SDNPHasChain, SDNPSideEffect]>;
+                                  [SDNPHasChain]>;
 def loongarch_iocsrwr_b : SDNode<"LoongArchISD::IOCSRWR_B",
                                   SDT_LoongArchIocsrwr,
                                   [SDNPHasChain, SDNPSideEffect]>;
@@ -139,7 +140,7 @@ def loongarch_iocsrwr_d : SDNode<"LoongArchISD::IOCSRWR_D",
                                   SDT_LoongArchIocsrwr,
                                   [SDNPHasChain, SDNPSideEffect]>;
 def loongarch_cpucfg : SDNode<"LoongArchISD::CPUCFG", SDTUnaryOp,
-                               [SDNPHasChain, SDNPSideEffect]>;
+                               [SDNPHasChain]>;
 
 //===----------------------------------------------------------------------===//
 // Operand and SDNode transformation definitions.
diff --git a/llvm/test/CodeGen/LoongArch/intrinsic-la64.ll b/llvm/test/CodeGen/LoongArch/intrinsic-la64.ll
index 8b251be99647c..94e3f532ed474 100644
--- a/llvm/test/CodeGen/LoongArch/intrinsic-la64.ll
+++ b/llvm/test/CodeGen/LoongArch/intrinsic-la64.ll
@@ -29,6 +29,14 @@ define i32 @crc_w_b_w(i32 %a, i32 %b) nounwind {
   ret i32 %res
 }
 
+define void @crc_w_b_w_noret(i32 %a, i32 %b) nounwind {
+; CHECK-LABEL: crc_w_b_w_noret:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret
+  %res = call i32 @llvm.loongarch.crc.w.b.w(i32 %a, i32 %b)
+  ret void
+}
+
 define i32 @crc_w_h_w(i32 %a, i32 %b) nounwind {
 ; CHECK-LABEL: crc_w_h_w:
 ; CHECK:       # %bb.0:
@@ -38,6 +46,14 @@ define i32 @crc_w_h_w(i32 %a, i32 %b) nounwind {
   ret i32 %res
 }
 
+define void @crc_w_h_w_noret(i32 %a, i32 %b) nounwind {
+; CHECK-LABEL: crc_w_h_w_noret:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret
+  %res = call i32 @llvm.loongarch.crc.w.h.w(i32 %a, i32 %b)
+  ret void
+}
+
 define i32 @crc_w_w_w(i32 %a, i32 %b) nounwind {
 ; CHECK-LABEL: crc_w_w_w:
 ; CHECK:       # %bb.0:
@@ -47,6 +63,14 @@ define i32 @crc_w_w_w(i32 %a, i32 %b) nounwind {
   ret i32 %res
 }
 
+define void @crc_w_w_w_noret(i32 %a, i32 %b) nounwind {
+; CHECK-LABEL: crc_w_w_w_noret:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret
+  %res = call i32 @llvm.loongarch.crc.w.w.w(i32 %a, i32 %b)
+  ret void
+}
+
 define void @cacop_d(i64 %a) nounwind {
 ; CHECK-LABEL: cacop_d:
 ; CHECK:       # %bb.0:
@@ -65,6 +89,14 @@ define i32 @crc_w_d_w(i64 %a, i32 %b) nounwind {
   ret i32 %res
 }
 
+define void @crc_w_d_w_noret(i64 %a, i32 %b) nounwind {
+; CHECK-LABEL: crc_w_d_w_noret:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret
+  %res = call i32 @llvm.loongarch.crc.w.d.w(i64 %a, i32 %b)
+  ret void
+}
+
 define i32 @crcc_w_b_w(i32 %a, i32 %b) nounwind {
 ; CHECK-LABEL: crcc_w_b_w:
 ; CHECK:       # %bb.0:
@@ -74,6 +106,14 @@ define i32 @crcc_w_b_w(i32 %a, i32 %b) nounwind {
   ret i32 %res
 }
 
+define void @crcc_w_b_w_noret(i32 %a, i32 %b) nounwind {
+; CHECK-LABEL: crcc_w_b_w_noret:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret
+  %res = call i32 @llvm.loongarch.crcc.w.b.w(i32 %a, i32 %b)
+  ret void
+}
+
 define i32 @crcc_w_h_w(i32 %a, i32 %b) nounwind {
 ; CHECK-LABEL: crcc_w_h_w:
 ; CHECK:       # %bb.0:
@@ -83,6 +123,14 @@ define i32 @crcc_w_h_w(i32 %a, i32 %b) nounwind {
   ret i32 %res
 }
 
+define void @crcc_w_h_w_noret(i32 %a, i32 %b) nounwind {
+; CHECK-LABEL: crcc_w_h_w_noret:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret
+  %res = call i32 @llvm.loongarch.crcc.w.h.w(i32 %a, i32 %b)
+  ret void
+}
+
 define i32 @crcc_w_w_w(i32 %a, i32 %b) nounwind {
 ; CHECK-LABEL: crcc_w_w_w:
 ; CHECK:       # %bb.0:
@@ -92,6 +140,14 @@ define i32 @crcc_w_w_w(i32 %a, i32 %b) nounwind {
   ret i32 %res
 }
 
+define void @crcc_w_w_w_noret(i32 %a, i32 %b) nounwind {
+; CHECK-LABEL: crcc_w_w_w_noret:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret
+  %res = call i32 @llvm.loongarch.crcc.w.w.w(i32 %a, i32 %b)
+  ret void
+}
+
 define i32 @crcc_w_d_w(i64 %a, i32 %b) nounwind {
 ; CHECK-LABEL: crcc_w_d_w:
 ; CHECK:       # %bb.0:
@@ -101,6 +157,14 @@ define i32 @crcc_w_d_w(i64 %a, i32 %b) nounwind {
   ret i32 %res
 }
 
+define void @crcc_w_d_w_noret(i64 %a, i32 %b) nounwind {
+; CHECK-LABEL: crcc_w_d_w_noret:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret
+  %res = call i32 @llvm.loongarch.crcc.w.d.w(i64 %a, i32 %b)
+  ret void
+}
+
 define i64 @csrrd_d() {
 ; CHECK-LABEL: csrrd_d:
 ; CHECK:       # %bb.0: # %entry
@@ -111,6 +175,15 @@ entry:
   ret i64 %0
 }
 
+define void @csrrd_d_noret() {
+; CHECK-LABEL: csrrd_d_noret:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call i64 @llvm.loongarch.csrrd.d(i32 1)
+  ret void
+}
+
 define i64 @csrwr_d(i64 %a) {
 ; CHECK-LABEL: csrwr_d:
 ; CHECK:       # %bb.0: # %entry
@@ -163,6 +236,15 @@ entry:
   ret i64 %0
 }
 
+define void @iocsrrd_d_noret(i32 %a) {
+; CHECK-LABEL: iocsrrd_d_noret:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call i64 @llvm.loongarch.iocsrrd.d(i32 %a)
+  ret void
+}
+
 define void @iocsrwr_d(i64 %a, i32 signext %b) {
 ; CHECK-LABEL: iocsrwr_d:
 ; CHECK:       # %bb.0: # %entry
@@ -203,6 +285,16 @@ entry:
   ret i64 %0
 }
 
+define void @lddir_d_noret(i64 %a) {
+; CHECK-LABEL: lddir_d_noret:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    lddir $a0, $a0, 1
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call i64 @llvm.loongarch.lddir.d(i64 %a, i64 1)
+  ret void
+}
+
 define void @ldpte_d(i64 %a) {
 ; CHECK-LABEL: ldpte_d:
 ; CHECK:       # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/LoongArch/intrinsic.ll b/llvm/test/CodeGen/LoongArch/intrinsic.ll
index 930e89b96d1b1..065bf7c78c17b 100644
--- a/llvm/test/CodeGen/LoongArch/intrinsic.ll
+++ b/llvm/test/CodeGen/LoongArch/intrinsic.ll
@@ -69,6 +69,17 @@ entry:
   ret i32 %res
 }
 
+;; TODO: Optimize out `movfcsr2gr` without data-dependency.
+define void @movfcsr2gr_noret() nounwind {
+; CHECK-LABEL: movfcsr2gr_noret:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    movfcsr2gr $a0, $fcsr1
+; CHECK-NEXT:    ret
+entry:
+  %res = call i32 @llvm.loongarch.movfcsr2gr(i32 1)
+  ret void
+}
+
 define void @syscall() nounwind {
 ; CHECK-LABEL: syscall:
 ; CHECK:       # %bb.0: # %entry
@@ -89,6 +100,15 @@ entry:
   ret i32 %0
 }
 
+define void @csrrd_w_noret() {
+; CHECK-LABEL: csrrd_w_noret:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call i32 @llvm.loongarch.csrrd.w(i32 1)
+  ret void
+}
+
 define i32 @csrwr_w(i32 signext %a) {
 ; CHECK-LABEL: csrwr_w:
 ; CHECK:       # %bb.0: # %entry
@@ -161,6 +181,33 @@ entry:
   ret i32 %0
 }
 
+define void @iocsrrd_b_noret(i32 %a) {
+; CHECK-LABEL: iocsrrd_b_noret:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call i32 @llvm.loongarch.iocsrrd.b(i32 %a)
+  ret void
+}
+
+define void @iocsrrd_h_noret(i32 %a) {
+; CHECK-LABEL: iocsrrd_h_noret:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call i32 @llvm.loongarch.iocsrrd.h(i32 %a)
+  ret void
+}
+
+define void @iocsrrd_w_noret(i32 %a) {
+; CHECK-LABEL: iocsrrd_w_noret:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call i32 @llvm.loongarch.iocsrrd.w(i32 %a)
+  ret void
+}
+
 define void @iocsrwr_b(i32 %a, i32 %b) {
 ; CHECK-LABEL: iocsrwr_b:
 ; CHECK:       # %bb.0: # %entry
@@ -200,3 +247,12 @@ entry:
   %0 = tail call i32 @llvm.loongarch.cpucfg(i32 %a)
   ret i32 %0
 }
+
+define void @cpucfg_noret(i32 %a) {
+; CHECK-LABEL: cpucfg_noret:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    ret
+entry:
+  %0 = tail call i32 @llvm.loongarch.cpucfg(i32 %a)
+  ret void
+}

From 5dbd5118ec5435269602de30ebd4982dd8b273dc Mon Sep 17 00:00:00 2001
From: Tue Ly <lntue@google.com>
Date: Thu, 15 Jun 2023 01:21:57 -0400
Subject: [PATCH 124/130] [libc][math] Improve tanhf performance.

Re-order exceptional branches and slightly adjust the evaluation.

Performance tested with the CORE-MATH project on AMD EPYC 7B12 (clocks/op)

Reciprocal throughputs:
```
--- BEFORE ---

$ CORE_MATH_PERF_MODE=rdtsc ./perf.sh tanhf
[####################] 100 %  (with -mavx2 -mfma)
Ntrial = 20 ; Min = 7.794 + 0.102 clc/call; Median-Min = 0.066 clc/call; Max = 8.267 clc/call;
[####################] 100 %. (with -msse4.2)
Ntrial = 20 ; Min = 10.783 + 0.172 clc/call; Median-Min = 0.144 clc/call; Max = 11.446 clc/call;
[####################] 100 %. (SSE2)
Ntrial = 20 ; Min = 18.926 + 0.381 clc/call; Median-Min = 0.342 clc/call; Max = 19.623 clc/call;

--- AFTER ---

$ CORE_MATH_PERF_MODE=rdtsc ./perf.sh tanhf
[####################] 100 %  (with -mavx2 -mfma)
Ntrial = 20 ; Min = 6.598 + 0.085 clc/call; Median-Min = 0.052 clc/call; Max = 6.868 clc/call;
[####################] 100 %  (with -msse4.2)
Ntrial = 20 ; Min = 9.245 + 0.304 clc/call; Median-Min = 0.248 clc/call; Max = 10.675 clc/call;
[####################] 100 %. (SSE2)
Ntrial = 20 ; Min = 11.724 + 0.440 clc/call; Median-Min = 0.444 clc/call; Max = 12.262 clc/call;
```

Latency:
```
--- BEFORE ---

$ PERF_ARGS="--latency" CORE_MATH_PERF_MODE=rdtsc ./perf.sh tanhf
[####################] 100 %  (with -mavx2 -mfma)
Ntrial = 20 ; Min = 38.821 + 0.157 clc/call; Median-Min = 0.122 clc/call; Max = 39.539 clc/call;
[####################] 100 %. (with -msse4.2)
Ntrial = 20 ; Min = 44.767 + 0.766 clc/call; Median-Min = 0.681 clc/call; Max = 45.951 clc/call;
[####################] 100 %. (SSE2)
Ntrial = 20 ; Min = 55.055 + 1.512 clc/call; Median-Min = 1.571 clc/call; Max = 57.039 clc/call;

--- AFTER ---

$ PERF_ARGS="--latency" CORE_MATH_PERF_MODE=rdtsc ./perf.sh tanhf
[####################] 100 %  (with -mavx2 -mfma)
Ntrial = 20 ; Min = 36.147 + 0.194 clc/call; Median-Min = 0.181 clc/call; Max = 36.536 clc/call;
[####################] 100 %  (with -msse4.2)
Ntrial = 20 ; Min = 40.904 + 0.728 clc/call; Median-Min = 0.557 clc/call; Max = 42.231 clc/call;
[####################] 100 %. (SSE2)
Ntrial = 20 ; Min = 55.776 + 0.557 clc/call; Median-Min = 0.542 clc/call; Max = 56.551 clc/call;
```

Reviewed By: michaelrj

Differential Revision: https://reviews.llvm.org/D153026
---
 .../macros/properties/CMakeLists.txt          |  14 +-
 .../macros/properties/cpu_features.h          |   8 ++
 libc/src/math/generic/CMakeLists.txt          |   2 +
 libc/src/math/generic/tanhf.cpp               | 134 ++++++++++++------
 libc/test/src/math/tanhf_test.cpp             |  40 +++---
 .../llvm-project-overlay/libc/BUILD.bazel     |   5 +-
 6 files changed, 131 insertions(+), 72 deletions(-)

diff --git a/libc/src/__support/macros/properties/CMakeLists.txt b/libc/src/__support/macros/properties/CMakeLists.txt
index 8ddd16a1e3669..f6c88c34ebb5a 100644
--- a/libc/src/__support/macros/properties/CMakeLists.txt
+++ b/libc/src/__support/macros/properties/CMakeLists.txt
@@ -1,17 +1,19 @@
 add_header_library(
-    architectures
-    HDRS
+  architectures
+  HDRS
     architectures.h
 )
 
 add_header_library(
-    compiler
-    HDRS
+  compiler
+  HDRS
     compiler.h
 )
 
 add_header_library(
-    cpu_features
-    HDRS
+  cpu_features
+  HDRS
     cpu_features.h
+  DEPENDS
+    .architectures
 )
diff --git a/libc/src/__support/macros/properties/cpu_features.h b/libc/src/__support/macros/properties/cpu_features.h
index 086b216e27573..493d9f446d374 100644
--- a/libc/src/__support/macros/properties/cpu_features.h
+++ b/libc/src/__support/macros/properties/cpu_features.h
@@ -12,6 +12,8 @@
 #ifndef LLVM_LIBC_SRC_SUPPORT_MACROS_PROPERTIES_CPU_FEATURES_H
 #define LLVM_LIBC_SRC_SUPPORT_MACROS_PROPERTIES_CPU_FEATURES_H
 
+#include "architectures.h"
+
 #if defined(__SSE2__)
 #define LIBC_TARGET_CPU_HAS_SSE2
 #endif
@@ -41,4 +43,10 @@
 #define LIBC_TARGET_CPU_HAS_FMA
 #endif
 
+#if defined(LIBC_TARGET_ARCH_IS_AARCH64) ||                                    \
+    (defined(LIBC_TARGET_ARCH_IS_X86_64) &&                                    \
+     defined(LIBC_TARGET_CPU_HAS_SSE4_2))
+#define LIBC_TARGET_CPU_HAS_NEAREST_INT
+#endif
+
 #endif // LLVM_LIBC_SRC_SUPPORT_MACROS_PROPERTIES_CPU_FEATURES_H
diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt
index 3d5273fa90cc1..098b69817fb20 100644
--- a/libc/src/math/generic/CMakeLists.txt
+++ b/libc/src/math/generic/CMakeLists.txt
@@ -1374,6 +1374,8 @@ add_entrypoint_object(
     .explogxf
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.rounding_mode
+    libc.src.__support.FPUtil.multiply_add
+    libc.src.__support.FPUtil.polyeval
     libc.src.__support.macros.optimization
   COMPILE_OPTIONS
     -O3
diff --git a/libc/src/math/generic/tanhf.cpp b/libc/src/math/generic/tanhf.cpp
index eb6b50a64b014..21c3ed8c8c333 100644
--- a/libc/src/math/generic/tanhf.cpp
+++ b/libc/src/math/generic/tanhf.cpp
@@ -8,66 +8,114 @@
 
 #include "src/math/tanhf.h"
 #include "src/__support/FPUtil/FPBits.h"
-#include "src/__support/FPUtil/rounding_mode.h"
-#include "src/__support/macros/optimization.h"            // LIBC_UNLIKELY
-#include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
+#include "src/__support/FPUtil/PolyEval.h"
+#include "src/__support/FPUtil/multiply_add.h"
+#include "src/__support/FPUtil/nearest_integer.h"
+#include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
+#include "src/__support/macros/properties/cpu_features.h"
 #include "src/math/generic/explogxf.h"
 
 namespace __llvm_libc {
 
+// 2^6 * log2(e)
+constexpr double LOG2_E_EXP2_6 = ExpBase::LOG2_B * 2.0;
+
 LLVM_LIBC_FUNCTION(float, tanhf, (float x)) {
   using FPBits = typename fputil::FPBits<float>;
   FPBits xbits(x);
-  bool sign = xbits.get_sign();
-  uint32_t x_abs = xbits.uintval() & FPBits::FloatProp::EXP_MANT_MASK;
+  uint32_t x_u = xbits.uintval();
+  uint32_t x_abs = x_u & FPBits::FloatProp::EXP_MANT_MASK;
 
-  // |x| <= 2^-26
-  if (LIBC_UNLIKELY(x_abs <= 0x3280'0000U)) {
-    return static_cast<float>(
-        LIBC_UNLIKELY(x_abs == 0) ? x : (x - 0x1.5555555555555p-2 * x * x * x));
-  }
+  // When |x| >= 15, or x is inf or nan, or |x| <= 0.078125
+  if (LIBC_UNLIKELY((x_abs >= 0x4170'0000U) || (x_abs <= 0x3da0'0000U))) {
+    if (x_abs <= 0x3da0'0000U) {
+      // |x| <= 0.078125
+      if (LIBC_UNLIKELY(x_abs <= 0x3280'0000U)) {
+        // |x| <= 2^-26
+        return (x_abs != 0)
+                   ? static_cast<float>(x - 0x1.5555555555555p-2 * x * x * x)
+                   : x;
+      }
+
+      const double TAYLOR[] = {-0x1.5555555555555p-2, 0x1.1111111111111p-3,
+                               -0x1.ba1ba1ba1ba1cp-5, 0x1.664f4882c10fap-6,
+                               -0x1.226e355e6c23dp-7};
+      double xdbl = x;
+      double x2 = xdbl * xdbl;
+      // Taylor polynomial.
+      double x4 = x2 * x2;
+      double c0 = x2 * TAYLOR[0];
+      double c1 = fputil::multiply_add(x2, TAYLOR[2], TAYLOR[1]);
+      double c2 = fputil::multiply_add(x2, TAYLOR[4], TAYLOR[3]);
+      double pe = fputil::polyeval(x4, c0, c1, c2);
 
-  // When |x| >= 15, or x is inf or nan
-  if (LIBC_UNLIKELY(x_abs >= 0x4170'0000U)) {
-    if (xbits.is_nan())
+      return static_cast<float>(fputil::multiply_add(xdbl, pe, xdbl));
+    }
+
+    // |x| >= 15
+    if (LIBC_UNLIKELY(xbits.is_nan()))
       return x + 1.0f; // sNaN to qNaN + signal
 
-    if (xbits.is_inf())
-      return sign ? -1.0f : 1.0f;
+    const double SIGNS[2][2] = {{1.0f, -0x1.0p-25f}, {-1.0f, 0x1.0p-25f}};
 
-    if (sign) {
-      return -1.0f + opt_barrier(FPBits(FPBits::MIN_NORMAL).get_val());
-    } else
-      return 1.0f - opt_barrier(FPBits(FPBits::MIN_NORMAL).get_val());
-  }
+    bool sign = xbits.get_sign();
+    int idx = static_cast<int>(sign);
 
-  // |x| <= 0.078125
-  if (LIBC_UNLIKELY(x_abs <= 0x3da0'0000U)) {
-    double xdbl = x;
-    double x2 = xdbl * xdbl;
-    // Pure Taylor series.
-    double pe = fputil::polyeval(x2, 0.0, -0x1.5555555555555p-2,
-                                 0x1.1111111111111p-3, -0x1.ba1ba1ba1ba1cp-5,
-                                 0x1.664f4882c10fap-6, -0x1.226e355e6c23dp-7);
-    return static_cast<float>(fputil::multiply_add(xdbl, pe, xdbl));
-  }
+    if (LIBC_UNLIKELY(xbits.is_inf()))
+      return SIGNS[idx][0];
 
-  if (LIBC_UNLIKELY(xbits.bits == 0x4058'e0a3U)) {
-    if (fputil::fenv_is_round_down())
-      return FPBits(0x3f7f'6ad9U).get_val();
+    return SIGNS[idx][0] + SIGNS[idx][1];
   }
 
-  // Range reduction: e^(2x) = 2^(mid + hi) * e^lo
-  auto ep = exp_b_range_reduc<ExpBase>(2.0f * x); // exp(2 * x)
-  double r = ExpBase::powb_lo(ep.lo);
-  // tanh(x) = (exp(2x) - 1) / (exp(2x) + 1)
-#if defined(LIBC_TARGET_CPU_HAS_FMA)
-  return static_cast<float>(fputil::multiply_add(ep.mh, r, -1.0) /
-                            fputil::multiply_add(ep.mh, r, 1.0));
+  // Range reduction: e^(2x) = 2^(hi + mid) * e^lo
+  // Let  k = round( x * 2^6 * log2(e)),
+  // So   k  = (hi + mid) * 2^5
+  // Then lo = 2x - (hi + mid) * log(2) = 2x - k * 2^-5 * log(2).
+
+  double xd = static_cast<double>(x);
+  // k = round( x* 2^6 * log2(e) )
+  double k;
+  // mk = -k
+  int mk;
+#ifdef LIBC_TARGET_CPU_HAS_NEAREST_INT
+  k = fputil::nearest_integer(xd * LOG2_E_EXP2_6);
+  mk = -static_cast<int>(k);
 #else
-  double exp_x = ep.mh * r;
-  return static_cast<float>((exp_x - 1.0) / (exp_x + 1.0));
-#endif // LIBC_TARGET_CPU_HAS_FMA
+  constexpr double HALF_WAY[2] = {-0.5, 0.5};
+
+  mk = static_cast<int>(
+      fputil::multiply_add(xd, -LOG2_E_EXP2_6, HALF_WAY[xbits.get_sign()]));
+  k = static_cast<double>(-mk);
+#endif // LIBC_TARGET_CPU_HAS_NEAREST_INT
+  // -hi = floor(-k * 2^(-MID_BITS))
+  // exp_mhi = shift -hi to the exponent field of double precision.
+  int64_t exp_mhi = static_cast<int64_t>(mk >> ExpBase::MID_BITS)
+                    << fputil::FloatProperties<double>::MANTISSA_WIDTH;
+  // mh = 2^(-hi - mid)
+  int64_t mh_bits = ExpBase::EXP_2_MID[mk & ExpBase::MID_MASK] + exp_mhi;
+  double mh = fputil::FPBits<double>(uint64_t(mh_bits)).get_val();
+  // dx = lo/2 = x - (hi + mid) * log(2)/2 = x - k * 2^-6 * log(2)
+  double dx = fputil::multiply_add(
+      k, ExpBase::M_LOGB_2_LO * 0.5,
+      fputil::multiply_add(k, ExpBase::M_LOGB_2_HI * 0.5, xd));
+
+  // > P = fpminimax(expm1(2*x)/x, 4, [|D...|], [-log(2)/128, log(2)/128]);
+  constexpr double COEFFS[] = {0x1.ffffffffe5bc8p0, 0x1.555555555cd67p0,
+                               0x1.5555c2a9b48b4p-1, 0x1.11112a0e34bdbp-2};
+
+  double dx2 = dx * dx;
+  double c0 = fputil::multiply_add(dx, 2.0, 1.0);
+  double c1 = fputil::multiply_add(dx, COEFFS[1], COEFFS[0]);
+  double c2 = fputil::multiply_add(dx, COEFFS[3], COEFFS[2]);
+  double r = fputil::polyeval(dx2, c0, c1, c2);
+
+  // tanh(x) = sinh(x) / cosh(x)
+  //         = (e^x - e^(-x)) / (e^x + e^(-x))
+  //         = (e^(2x) - 1) / (e^(2x) + 1)
+  //         = (2^(hi + mid) * e^lo - 1) / (2^(hi + mid) * e^lo + 1)
+  //         = (e^lo - 2^(-hi - mid)) / (e^lo + 2^(-hi - mid))
+  //         = (r - mh) / (r + mh)
+  return static_cast<float>((r - mh) / (r + mh));
 }
 
 } // namespace __llvm_libc
diff --git a/libc/test/src/math/tanhf_test.cpp b/libc/test/src/math/tanhf_test.cpp
index f4a4b72dd162a..07afe14a263d0 100644
--- a/libc/test/src/math/tanhf_test.cpp
+++ b/libc/test/src/math/tanhf_test.cpp
@@ -43,35 +43,31 @@ TEST(LlvmLibcTanhfTest, SpecialNumbers) {
 }
 
 TEST(LlvmLibcTanhfTest, InFloatRange) {
-  constexpr uint32_t COUNT = 100'000;
+  constexpr uint32_t COUNT = 100'001;
   constexpr uint32_t STEP = UINT32_MAX / COUNT;
   for (uint32_t i = 0, v = 0; i <= COUNT; ++i, v += STEP) {
     float x = float(FPBits(v));
     if (isnan(x) || isinf(x))
       continue;
-    ASSERT_MPFR_MATCH(mpfr::Operation::Tanh, x, __llvm_libc::tanhf(x), 0.5);
+    ASSERT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Tanh, x,
+                                   __llvm_libc::tanhf(x), 0.5);
   }
 }
 
-// For small values, tanh(x) is x.
-TEST(LlvmLibcTanhfTest, SmallValues) {
-  float x = float(FPBits(uint32_t(0x17800000)));
-  float result = __llvm_libc::tanhf(x);
-  EXPECT_MPFR_MATCH(mpfr::Operation::Tanh, x, result, 0.5);
-  EXPECT_FP_EQ(x, result);
-
-  x = float(FPBits(uint32_t(0x00400000)));
-  result = __llvm_libc::tanhf(x);
-  EXPECT_MPFR_MATCH(mpfr::Operation::Tanh, x, result, 0.5);
-  EXPECT_FP_EQ(x, result);
-}
-
 TEST(LlvmLibcTanhfTest, ExceptionalValues) {
-  float x = float(FPBits(uint32_t(0x3a12'85ffU)));
-  EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Tanh, x,
-                                 __llvm_libc::tanhf(x), 0.5);
-
-  x = -float(FPBits(uint32_t(0x3a12'85ffU)));
-  EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Tanh, x,
-                                 __llvm_libc::tanhf(x), 0.5);
+  constexpr int N = 4;
+  constexpr uint32_t INPUTS[N] = {
+      0x0040'0000,
+      0x1780'0000,
+      0x3a12'85ff,
+      0x4058'e0a3,
+  };
+
+  for (int i = 0; i < N; ++i) {
+    float x = float(FPBits(INPUTS[i]));
+    EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Tanh, x,
+                                   __llvm_libc::tanhf(x), 0.5);
+    EXPECT_MPFR_MATCH_ALL_ROUNDING(mpfr::Operation::Tanh, -x,
+                                   __llvm_libc::tanhf(-x), 0.5);
+  }
 }
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 58aa99d007743..eb7a3c87fc0f7 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -81,7 +81,10 @@ libc_support_library(
 libc_support_library(
     name = "__support_macros_properties_cpu_features",
     hdrs = ["src/__support/macros/properties/cpu_features.h"],
-    deps = [":libc_root"],
+    deps = [
+        "__support_macros_properties_architectures",
+        ":libc_root",
+    ],
 )
 
 libc_support_library(

From 0ae409c0d70d6251777351296599b60c81c54d2d Mon Sep 17 00:00:00 2001
From: Tue Ly <lntue@google.com>
Date: Thu, 15 Jun 2023 14:56:08 -0400
Subject: [PATCH 125/130] [libc][math] Slightly improve sinhf and coshf
 performance.

Re-order exceptional branches and slightly adjust the evaluation.
Depends on https://reviews.llvm.org/D153026 .

Reviewed By: michaelrj

Differential Revision: https://reviews.llvm.org/D153062
---
 libc/src/math/generic/coshf.cpp  | 12 +++----
 libc/src/math/generic/explogxf.h | 36 +++++++++++++-------
 libc/src/math/generic/sinhf.cpp  | 56 ++++++++++++++++----------------
 3 files changed, 58 insertions(+), 46 deletions(-)

diff --git a/libc/src/math/generic/coshf.cpp b/libc/src/math/generic/coshf.cpp
index 8b30d51c69dce..67d2667711de9 100644
--- a/libc/src/math/generic/coshf.cpp
+++ b/libc/src/math/generic/coshf.cpp
@@ -23,13 +23,13 @@ LLVM_LIBC_FUNCTION(float, coshf, (float x)) {
 
   uint32_t x_u = xbits.uintval();
 
-  // |x| <= 2^-26
-  if (LIBC_UNLIKELY(x_u <= 0x3280'0000U)) {
-    return 1.0f + x;
-  }
-
   // When |x| >= 90, or x is inf or nan
-  if (LIBC_UNLIKELY(x_u >= 0x42b4'0000U)) {
+  if (LIBC_UNLIKELY(x_u >= 0x42b4'0000U || x_u <= 0x3280'0000U)) {
+    // |x| <= 2^-26
+    if (x_u <= 0x3280'0000U) {
+      return 1.0f + x;
+    }
+
     if (xbits.is_inf_or_nan())
       return x + FPBits::inf().get_val();
 
diff --git a/libc/src/math/generic/explogxf.h b/libc/src/math/generic/explogxf.h
index cb8efc92f28c7..827762ca48aeb 100644
--- a/libc/src/math/generic/explogxf.h
+++ b/libc/src/math/generic/explogxf.h
@@ -16,6 +16,7 @@
 #include "src/__support/FPUtil/PolyEval.h"
 #include "src/__support/FPUtil/nearest_integer.h"
 #include "src/__support/common.h"
+#include "src/__support/macros/properties/cpu_features.h"
 
 #include <errno.h>
 
@@ -210,13 +211,24 @@ template <class Base> LIBC_INLINE exp_b_reduc_t exp_b_range_reduc(float x) {
 template <bool is_sinh> LIBC_INLINE double exp_pm_eval(float x) {
   double xd = static_cast<double>(x);
 
-  // round(x * log2(e) * 2^5)
-  double kd = fputil::nearest_integer(ExpBase::LOG2_B * xd);
-
+  // kd = round(x * log2(e) * 2^5)
   // k_p = round(x * log2(e) * 2^5)
-  int k_p = static_cast<int>(kd);
   // k_m = round(-x * log2(e) * 2^5)
-  int k_m = -k_p;
+  double kd;
+  int k_p, k_m;
+
+#ifdef LIBC_TARGET_CPU_HAS_NEAREST_INT
+  kd = fputil::nearest_integer(ExpBase::LOG2_B * xd);
+  k_p = static_cast<int>(kd);
+  k_m = -k_p;
+#else
+  constexpr double HALF_WAY[2] = {0.5, -0.5};
+
+  k_p = static_cast<int>(
+      fputil::multiply_add(xd, ExpBase::LOG2_B, HALF_WAY[x < 0.0f]));
+  k_m = -k_p;
+  kd = static_cast<double>(k_p);
+#endif // LIBC_TARGET_CPU_HAS_NEAREST_INT
 
   // hi = floor(kf * 2^(-5))
   // exp_hi = shift hi to the exponent field of double precision.
@@ -243,19 +255,19 @@ template <bool is_sinh> LIBC_INLINE double exp_pm_eval(float x) {
   double dx2 = dx * dx;
 
   // c0 = 1 + COEFFS[0] * lo^2
-  // P_even = 1 + COEFFS[0] * lo^2 + COEFFS[2] * lo^4
-  double p_even =
-      fputil::polyeval(dx2, 1.0, ExpBase::COEFFS[0], ExpBase::COEFFS[2]);
-  // P_odd = 1 + COEFFS[1] * lo^2 + COEFFS[3] * lo^4
-  double p_odd =
-      fputil::polyeval(dx2, 1.0, ExpBase::COEFFS[1], ExpBase::COEFFS[3]);
+  // P_even = (1 + COEFFS[0] * lo^2 + COEFFS[2] * lo^4) / 2
+  double p_even = fputil::polyeval(dx2, 0.5, ExpBase::COEFFS[0] * 0.5,
+                                   ExpBase::COEFFS[2] * 0.5);
+  // P_odd = (1 + COEFFS[1] * lo^2 + COEFFS[3] * lo^4) / 2
+  double p_odd = fputil::polyeval(dx2, 0.5, ExpBase::COEFFS[1] * 0.5,
+                                  ExpBase::COEFFS[3] * 0.5);
 
   double r;
   if constexpr (is_sinh)
     r = fputil::multiply_add(dx * mh_sum, p_odd, p_even * mh_diff);
   else
     r = fputil::multiply_add(dx * mh_diff, p_odd, p_even * mh_sum);
-  return 0.5 * r;
+  return r;
 }
 
 // x should be positive, normal finite value
diff --git a/libc/src/math/generic/sinhf.cpp b/libc/src/math/generic/sinhf.cpp
index 7f4d0d6e3af2e..3ebfe6ba07009 100644
--- a/libc/src/math/generic/sinhf.cpp
+++ b/libc/src/math/generic/sinhf.cpp
@@ -17,23 +17,43 @@ namespace __llvm_libc {
 LLVM_LIBC_FUNCTION(float, sinhf, (float x)) {
   using FPBits = typename fputil::FPBits<float>;
   FPBits xbits(x);
-  bool sign = xbits.get_sign();
   uint32_t x_abs = xbits.uintval() & FPBits::FloatProp::EXP_MANT_MASK;
 
-  // |x| <= 2^-26
-  if (LIBC_UNLIKELY(x_abs <= 0x3280'0000U)) {
-    return static_cast<float>(
-        LIBC_UNLIKELY(x_abs == 0) ? x : (x + 0.25 * x * x * x));
-  }
-
   // When |x| >= 90, or x is inf or nan
-  if (LIBC_UNLIKELY(x_abs >= 0x42b4'0000U)) {
+  if (LIBC_UNLIKELY(x_abs >= 0x42b4'0000U || x_abs <= 0x3da0'0000U)) {
+    // |x| <= 0.078125
+    if (x_abs <= 0x3da0'0000U) {
+      // |x| = 0.0005589424981735646724700927734375
+      if (LIBC_UNLIKELY(x_abs == 0x3a12'85ffU)) {
+        if (fputil::fenv_is_round_to_nearest())
+          return x;
+      }
+
+      // |x| <= 2^-26
+      if (LIBC_UNLIKELY(x_abs <= 0x3280'0000U)) {
+        return static_cast<float>(
+            LIBC_UNLIKELY(x_abs == 0) ? x : (x + 0.25 * x * x * x));
+      }
+
+      double xdbl = x;
+      double x2 = xdbl * xdbl;
+      // Sollya: fpminimax(sinh(x),[|3,5,7|],[|D...|],[-1/16-1/64;1/16+1/64],x);
+      // Sollya output: x * (0x1p0 + x^0x1p1 * (0x1.5555555556583p-3 + x^0x1p1
+      //                  * (0x1.111110d239f1fp-7
+      //                  + x^0x1p1 * 0x1.a02b5a284013cp-13)))
+      // Therefore, output of Sollya = x * pe;
+      double pe = fputil::polyeval(x2, 0.0, 0x1.5555555556583p-3,
+                                   0x1.111110d239f1fp-7, 0x1.a02b5a284013cp-13);
+      return static_cast<float>(fputil::multiply_add(xdbl, pe, xdbl));
+    }
+
     if (xbits.is_nan())
       return x + 1.0f; // sNaN to qNaN + signal
 
     if (xbits.is_inf())
       return x;
 
+    bool sign = xbits.get_sign();
     int rounding = fputil::quick_get_round();
     if (sign) {
       if (LIBC_UNLIKELY(rounding == FE_UPWARD || rounding == FE_TOWARDZERO))
@@ -50,26 +70,6 @@ LLVM_LIBC_FUNCTION(float, sinhf, (float x)) {
     return x + FPBits::inf(sign).get_val();
   }
 
-  // |x| <= 0.078125
-  if (LIBC_UNLIKELY(x_abs <= 0x3da0'0000U)) {
-    // |x| = 0.0005589424981735646724700927734375
-    if (LIBC_UNLIKELY(x_abs == 0x3a12'85ffU)) {
-      if (fputil::fenv_is_round_to_nearest())
-        return x;
-    }
-
-    double xdbl = x;
-    double x2 = xdbl * xdbl;
-    // Sollya: fpminimax(sinh(x),[|3,5,7|],[|D...|],[-1/16-1/64;1/16+1/64],x);
-    // Sollya output: x * (0x1p0 + x^0x1p1 * (0x1.5555555556583p-3 + x^0x1p1
-    //                  * (0x1.111110d239f1fp-7
-    //                  + x^0x1p1 * 0x1.a02b5a284013cp-13)))
-    // Therefore, output of Sollya = x * pe;
-    double pe = fputil::polyeval(x2, 0.0, 0x1.5555555556583p-3,
-                                 0x1.111110d239f1fp-7, 0x1.a02b5a284013cp-13);
-    return static_cast<float>(fputil::multiply_add(xdbl, pe, xdbl));
-  }
-
   // sinh(x) = (e^x - e^(-x)) / 2.
   return static_cast<float>(exp_pm_eval</*is_sinh*/ true>(x));
 }

From 46aa659a32e78481d90950eda9d464fc22eaad31 Mon Sep 17 00:00:00 2001
From: Tue Ly <lntue@google.com>
Date: Fri, 16 Jun 2023 09:22:00 -0400
Subject: [PATCH 126/130] [libc][math] Improve exp2f performance.

Re-organize special cases and add a special case when `|x| < 2^-5`.

Reviewed By: michaelrj

Differential Revision: https://reviews.llvm.org/D153134
---
 libc/src/math/generic/CMakeLists.txt          |  1 +
 libc/src/math/generic/exp2f.cpp               | 63 +++++++++++++------
 .../llvm-project-overlay/libc/BUILD.bazel     |  1 +
 3 files changed, 46 insertions(+), 19 deletions(-)

diff --git a/libc/src/math/generic/CMakeLists.txt b/libc/src/math/generic/CMakeLists.txt
index 098b69817fb20..7772c6d826905 100644
--- a/libc/src/math/generic/CMakeLists.txt
+++ b/libc/src/math/generic/CMakeLists.txt
@@ -561,6 +561,7 @@ add_entrypoint_object(
     ../exp2f.h
   DEPENDS
     .explogxf
+    libc.src.__support.FPUtil.except_value_utils
     libc.src.__support.FPUtil.fenv_impl
     libc.src.__support.FPUtil.fp_bits
     libc.src.__support.FPUtil.multiply_add
diff --git a/libc/src/math/generic/exp2f.cpp b/libc/src/math/generic/exp2f.cpp
index f2c005d5e00a0..b967a4bb45371 100644
--- a/libc/src/math/generic/exp2f.cpp
+++ b/libc/src/math/generic/exp2f.cpp
@@ -10,11 +10,13 @@
 #include "src/__support/FPUtil/FEnvImpl.h"
 #include "src/__support/FPUtil/FPBits.h"
 #include "src/__support/FPUtil/PolyEval.h"
+#include "src/__support/FPUtil/except_value_utils.h"
 #include "src/__support/FPUtil/multiply_add.h"
 #include "src/__support/FPUtil/nearest_integer.h"
 #include "src/__support/FPUtil/rounding_mode.h"
 #include "src/__support/common.h"
 #include "src/__support/macros/optimization.h" // LIBC_UNLIKELY
+#include "src/__support/macros/properties/cpu_features.h"
 
 #include <errno.h>
 
@@ -33,13 +35,38 @@ LLVM_LIBC_FUNCTION(float, exp2f, (float x)) {
   uint32_t x_u = xbits.uintval();
   uint32_t x_abs = x_u & 0x7fff'ffffU;
 
-  // |x| < 2^-25
-  if (LIBC_UNLIKELY(x_abs <= 0x3280'0000U)) {
-    return 1.0f + x;
-  }
+  // When |x| >= 128, or x is nan, or |x| <= 2^-5
+  if (LIBC_UNLIKELY(x_abs >= 0x4300'0000U || x_abs <= 0x3d00'0000U)) {
+    // |x| <= 2^-5
+    if (x_abs <= 0x3d00'0000) {
+      // |x| < 2^-25
+      if (LIBC_UNLIKELY(x_abs <= 0x3280'0000U)) {
+        return 1.0f + x;
+      }
 
-  // // When |x| >= 128, or x is nan
-  if (LIBC_UNLIKELY(x_abs >= 0x4300'0000U)) {
+      // Check exceptional values.
+      if (LIBC_UNLIKELY((x_u & EXVAL_MASK) == EXVAL_MASK)) {
+        if (LIBC_UNLIKELY(x_u == EXVAL1)) { // x = 0x1.853a6ep-9f
+          return fputil::round_result_slightly_down(0x1.00870ap+0f);
+        } else if (LIBC_UNLIKELY(x_u == EXVAL2)) { // x = -0x1.e7526ep-6f
+          return fputil::round_result_slightly_down(0x1.f58d62p-1f);
+        }
+      }
+
+      // Minimax polynomial generated by Sollya with:
+      // > P = fpminimax((2^x - 1)/x, 5, [|D...|], [-2^-5, 2^-5]);
+      constexpr double COEFFS[] = {
+          0x1.62e42fefa39f3p-1, 0x1.ebfbdff82c57bp-3,  0x1.c6b08d6f2d7aap-5,
+          0x1.3b2ab6fc92f5dp-7, 0x1.5d897cfe27125p-10, 0x1.43090e61e6af1p-13};
+      double xd = static_cast<double>(x);
+      double xsq = xd * xd;
+      double c0 = fputil::multiply_add(xd, COEFFS[1], COEFFS[0]);
+      double c1 = fputil::multiply_add(xd, COEFFS[3], COEFFS[2]);
+      double c2 = fputil::multiply_add(xd, COEFFS[5], COEFFS[4]);
+      double p = fputil::polyeval(xsq, c0, c1, c2);
+      double r = fputil::multiply_add(p, xd, 1.0);
+      return r;
+    }
 
     // x >= 128
     if (!xbits.get_sign()) {
@@ -73,17 +100,6 @@ LLVM_LIBC_FUNCTION(float, exp2f, (float x)) {
     }
   }
 
-  // Check exceptional values.
-  if (LIBC_UNLIKELY((x_u & EXVAL_MASK) == EXVAL_MASK)) {
-    if (LIBC_UNLIKELY(x_u == EXVAL1)) { // x = 0x1.853a6ep-9f
-      if (fputil::fenv_is_round_to_nearest())
-        return 0x1.00870ap+0f;
-    } else if (LIBC_UNLIKELY(x_u == EXVAL2)) { // x = -0x1.e7526ep-6f
-      if (fputil::fenv_is_round_to_nearest())
-        return 0x1.f58d62p-1f;
-    }
-  }
-
   // For -150 < x < 128, to compute 2^x, we perform the following range
   // reduction: find hi, mid, lo such that:
   //   x = hi + mid + lo, in which
@@ -101,11 +117,20 @@ LLVM_LIBC_FUNCTION(float, exp2f, (float x)) {
   // of 2^mid.
 
   // kf = (hi + mid) * 2^5 = round(x * 2^5)
-  float kf = fputil::nearest_integer(x * 32.0f);
+  float kf;
+  int k;
+#ifdef LIBC_TARGET_CPU_HAS_NEAREST_INT
+  kf = fputil::nearest_integer(x * 32.0f);
+  k = static_cast<int>(kf);
+#else
+  constexpr float HALF[2] = {0.5f, -0.5f};
+  k = static_cast<int>(fputil::multiply_add(x, 32.0f, HALF[x < 0.0f]));
+  kf = static_cast<float>(k);
+#endif // LIBC_TARGET_CPU_HAS_NEAREST_INT
+
   // dx = lo = x - (hi + mid) = x - kf * 2^(-5)
   double dx = fputil::multiply_add(-0x1.0p-5f, kf, x);
 
-  int k = static_cast<int>(kf);
   // hi = floor(kf * 2^(-4))
   // exp_hi = shift hi to the exponent field of double precision.
   int64_t exp_hi =
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index eb7a3c87fc0f7..2d9d352ce5204 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -1200,6 +1200,7 @@ libc_math_function(
         ":__support_fputil_polyeval",
         ":__support_fputil_rounding_mode",
         ":__support_macros_optimization",
+        ":__support_macros_properties_cpu_features",
         ":explogxf",
     ],
 )

From 1f1385d93204b334d1d8ca13792916ca726e17eb Mon Sep 17 00:00:00 2001
From: Weining Lu <luweining@loongson.cn>
Date: Tue, 20 Jun 2023 21:49:28 +0800
Subject: [PATCH 127/130] [LoongArch] Indent LoongArchInstrInfo.td a little
 bit. NFC

---
 .../Target/LoongArch/LoongArchInstrInfo.td    | 25 ++++++++++---------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
index 3fbcc3e05a996..9129193e71107 100644
--- a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
+++ b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
@@ -82,14 +82,14 @@ def loongarch_crc_w_w_w
     : SDNode<"LoongArchISD::CRC_W_W_W", SDT_LoongArchIntBinOpW, [SDNPHasChain]>;
 def loongarch_crc_w_d_w
     : SDNode<"LoongArchISD::CRC_W_D_W", SDT_LoongArchIntBinOpW, [SDNPHasChain]>;
-def loongarch_crcc_w_b_w
-    : SDNode<"LoongArchISD::CRCC_W_B_W", SDT_LoongArchIntBinOpW, [SDNPHasChain]>;
-def loongarch_crcc_w_h_w
-    : SDNode<"LoongArchISD::CRCC_W_H_W", SDT_LoongArchIntBinOpW, [SDNPHasChain]>;
-def loongarch_crcc_w_w_w
-    : SDNode<"LoongArchISD::CRCC_W_W_W", SDT_LoongArchIntBinOpW, [SDNPHasChain]>;
-def loongarch_crcc_w_d_w
-    : SDNode<"LoongArchISD::CRCC_W_D_W", SDT_LoongArchIntBinOpW, [SDNPHasChain]>;
+def loongarch_crcc_w_b_w : SDNode<"LoongArchISD::CRCC_W_B_W",
+                                  SDT_LoongArchIntBinOpW, [SDNPHasChain]>;
+def loongarch_crcc_w_h_w : SDNode<"LoongArchISD::CRCC_W_H_W",
+                                  SDT_LoongArchIntBinOpW, [SDNPHasChain]>;
+def loongarch_crcc_w_w_w : SDNode<"LoongArchISD::CRCC_W_W_W",
+                                  SDT_LoongArchIntBinOpW, [SDNPHasChain]>;
+def loongarch_crcc_w_d_w : SDNode<"LoongArchISD::CRCC_W_D_W",
+                                  SDT_LoongArchIntBinOpW, [SDNPHasChain]>;
 def loongarch_bstrins
     : SDNode<"LoongArchISD::BSTRINS", SDT_LoongArchBStrIns>;
 def loongarch_bstrpick
@@ -106,10 +106,11 @@ def loongarch_ibar : SDNode<"LoongArchISD::IBAR", SDT_LoongArchVI,
                              [SDNPHasChain, SDNPSideEffect]>;
 def loongarch_break : SDNode<"LoongArchISD::BREAK", SDT_LoongArchVI,
                               [SDNPHasChain, SDNPSideEffect]>;
-def loongarch_movfcsr2gr : SDNode<"LoongArchISD::MOVFCSR2GR", SDT_LoongArchMovfcsr2gr,
-                                  [SDNPHasChain]>;
-def loongarch_movgr2fcsr : SDNode<"LoongArchISD::MOVGR2FCSR", SDT_LoongArchMovgr2fcsr,
-                                   [SDNPHasChain, SDNPSideEffect]>;
+def loongarch_movfcsr2gr : SDNode<"LoongArchISD::MOVFCSR2GR",
+                                  SDT_LoongArchMovfcsr2gr, [SDNPHasChain]>;
+def loongarch_movgr2fcsr : SDNode<"LoongArchISD::MOVGR2FCSR",
+                                  SDT_LoongArchMovgr2fcsr,
+                                  [SDNPHasChain, SDNPSideEffect]>;
 def loongarch_syscall : SDNode<"LoongArchISD::SYSCALL", SDT_LoongArchVI,
                                 [SDNPHasChain, SDNPSideEffect]>;
 def loongarch_csrrd : SDNode<"LoongArchISD::CSRRD", SDT_LoongArchCsrrd,

From 699addeff03515928b5e655b45199cda172a7893 Mon Sep 17 00:00:00 2001
From: Pravin Jagtap <Pravin.Jagtap@amd.com>
Date: Tue, 20 Jun 2023 09:52:58 -0400
Subject: [PATCH 128/130] [AMDGPU] Use verify<domtree> instead of intra-pass
 asserts.

Verifying dominator tree is expensive using intra-pass
asserts. Asserts added during D147408 are
increasing the build time of libc significantly. This change
does the verification after the atomic optimizer pass
and should fix the regression reported in D153232.

Reviewed By: arsenm, #amdgpu

Differential Revision: https://reviews.llvm.org/D153261
---
 llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp          | 4 ----
 llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan.ll | 2 +-
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
index 04a9b651d9a81..21696ace0aab7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp
@@ -770,9 +770,6 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I,
   Instruction *const SingleLaneTerminator =
       SplitBlockAndInsertIfThen(Cond, &I, false, nullptr, &DTU, nullptr);
 
-  // Control flow is changed here after splitting I's block
-  assert(DTU.getDomTree().verify(DominatorTree::VerificationLevel::Full));
-
   // At this point, we have split the I's block to allow one lane in wavefront
   // to update the precomputed reduced value. Also, completed the codegen for
   // new control flow i.e. iterative loop which perform reduction and scan using
@@ -799,7 +796,6 @@ void AMDGPUAtomicOptimizerImpl::optimizeAtomic(Instruction &I,
         {{DominatorTree::Insert, EntryBB, ComputeLoop},
          {DominatorTree::Insert, ComputeLoop, ComputeEnd},
          {DominatorTree::Delete, EntryBB, SingleLaneTerminator->getParent()}});
-    assert(DTU.getDomTree().verify(DominatorTree::VerificationLevel::Full));
 
     Predecessor = ComputeEnd;
   } else {
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan.ll
index 229c3aaff6f5a..f5ba59b616d71 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_iterative_scan.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN:  opt -S -mtriple=amdgcn-- -amdgpu-atomic-optimizer-strategy=Iterative -passes=amdgpu-atomic-optimizer %s | FileCheck -check-prefix=IR %s
+; RUN:  opt -S -mtriple=amdgcn-- -amdgpu-atomic-optimizer-strategy=Iterative -passes='amdgpu-atomic-optimizer,verify<domtree>' %s | FileCheck -check-prefix=IR %s
 
 define amdgpu_kernel void @uniform_value(ptr addrspace(1) , ptr addrspace(1) %val) #0 {
 ; IR-LABEL: @uniform_value(

From 3dd319ecf3be64598ea84d1730033854cade7123 Mon Sep 17 00:00:00 2001
From: Weining Lu <luweining@loongson.cn>
Date: Tue, 20 Jun 2023 21:50:54 +0800
Subject: [PATCH 129/130] [LoongArch] Optimize conditional selection of integer

This patch optimizes code generation by leveraging the zeroing behavior of the `maskeqz`/`masknez` instructions.

```
int sel(int a, int b)
{
    return (a < b) ? a : 0;
}
```

```
slt	$a1,$a0,$a1
masknez	$a2,$r0,$a1
maskeqz	$a0,$a0,$a1
or	$a0,$a0,$a2
```

=>

```
slt	$a1,$a0,$a1
maskeqz	$a0,$a0,$a1
```

Reviewed By: SixWeining

Differential Revision: https://reviews.llvm.org/D153193
---
 .../Target/LoongArch/LoongArchInstrInfo.td    |  2 ++
 .../LoongArch/atomicrmw-uinc-udec-wrap.ll     |  8 -----
 .../ir-instruction/select-bare-int.ll         | 32 +++++++++++++++++++
 3 files changed, 34 insertions(+), 8 deletions(-)

diff --git a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
index 9129193e71107..2bdc291abdbf6 100644
--- a/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
+++ b/llvm/lib/Target/LoongArch/LoongArchInstrInfo.td
@@ -1201,6 +1201,8 @@ def : Pat<(setle GPR:$rj, GPR:$rk), (XORI (SLT GPR:$rk, GPR:$rj), 1)>;
 
 /// Select
 
+def : Pat<(select GPR:$cond, GPR:$t, 0), (MASKEQZ GPR:$t, GPR:$cond)>;
+def : Pat<(select GPR:$cond, 0, GPR:$f), (MASKNEZ GPR:$f, GPR:$cond)>;
 def : Pat<(select GPR:$cond, GPR:$t, GPR:$f),
           (OR (MASKEQZ GPR:$t, GPR:$cond), (MASKNEZ GPR:$f, GPR:$cond))>;
 
diff --git a/llvm/test/CodeGen/LoongArch/atomicrmw-uinc-udec-wrap.ll b/llvm/test/CodeGen/LoongArch/atomicrmw-uinc-udec-wrap.ll
index 5b166046008e6..67823343d1c80 100644
--- a/llvm/test/CodeGen/LoongArch/atomicrmw-uinc-udec-wrap.ll
+++ b/llvm/test/CodeGen/LoongArch/atomicrmw-uinc-udec-wrap.ll
@@ -22,8 +22,6 @@ define i8 @atomicrmw_uinc_wrap_i8(ptr %ptr, i8 %val) {
 ; LA64-NEXT:    addi.d $a5, $a5, 1
 ; LA64-NEXT:    xori $a6, $a6, 1
 ; LA64-NEXT:    masknez $a5, $a5, $a6
-; LA64-NEXT:    maskeqz $a6, $zero, $a6
-; LA64-NEXT:    or $a5, $a6, $a5
 ; LA64-NEXT:    andi $a5, $a5, 255
 ; LA64-NEXT:    sll.w $a5, $a5, $a0
 ; LA64-NEXT:    and $a6, $a3, $a4
@@ -77,8 +75,6 @@ define i16 @atomicrmw_uinc_wrap_i16(ptr %ptr, i16 %val) {
 ; LA64-NEXT:    addi.d $a5, $a5, 1
 ; LA64-NEXT:    xori $a6, $a6, 1
 ; LA64-NEXT:    masknez $a5, $a5, $a6
-; LA64-NEXT:    maskeqz $a6, $zero, $a6
-; LA64-NEXT:    or $a5, $a6, $a5
 ; LA64-NEXT:    bstrpick.d $a5, $a5, 15, 0
 ; LA64-NEXT:    sll.w $a5, $a5, $a0
 ; LA64-NEXT:    and $a6, $a3, $a4
@@ -123,8 +119,6 @@ define i32 @atomicrmw_uinc_wrap_i32(ptr %ptr, i32 %val) {
 ; LA64-NEXT:    xori $a1, $a1, 1
 ; LA64-NEXT:    addi.d $a4, $a3, 1
 ; LA64-NEXT:    masknez $a4, $a4, $a1
-; LA64-NEXT:    maskeqz $a1, $zero, $a1
-; LA64-NEXT:    or $a4, $a1, $a4
 ; LA64-NEXT:  .LBB2_3: # %atomicrmw.start
 ; LA64-NEXT:    # Parent Loop BB2_1 Depth=1
 ; LA64-NEXT:    # => This Inner Loop Header: Depth=2
@@ -164,8 +158,6 @@ define i64 @atomicrmw_uinc_wrap_i64(ptr %ptr, i64 %val) {
 ; LA64-NEXT:    xori $a2, $a2, 1
 ; LA64-NEXT:    addi.d $a4, $a3, 1
 ; LA64-NEXT:    masknez $a4, $a4, $a2
-; LA64-NEXT:    maskeqz $a2, $zero, $a2
-; LA64-NEXT:    or $a4, $a2, $a4
 ; LA64-NEXT:  .LBB3_3: # %atomicrmw.start
 ; LA64-NEXT:    # Parent Loop BB3_1 Depth=1
 ; LA64-NEXT:    # => This Inner Loop Header: Depth=2
diff --git a/llvm/test/CodeGen/LoongArch/ir-instruction/select-bare-int.ll b/llvm/test/CodeGen/LoongArch/ir-instruction/select-bare-int.ll
index ddbc4ad719446..ad0a241f5fd38 100644
--- a/llvm/test/CodeGen/LoongArch/ir-instruction/select-bare-int.ll
+++ b/llvm/test/CodeGen/LoongArch/ir-instruction/select-bare-int.ll
@@ -106,3 +106,35 @@ define i64 @bare_select_i64(i1 %a, i64 %b, i64 %c) {
   %res = select i1 %a, i64 %b, i64 %c
   ret i64 %res
 }
+
+define i16 @bare_select_zero_i16(i1 %a, i16 %b) {
+; LA32-LABEL: bare_select_zero_i16:
+; LA32:       # %bb.0:
+; LA32-NEXT:    andi $a0, $a0, 1
+; LA32-NEXT:    masknez	$a0, $a1, $a0
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: bare_select_zero_i16:
+; LA64:       # %bb.0:
+; LA64-NEXT:    andi $a0, $a0, 1
+; LA64-NEXT:    masknez	$a0, $a1, $a0
+; LA64-NEXT:    ret
+  %res = select i1 %a, i16 0, i16 %b
+  ret i16 %res
+}
+
+define i32 @bare_select_zero_i32(i1 %a, i32 %b) {
+; LA32-LABEL: bare_select_zero_i32:
+; LA32:       # %bb.0:
+; LA32-NEXT:    andi $a0, $a0, 1
+; LA32-NEXT:    maskeqz $a0, $a1, $a0
+; LA32-NEXT:    ret
+;
+; LA64-LABEL: bare_select_zero_i32:
+; LA64:       # %bb.0:
+; LA64-NEXT:    andi $a0, $a0, 1
+; LA64-NEXT:    maskeqz $a0, $a1, $a0
+; LA64-NEXT:    ret
+  %res = select i1 %a, i32 %b, i32 0
+  ret i32 %res
+}

From eb3ca155426100b815db156677b15418c6730664 Mon Sep 17 00:00:00 2001
From: Tobias Gysi <tobias.gysi@nextsilicon.com>
Date: Mon, 19 Jun 2023 09:09:37 +0000
Subject: [PATCH 130/130] [mlir] Add a builtin distinct attribute

A distinct attribute associates a referenced attribute to a unique
identifier. Every call to its create function allocates a new
distinct attribute instance. The address of the attribute instance
temporarily serves as its unique identifier. Similar to the names
of SSA values, the final unique identifiers are generated during
pretty printing.

Examples:
 #distinct = distinct[0]<42.0 : f32>
 #distinct1 = distinct[1]<42.0 : f32>
 #distinct2 = distinct[2]<array<i32: 10, 42>>

This mechanism is meant to generate attributes with a unique
identifier, which can be used to mark groups of operations
that share a common properties such as if they are aliasing.

The design of the distinct attribute ensures minimal memory
footprint per distinct attribute since it only contains a reference
to another attribute. All distinct attributes are stored outside of
the storage uniquer in a thread local store that is part of the
context. It uses one bump pointer allocator per thread to ensure
distinct attributes can be created in-parallel.

Differential Revision: https://reviews.llvm.org/D153360
---
 mlir/docs/Dialects/Builtin.md                 | 31 ++++++++
 mlir/include/mlir/IR/AttributeSupport.h       |  2 +
 mlir/include/mlir/IR/BuiltinAttributes.h      | 37 +++++++++
 .../include/mlir/IR/BuiltinDialectBytecode.td |  7 +-
 mlir/lib/AsmParser/AttributeParser.cpp        | 56 ++++++++++++++
 mlir/lib/AsmParser/Parser.h                   |  3 +
 mlir/lib/AsmParser/ParserState.h              |  3 +
 mlir/lib/AsmParser/TokenKinds.def             |  1 +
 mlir/lib/IR/AsmPrinter.cpp                    | 38 +++++++++
 mlir/lib/IR/AttributeDetail.h                 | 66 ++++++++++++++++
 mlir/lib/IR/BuiltinAttributes.cpp             | 13 ++++
 mlir/lib/IR/BuiltinDialect.cpp                |  4 +
 mlir/lib/IR/BuiltinDialectBytecode.cpp        |  1 +
 mlir/lib/IR/MLIRContext.cpp                   | 12 +++
 mlir/test/Dialect/Builtin/Bytecode/attrs.mlir | 14 ++++
 mlir/test/IR/distinct-attr.mlir               | 22 ++++++
 mlir/test/IR/invalid-builtin-attributes.mlir  | 41 ++++++++++
 mlir/test/IR/test-builtin-distinct-attrs.mlir | 77 +++++++++++++++++++
 mlir/test/IR/test-symbol-rauw.mlir            |  6 +-
 mlir/test/IR/test-symbol-uses.mlir            |  8 +-
 mlir/test/lib/IR/CMakeLists.txt               |  1 +
 .../lib/IR/TestBuiltinDistinctAttributes.cpp  | 49 ++++++++++++
 mlir/tools/mlir-opt/mlir-opt.cpp              |  2 +
 23 files changed, 487 insertions(+), 7 deletions(-)
 create mode 100644 mlir/test/IR/distinct-attr.mlir
 create mode 100644 mlir/test/IR/test-builtin-distinct-attrs.mlir
 create mode 100644 mlir/test/lib/IR/TestBuiltinDistinctAttributes.cpp

diff --git a/mlir/docs/Dialects/Builtin.md b/mlir/docs/Dialects/Builtin.md
index b39506a39b5ab..5fc36b3e50fbd 100644
--- a/mlir/docs/Dialects/Builtin.md
+++ b/mlir/docs/Dialects/Builtin.md
@@ -23,6 +23,37 @@ Operations.
 
 [include "Dialects/BuiltinLocationAttributes.md"]
 
+## DistinctAttribute
+
+A DistinctAttribute associates an attribute to a unique identifier.
+As a result, multiple DistinctAttribute instances may point to the same
+attribute. Every call to the `create` function allocates a new
+DistinctAttribute instance. The address of the attribute instance serves as a
+temporary unique identifier. Similar to the names of SSA values, the final
+unique identifiers are generated during pretty printing. This delayed
+numbering ensures the printed identifiers are deterministic even if
+multiple DistinctAttribute instances are created in-parallel.
+
+Syntax:
+
+```
+distinct-id ::= integer-literal
+distinct-attribute ::= `distinct` `[` distinct-id `]<` attribute `>`
+```
+
+Examples:
+
+```mlir
+#distinct = distinct[0]<42.0 : f32>
+#distinct1 = distinct[1]<42.0 : f32>
+#distinct2 = distinct[2]<array<i32: 10, 42>>
+```
+
+This mechanism is meant to generate attributes with a unique
+identifier, which can be used to mark groups of operations that share a
+common property. For example, groups of aliasing memory operations may be
+marked using one DistinctAttribute instance per alias group.
+
 ## Operations
 
 [include "Dialects/BuiltinOps.md"]
diff --git a/mlir/include/mlir/IR/AttributeSupport.h b/mlir/include/mlir/IR/AttributeSupport.h
index 796adbe8531a7..75ea1ce24753c 100644
--- a/mlir/include/mlir/IR/AttributeSupport.h
+++ b/mlir/include/mlir/IR/AttributeSupport.h
@@ -149,12 +149,14 @@ class AbstractAttribute {
 
 namespace detail {
 class AttributeUniquer;
+class DistinctAttributeUniquer;
 } // namespace detail
 
 /// Base storage class appearing in an attribute. Derived storage classes should
 /// only be constructed within the context of the AttributeUniquer.
 class alignas(8) AttributeStorage : public StorageUniquer::BaseStorage {
   friend detail::AttributeUniquer;
+  friend detail::DistinctAttributeUniquer;
   friend StorageUniquer;
 
 public:
diff --git a/mlir/include/mlir/IR/BuiltinAttributes.h b/mlir/include/mlir/IR/BuiltinAttributes.h
index 7c4136021cb52..6882ff245350f 100644
--- a/mlir/include/mlir/IR/BuiltinAttributes.h
+++ b/mlir/include/mlir/IR/BuiltinAttributes.h
@@ -1000,6 +1000,43 @@ auto SparseElementsAttr::try_value_begin_impl(OverloadToken<T>) const
   return iterator<T>(llvm::seq<ptrdiff_t>(0, getNumElements()).begin(), mapFn);
 }
 
+//===----------------------------------------------------------------------===//
+// DistinctAttr
+//===----------------------------------------------------------------------===//
+
+namespace detail {
+struct DistinctAttrStorage;
+class DistinctAttributeUniquer;
+} // namespace detail
+
+/// An attribute that associates a referenced attribute to a unique identifier.
+/// Every call to the create function allocates a new distinct attribute
+/// instance. The address of the attribute instance serves as a temporary
+/// identifier. Similar to the names of SSA values, the final identifiers are
+/// generated during pretty printing. This delayed numbering ensures the
+/// printed identifiers are deterministic even if multiple distinct attribute
+/// instances are created in-parallel.
+///
+/// Examples:
+///
+/// #distinct = distinct[0]<42.0 : f32>
+/// #distinct1 = distinct[1]<42.0 : f32>
+/// #distinct2 = distinct[2]<array<i32: 10, 42>>
+class DistinctAttr
+    : public detail::StorageUserBase<DistinctAttr, Attribute,
+                                     detail::DistinctAttrStorage,
+                                     detail::DistinctAttributeUniquer> {
+public:
+  using Base::Base;
+
+  /// Returns the referenced attribute.
+  Attribute getReferencedAttr() const;
+
+  /// Creates a distinct attribute that associates a referenced attribute to a
+  /// unique identifier.
+  static DistinctAttr create(Attribute referencedAttr);
+};
+
 //===----------------------------------------------------------------------===//
 // StringAttr
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/IR/BuiltinDialectBytecode.td b/mlir/include/mlir/IR/BuiltinDialectBytecode.td
index 47d6c0df55485..9da796a5f34eb 100644
--- a/mlir/include/mlir/IR/BuiltinDialectBytecode.td
+++ b/mlir/include/mlir/IR/BuiltinDialectBytecode.td
@@ -181,6 +181,10 @@ def SparseElementsAttr : DialectAttribute<(attr
   DenseElementsAttr:$values
 )>;
 
+def DistinctAttr : DialectAttribute<(attr
+  Attribute:$referencedAttr
+)>;
+
 // Types
 // -----
 
@@ -316,7 +320,8 @@ def BuiltinDialectAttributes : DialectAttributes<"Builtin"> {
     DenseArrayAttr,
     DenseIntOrFPElementsAttr,
     DenseStringElementsAttr,
-    SparseElementsAttr
+    SparseElementsAttr,
+    DistinctAttr
   ];
 }
 
diff --git a/mlir/lib/AsmParser/AttributeParser.cpp b/mlir/lib/AsmParser/AttributeParser.cpp
index 95017942165f7..20c89d342e13f 100644
--- a/mlir/lib/AsmParser/AttributeParser.cpp
+++ b/mlir/lib/AsmParser/AttributeParser.cpp
@@ -46,6 +46,7 @@ using namespace mlir::detail;
 ///                      `:` (tensor-type | vector-type)
 ///                    | `strided` `<` `[` comma-separated-int-or-question `]`
 ///                      (`,` `offset` `:` integer-literal)? `>`
+///                    | distinct-attribute
 ///                    | extended-attribute
 ///
 Attribute Parser::parseAttribute(Type type) {
@@ -155,6 +156,10 @@ Attribute Parser::parseAttribute(Type type) {
   case Token::kw_strided:
     return parseStridedLayoutAttr();
 
+  // Parse a distinct attribute.
+  case Token::kw_distinct:
+    return parseDistinctAttr(type);
+
   // Parse a string attribute.
   case Token::string: {
     auto val = getToken().getStringValue();
@@ -1214,3 +1219,54 @@ Attribute Parser::parseStridedLayoutAttr() {
   return StridedLayoutAttr::get(getContext(), *offset, strides);
   // return getChecked<StridedLayoutAttr>(loc,getContext(), *offset, strides);
 }
+
+/// Parse a distinct attribute.
+///
+///  distinct-attribute ::= `distinct`
+///                         `[` integer-literal `]<` attribute-value `>`
+///
+Attribute Parser::parseDistinctAttr(Type type) {
+  consumeToken(Token::kw_distinct);
+  if (parseToken(Token::l_square, "expected '[' after 'distinct'"))
+    return {};
+
+  // Parse the distinct integer identifier.
+  Token token = getToken();
+  if (parseToken(Token::integer, "expected distinct ID"))
+    return {};
+  std::optional<uint64_t> value = token.getUInt64IntegerValue();
+  if (!value) {
+    emitError("expected an unsigned 64-bit integer");
+    return {};
+  }
+
+  // Parse the referenced attribute.
+  if (parseToken(Token::r_square, "expected ']' to close distinct ID") ||
+      parseToken(Token::less, "expected '<' after distinct ID"))
+    return {};
+  Attribute referencedAttr = parseAttribute(type);
+  if (!referencedAttr) {
+    emitError("expected attribute");
+    return {};
+  }
+
+  // Add the distinct attribute to the parser state, if it has not been parsed
+  // before. Otherwise, check if the parsed reference attribute matches the one
+  // found in the parser state.
+  DenseMap<uint64_t, DistinctAttr> &distinctAttrs =
+      state.symbols.distinctAttributes;
+  auto it = distinctAttrs.find(*value);
+  if (it == distinctAttrs.end()) {
+    DistinctAttr distinctAttr = DistinctAttr::create(referencedAttr);
+    it = distinctAttrs.try_emplace(*value, distinctAttr).first;
+  } else if (it->getSecond().getReferencedAttr() != referencedAttr) {
+    emitError("referenced attribute does not match previous definition: ")
+        << it->getSecond().getReferencedAttr();
+    return {};
+  }
+
+  if (parseToken(Token::greater, "expected '>' to close distinct attribute"))
+    return {};
+
+  return it->getSecond();
+}
diff --git a/mlir/lib/AsmParser/Parser.h b/mlir/lib/AsmParser/Parser.h
index 749b82c2ed4c6..d2caf78ffeede 100644
--- a/mlir/lib/AsmParser/Parser.h
+++ b/mlir/lib/AsmParser/Parser.h
@@ -250,6 +250,9 @@ class Parser {
   /// Parse an attribute dictionary.
   ParseResult parseAttributeDict(NamedAttrList &attributes);
 
+  /// Parse a distinct attribute.
+  Attribute parseDistinctAttr(Type type);
+
   /// Parse an extended attribute.
   Attribute parseExtendedAttr(Type type);
 
diff --git a/mlir/lib/AsmParser/ParserState.h b/mlir/lib/AsmParser/ParserState.h
index 62e932ff45cf1..0bf9bffaf1b60 100644
--- a/mlir/lib/AsmParser/ParserState.h
+++ b/mlir/lib/AsmParser/ParserState.h
@@ -36,6 +36,9 @@ struct SymbolState {
   DenseMap<const OpAsmDialectInterface *,
            llvm::StringMap<std::pair<std::string, AsmDialectResourceHandle>>>
       dialectResources;
+
+  /// A map from unique integer identifier to DistinctAttr.
+  DenseMap<uint64_t, DistinctAttr> distinctAttributes;
 };
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/AsmParser/TokenKinds.def b/mlir/lib/AsmParser/TokenKinds.def
index 9a632e3570fb5..4fbb7bbe30099 100644
--- a/mlir/lib/AsmParser/TokenKinds.def
+++ b/mlir/lib/AsmParser/TokenKinds.def
@@ -89,6 +89,7 @@ TOK_KEYWORD(ceildiv)
 TOK_KEYWORD(complex)
 TOK_KEYWORD(dense)
 TOK_KEYWORD(dense_resource)
+TOK_KEYWORD(distinct)
 TOK_KEYWORD(f16)
 TOK_KEYWORD(f32)
 TOK_KEYWORD(f64)
diff --git a/mlir/lib/IR/AsmPrinter.cpp b/mlir/lib/IR/AsmPrinter.cpp
index ef0bf75a9cd67..c3fcb3d538d11 100644
--- a/mlir/lib/IR/AsmPrinter.cpp
+++ b/mlir/lib/IR/AsmPrinter.cpp
@@ -806,6 +806,8 @@ class DummyAliasDialectAsmPrinter : public DialectAsmPrinter {
     } else if (llvm::isa<AffineMapAttr, DenseArrayAttr, FloatAttr, IntegerAttr,
                          IntegerSetAttr, UnitAttr>(attr)) {
       return;
+    } else if (auto distinctAttr = dyn_cast<DistinctAttr>(attr)) {
+      printAttribute(distinctAttr.getReferencedAttr());
     } else if (auto dictAttr = dyn_cast<DictionaryAttr>(attr)) {
       for (const NamedAttribute &nestedAttr : dictAttr.getValue()) {
         printAttribute(nestedAttr.getName());
@@ -1604,6 +1606,31 @@ StringRef SSANameState::uniqueValueName(StringRef name) {
   return name;
 }
 
+//===----------------------------------------------------------------------===//
+// DistinctState
+//===----------------------------------------------------------------------===//
+
+namespace {
+/// This class manages the state for distinct attributes.
+class DistinctState {
+public:
+  /// Returns a unique identifier for the given distinct attribute.
+  uint64_t getId(DistinctAttr distinctAttr);
+
+private:
+  uint64_t distinctCounter = 0;
+  DenseMap<DistinctAttr, uint64_t> distinctAttrMap;
+};
+} // namespace
+
+uint64_t DistinctState::getId(DistinctAttr distinctAttr) {
+  auto [it, inserted] =
+      distinctAttrMap.try_emplace(distinctAttr, distinctCounter);
+  if (inserted)
+    distinctCounter++;
+  return it->getSecond();
+}
+
 //===----------------------------------------------------------------------===//
 // Resources
 //===----------------------------------------------------------------------===//
@@ -1715,6 +1742,9 @@ class AsmStateImpl {
   /// Get the state used for SSA names.
   SSANameState &getSSANameState() { return nameState; }
 
+  /// Get the state used for distinct attribute identifiers.
+  DistinctState &getDistinctState() { return distinctState; }
+
   /// Return the dialects within the context that implement
   /// OpAsmDialectInterface.
   DialectInterfaceCollection<OpAsmDialectInterface> &getDialectInterfaces() {
@@ -1758,6 +1788,9 @@ class AsmStateImpl {
   /// The state used for SSA value names.
   SSANameState nameState;
 
+  /// The state used for distinct attribute identifiers.
+  DistinctState distinctState;
+
   /// Flags that control op output.
   OpPrintingFlags printerFlags;
 
@@ -2106,6 +2139,11 @@ void AsmPrinter::Impl::printAttributeImpl(Attribute attr,
   } else if (llvm::isa<UnitAttr>(attr)) {
     os << "unit";
     return;
+  } else if (auto distinctAttr = llvm::dyn_cast<DistinctAttr>(attr)) {
+    os << "distinct[" << state.getDistinctState().getId(distinctAttr) << "]<";
+    printAttribute(distinctAttr.getReferencedAttr());
+    os << '>';
+    return;
   } else if (auto dictAttr = llvm::dyn_cast<DictionaryAttr>(attr)) {
     os << '{';
     interleaveComma(dictAttr.getValue(),
diff --git a/mlir/lib/IR/AttributeDetail.h b/mlir/lib/IR/AttributeDetail.h
index 5bf7caa440277..b76da1b91bff9 100644
--- a/mlir/lib/IR/AttributeDetail.h
+++ b/mlir/lib/IR/AttributeDetail.h
@@ -14,11 +14,13 @@
 #define ATTRIBUTEDETAIL_H_
 
 #include "mlir/IR/AffineMap.h"
+#include "mlir/IR/AttributeSupport.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinTypes.h"
 #include "mlir/IR/IntegerSet.h"
 #include "mlir/IR/MLIRContext.h"
 #include "mlir/Support/StorageUniquer.h"
+#include "mlir/Support/ThreadLocalCache.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/Support/TrailingObjects.h"
@@ -349,6 +351,70 @@ struct StringAttrStorage : public AttributeStorage {
   Dialect *referencedDialect;
 };
 
+//===----------------------------------------------------------------------===//
+// DistinctAttr
+//===----------------------------------------------------------------------===//
+
+/// An attribute to store a distinct reference to another attribute.
+struct DistinctAttrStorage : public AttributeStorage {
+  using KeyTy = Attribute;
+
+  DistinctAttrStorage(Attribute referencedAttr)
+      : referencedAttr(referencedAttr) {}
+
+  /// Returns the referenced attribute as key.
+  KeyTy getAsKey() const { return KeyTy(referencedAttr); }
+
+  /// The referenced attribute.
+  Attribute referencedAttr;
+};
+
+/// A specialized attribute uniquer for distinct attributes that always
+/// allocates since the distinct attribute instances use the address of their
+/// storage as unique identifier.
+class DistinctAttributeUniquer {
+public:
+  /// Creates a distinct attribute storage. Allocates every time since the
+  /// address of the storage serves as unique identifier.
+  template <typename T, typename... Args>
+  static T get(MLIRContext *context, Args &&...args) {
+    static_assert(std::is_same_v<typename T::ImplType, DistinctAttrStorage>,
+                  "expects a distinct attribute storage");
+    DistinctAttrStorage *storage = DistinctAttributeUniquer::allocateStorage(
+        context, std::forward<Args>(args)...);
+    storage->initializeAbstractAttribute(
+        AbstractAttribute::lookup(DistinctAttr::getTypeID(), context));
+    return storage;
+  }
+
+private:
+  /// Allocates a distinct attribute storage.
+  static DistinctAttrStorage *allocateStorage(MLIRContext *context,
+                                              Attribute referencedAttr);
+};
+
+/// An allocator for distinct attribute storage instances. It uses thread local
+/// bump pointer allocators stored in a thread local cache to ensure the storage
+/// is freed after the destruction of the distinct attribute allocator.
+class DistinctAttributeAllocator {
+public:
+  DistinctAttributeAllocator() = default;
+
+  DistinctAttributeAllocator(DistinctAttributeAllocator &&) = delete;
+  DistinctAttributeAllocator(const DistinctAttributeAllocator &) = delete;
+  DistinctAttributeAllocator &
+  operator=(const DistinctAttributeAllocator &) = delete;
+
+  /// Allocates a distinct attribute storage using a thread local bump pointer
+  /// allocator to enable synchronization free parallel allocations.
+  DistinctAttrStorage *allocate(Attribute referencedAttr) {
+    return new (allocatorCache.get().Allocate<DistinctAttrStorage>())
+        DistinctAttrStorage(referencedAttr);
+  }
+
+private:
+  ThreadLocalCache<llvm::BumpPtrAllocator> allocatorCache;
+};
 } // namespace detail
 } // namespace mlir
 
diff --git a/mlir/lib/IR/BuiltinAttributes.cpp b/mlir/lib/IR/BuiltinAttributes.cpp
index 9a18f07efed01..c670359b02ec9 100644
--- a/mlir/lib/IR/BuiltinAttributes.cpp
+++ b/mlir/lib/IR/BuiltinAttributes.cpp
@@ -42,6 +42,7 @@ void BuiltinDialect::registerAttributes() {
 #define GET_ATTRDEF_LIST
 #include "mlir/IR/BuiltinAttributes.cpp.inc"
       >();
+  addAttributes<DistinctAttr>();
 }
 
 //===----------------------------------------------------------------------===//
@@ -1745,6 +1746,18 @@ SparseElementsAttr::verify(function_ref<InFlightDiagnostic()> emitError,
   return success();
 }
 
+//===----------------------------------------------------------------------===//
+// DistinctAttr
+//===----------------------------------------------------------------------===//
+
+DistinctAttr DistinctAttr::create(Attribute referencedAttr) {
+  return Base::get(referencedAttr.getContext(), referencedAttr);
+}
+
+Attribute DistinctAttr::getReferencedAttr() const {
+  return getImpl()->referencedAttr;
+}
+
 //===----------------------------------------------------------------------===//
 // Attribute Utilities
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/IR/BuiltinDialect.cpp b/mlir/lib/IR/BuiltinDialect.cpp
index a791b97c12a69..a0f6af389f19a 100644
--- a/mlir/lib/IR/BuiltinDialect.cpp
+++ b/mlir/lib/IR/BuiltinDialect.cpp
@@ -60,6 +60,10 @@ struct BuiltinOpAsmDialectInterface : public OpAsmDialectInterface {
       os << "loc";
       return AliasResult::OverridableAlias;
     }
+    if (llvm::isa<DistinctAttr>(attr)) {
+      os << "distinct";
+      return AliasResult::OverridableAlias;
+    }
     return AliasResult::NoAlias;
   }
 
diff --git a/mlir/lib/IR/BuiltinDialectBytecode.cpp b/mlir/lib/IR/BuiltinDialectBytecode.cpp
index b7a27842e2681..9487e6c50ded9 100644
--- a/mlir/lib/IR/BuiltinDialectBytecode.cpp
+++ b/mlir/lib/IR/BuiltinDialectBytecode.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "BuiltinDialectBytecode.h"
+#include "AttributeDetail.h"
 #include "mlir/Bytecode/BytecodeImplementation.h"
 #include "mlir/IR/BuiltinAttributes.h"
 #include "mlir/IR/BuiltinDialect.h"
diff --git a/mlir/lib/IR/MLIRContext.cpp b/mlir/lib/IR/MLIRContext.cpp
index cc4b33f9ca669..0b925f9bcdacf 100644
--- a/mlir/lib/IR/MLIRContext.cpp
+++ b/mlir/lib/IR/MLIRContext.cpp
@@ -247,6 +247,12 @@ class MLIRContextImpl {
   DenseMap<StringRef, SmallVector<StringAttrStorage *>>
       dialectReferencingStrAttrs;
 
+  /// A distinct attribute allocator that allocates every time since the
+  /// address of the distinct attribute storage serves as unique identifier. The
+  /// allocator is thread safe and frees the allocated storage after its
+  /// destruction.
+  DistinctAttributeAllocator distinctAttributeAllocator;
+
 public:
   MLIRContextImpl(bool threadingIsEnabled)
       : threadingIsEnabled(threadingIsEnabled) {
@@ -1059,6 +1065,12 @@ UnknownLoc UnknownLoc::get(MLIRContext *context) {
   return context->getImpl().unknownLocAttr;
 }
 
+DistinctAttrStorage *
+detail::DistinctAttributeUniquer::allocateStorage(MLIRContext *context,
+                                                  Attribute referencedAttr) {
+  return context->getImpl().distinctAttributeAllocator.allocate(referencedAttr);
+}
+
 /// Return empty dictionary.
 DictionaryAttr DictionaryAttr::getEmpty(MLIRContext *context) {
   return context->getImpl().emptyDictionaryAttr;
diff --git a/mlir/test/Dialect/Builtin/Bytecode/attrs.mlir b/mlir/test/Dialect/Builtin/Bytecode/attrs.mlir
index 2702a159be96a..f252b50e11552 100644
--- a/mlir/test/Dialect/Builtin/Bytecode/attrs.mlir
+++ b/mlir/test/Dialect/Builtin/Bytecode/attrs.mlir
@@ -128,6 +128,20 @@ module @TestType attributes {
   bytecode.type = i178
 } {}
 
+//===----------------------------------------------------------------------===//
+// DistinctAttr
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @TestDistinct
+module @TestDistinct attributes {
+  // CHECK: bytecode.distinct = distinct[0]<42 : i32>
+  // CHECK: bytecode.distinct2 = distinct[0]<42 : i32>
+  // CHECK: bytecode.distinct3 = distinct[1]<42 : i32>
+  bytecode.distinct = distinct[0]<42 : i32>,
+  bytecode.distinct2 = distinct[0]<42 : i32>,
+  bytecode.distinct3 = distinct[1]<42 : i32>
+} {}
+
 //===----------------------------------------------------------------------===//
 // CallSiteLoc
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/IR/distinct-attr.mlir b/mlir/test/IR/distinct-attr.mlir
new file mode 100644
index 0000000000000..cd151feb1f27f
--- /dev/null
+++ b/mlir/test/IR/distinct-attr.mlir
@@ -0,0 +1,22 @@
+// RUN: mlir-opt -allow-unregistered-dialect %s | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect -mlir-print-local-scope %s | FileCheck %s --check-prefix=CHECK-GENERIC
+
+// CHECK: #[[DISTINCT0:.*]] = distinct[0]<42 : i32>
+// CHECK: #[[DISTINCT1:.*]] = distinct[1]<array<i32: 10, 42>>
+// CHECK: #[[DISTINCT2:.*]] = distinct[2]<42 : i32>
+
+// CHECK:         distinct.attr = #[[DISTINCT0]]
+// CHECK-GENERIC: distinct.attr = distinct[0]<42 : i32>
+"foo.op"() {distinct.attr = distinct[0]<42 : i32>} : () -> ()
+
+// CHECK:         distinct.attr = #[[DISTINCT1]]
+// CHECK-GENERIC: distinct.attr = distinct[1]<array<i32: 10, 42>>
+"foo.op"() {distinct.attr = distinct[1]<array<i32: 10, 42>>} : () -> ()
+
+// CHECK:         distinct.attr = #[[DISTINCT0]]
+// CHECK-GENERIC: distinct.attr = distinct[0]<42 : i32>
+"foo.op"() {distinct.attr = distinct[0]<42 : i32>} : () -> ()
+
+// CHECK:         distinct.attr = #[[DISTINCT2]]
+// CHECK-GENERIC: distinct.attr = distinct[2]<42 : i32>
+"foo.op"() {distinct.attr = distinct[42]<42 : i32>} : () -> ()
diff --git a/mlir/test/IR/invalid-builtin-attributes.mlir b/mlir/test/IR/invalid-builtin-attributes.mlir
index 5343f971537b9..1ff44605cb7ec 100644
--- a/mlir/test/IR/invalid-builtin-attributes.mlir
+++ b/mlir/test/IR/invalid-builtin-attributes.mlir
@@ -546,3 +546,44 @@ func.func @duplicate_dictionary_attr_key() {
 
 // expected-error@below {{expected '>' to close an array attribute}}
 #attr = array<i8: 1)
+
+// -----
+
+// expected-error@below {{expected '[' after 'distinct'}}
+#attr = distinct<
+
+// -----
+
+// expected-error@below {{expected distinct ID}}
+#attr = distinct[i8
+
+// -----
+
+// expected-error@below {{expected an unsigned 64-bit integer}}
+#attr = distinct[0xAAAABBBBEEEEFFFF1]
+
+// -----
+
+// expected-error@below {{expected ']' to close distinct ID}}
+#attr = distinct[8)
+
+// -----
+
+// expected-error@below {{expected '<' after distinct ID}}
+#attr = distinct[8](
+
+// -----
+
+// expected-error@below {{expected attribute}}
+#attr = distinct[8]<attribute
+
+// -----
+
+// expected-error@below {{expected '>' to close distinct attribute}}
+#attr = distinct[8]<@foo]
+
+// -----
+
+#attr = distinct[0]<42 : i32>
+// expected-error@below {{referenced attribute does not match previous definition: 42 : i32}}
+#attr1 = distinct[0]<43 : i32>
diff --git a/mlir/test/IR/test-builtin-distinct-attrs.mlir b/mlir/test/IR/test-builtin-distinct-attrs.mlir
new file mode 100644
index 0000000000000..861d664ca4855
--- /dev/null
+++ b/mlir/test/IR/test-builtin-distinct-attrs.mlir
@@ -0,0 +1,77 @@
+// RUN: mlir-opt -allow-unregistered-dialect %s -test-distinct-attrs | FileCheck %s
+
+// CHECK: #[[DIST0:.*]] = distinct[0]<42 : i32>
+// CHECK: #[[DIST1:.*]] = distinct[1]<42 : i32>
+#distinct = distinct[0]<42 : i32>
+// CHECK: #[[DIST2:.*]] = distinct[2]<42 : i32>
+// CHECK: #[[DIST3:.*]] = distinct[3]<42 : i32>
+#distinct1 = distinct[1]<42 : i32>
+// CHECK: #[[DIST4:.*]] = distinct[4]<43 : i32>
+// CHECK: #[[DIST5:.*]] = distinct[5]<43 : i32>
+#distinct2 = distinct[2]<43 : i32>
+// CHECK: #[[DIST6:.*]] = distinct[6]<@foo_1>
+// CHECK: #[[DIST7:.*]] = distinct[7]<@foo_1>
+#distinct3 = distinct[3]<@foo_1>
+
+// Copies made for foo_2
+// CHECK: #[[DIST8:.*]] = distinct[8]<42 : i32>
+// CHECK: #[[DIST9:.*]] = distinct[9]<42 : i32>
+// CHECK: #[[DIST10:.*]] = distinct[10]<43 : i32>
+// CHECK: #[[DIST11:.*]] = distinct[11]<@foo_1>
+
+// Copies made for foo_3
+// CHECK: #[[DIST12:.*]] = distinct[12]<42 : i32>
+// CHECK: #[[DIST13:.*]] = distinct[13]<42 : i32>
+// CHECK: #[[DIST14:.*]] = distinct[14]<43 : i32>
+// CHECK: #[[DIST15:.*]] = distinct[15]<@foo_1>
+
+// Copies made for foo_4
+// CHECK: #[[DIST16:.*]] = distinct[16]<42 : i32>
+// CHECK: #[[DIST17:.*]] = distinct[17]<42 : i32>
+// CHECK: #[[DIST18:.*]] = distinct[18]<43 : i32>
+// CHECK: #[[DIST19:.*]] = distinct[19]<@foo_1>
+
+// CHECK: @foo_1
+func.func @foo_1() {
+  // CHECK: "foo.op"() {distinct.input = #[[DIST0]], distinct.output = #[[DIST1]]}
+  "foo.op"() {distinct.input = #distinct} : () -> ()
+  // CHECK: "foo.op"() {distinct.input = #[[DIST2]], distinct.output = #[[DIST3]]}
+  "foo.op"() {distinct.input = #distinct1} : () -> ()
+  // CHECK: "foo.op"() {distinct.input = #[[DIST4]], distinct.output = #[[DIST5]]}
+  "foo.op"() {distinct.input = #distinct2} : () -> ()
+  // CHECK: "foo.op"() {distinct.input = #[[DIST6]], distinct.output = #[[DIST7]]}
+  "foo.op"() {distinct.input = #distinct3} : () -> ()
+}
+
+func.func @foo_2() {
+  // CHECK: "foo.op"() {distinct.input = #[[DIST0]], distinct.output = #[[DIST8]]}
+  "foo.op"() {distinct.input = #distinct} : () -> ()
+  // CHECK: "foo.op"() {distinct.input = #[[DIST2]], distinct.output = #[[DIST9]]}
+  "foo.op"() {distinct.input = #distinct1} : () -> ()
+  // CHECK: "foo.op"() {distinct.input = #[[DIST4]], distinct.output = #[[DIST10]]}
+  "foo.op"() {distinct.input = #distinct2} : () -> ()
+  // CHECK: "foo.op"() {distinct.input = #[[DIST6]], distinct.output = #[[DIST11]]}
+  "foo.op"() {distinct.input = #distinct3} : () -> ()
+}
+
+func.func @foo_3() {
+  // CHECK: "foo.op"() {distinct.input = #[[DIST0]], distinct.output = #[[DIST12]]}
+  "foo.op"() {distinct.input = #distinct} : () -> ()
+  // CHECK: "foo.op"() {distinct.input = #[[DIST2]], distinct.output = #[[DIST13]]}
+  "foo.op"() {distinct.input = #distinct1} : () -> ()
+  // CHECK: "foo.op"() {distinct.input = #[[DIST4]], distinct.output = #[[DIST14]]}
+  "foo.op"() {distinct.input = #distinct2} : () -> ()
+  // CHECK: "foo.op"() {distinct.input = #[[DIST6]], distinct.output = #[[DIST15]]}
+  "foo.op"() {distinct.input = #distinct3} : () -> ()
+}
+
+func.func @foo_4() {
+  // CHECK: "foo.op"() {distinct.input = #[[DIST0]], distinct.output = #[[DIST16]]}
+  "foo.op"() {distinct.input = #distinct} : () -> ()
+  // CHECK: "foo.op"() {distinct.input = #[[DIST2]], distinct.output = #[[DIST17]]}
+  "foo.op"() {distinct.input = #distinct1} : () -> ()
+  // CHECK: "foo.op"() {distinct.input = #[[DIST4]], distinct.output = #[[DIST18]]}
+  "foo.op"() {distinct.input = #distinct2} : () -> ()
+  // CHECK: "foo.op"() {distinct.input = #[[DIST6]], distinct.output = #[[DIST19]]}
+  "foo.op"() {distinct.input = #distinct3} : () -> ()
+}
diff --git a/mlir/test/IR/test-symbol-rauw.mlir b/mlir/test/IR/test-symbol-rauw.mlir
index fe1c80e8f400e..ba17cf9d10426 100644
--- a/mlir/test/IR/test-symbol-rauw.mlir
+++ b/mlir/test/IR/test-symbol-rauw.mlir
@@ -1,4 +1,4 @@
-// RUN: mlir-opt -allow-unregistered-dialect %s -test-symbol-rauw -split-input-file | FileCheck %s
+// RUN: mlir-opt -allow-unregistered-dialect -mlir-print-local-scope %s -test-symbol-rauw -split-input-file | FileCheck %s
 
 // Symbol references to the module itself don't affect uses of symbols within
 // its table.
@@ -85,11 +85,11 @@ module {
     func.func @symbol_bar() {
       // CHECK: foo.op
       // CHECK-SAME: non_symbol_attr,
-      // CHECK-SAME: use = [#test.sub_elements_access<[@replaced_foo], @symbol_bar, @replaced_foo>],
+      // CHECK-SAME: use = [#test.sub_elements_access<[@replaced_foo], @symbol_bar, @replaced_foo>, distinct[0]<@replaced_foo>],
       // CHECK-SAME: z_non_symbol_attr_3
       "foo.op"() {
         non_symbol_attr,
-        use = [#test.sub_elements_access<[@symbol_foo],@symbol_bar,@symbol_foo>],
+        use = [#test.sub_elements_access<[@symbol_foo],@symbol_bar,@symbol_foo>, distinct[0]<@symbol_foo>],
         z_non_symbol_attr_3
       } : () -> ()
     }
diff --git a/mlir/test/IR/test-symbol-uses.mlir b/mlir/test/IR/test-symbol-uses.mlir
index 75be62778051f..54e3ef1812510 100644
--- a/mlir/test/IR/test-symbol-uses.mlir
+++ b/mlir/test/IR/test-symbol-uses.mlir
@@ -4,19 +4,21 @@
 // its table.
 // expected-remark@below {{symbol_removable function successfully erased}}
 module attributes {sym.outside_use = @symbol_foo } {
-  // expected-remark@+1 {{symbol has 2 uses}}
+  // expected-remark@+1 {{symbol has 3 uses}}
   func.func private @symbol_foo()
 
   // expected-remark@below {{symbol has no uses}}
   // expected-remark@below {{found use of symbol : @symbol_foo}}
-  // expected-remark@below {{symbol contains 2 nested references}}
+  // expected-remark@below {{symbol contains 3 nested references}}
   func.func @symbol_bar() attributes {sym.use = @symbol_foo} {
     // expected-remark@+1 {{found use of symbol : @symbol_foo}}
     "foo.op"() {
       non_symbol_attr,
-      use = [{ nested_symbol = [@symbol_foo]}],
+      use = [{nested_symbol = [@symbol_foo]}],
       z_other_non_symbol_attr
     } : () -> ()
+    // expected-remark@+1 {{found use of symbol : @symbol_foo}}
+    "foo.op"() { use = distinct[0]<@symbol_foo> } : () -> ()
   }
 
   // expected-remark@below {{symbol has no uses}}
diff --git a/mlir/test/lib/IR/CMakeLists.txt b/mlir/test/lib/IR/CMakeLists.txt
index 004e728b4b877..447a2481e8dba 100644
--- a/mlir/test/lib/IR/CMakeLists.txt
+++ b/mlir/test/lib/IR/CMakeLists.txt
@@ -1,6 +1,7 @@
 # Exclude tests from libMLIR.so
 add_mlir_library(MLIRTestIR
   TestBuiltinAttributeInterfaces.cpp
+  TestBuiltinDistinctAttributes.cpp
   TestClone.cpp
   TestDiagnostics.cpp
   TestDominance.cpp
diff --git a/mlir/test/lib/IR/TestBuiltinDistinctAttributes.cpp b/mlir/test/lib/IR/TestBuiltinDistinctAttributes.cpp
new file mode 100644
index 0000000000000..4717ce345fc1f
--- /dev/null
+++ b/mlir/test/lib/IR/TestBuiltinDistinctAttributes.cpp
@@ -0,0 +1,49 @@
+//===- TestBuiltinDistinctAttributes.cpp - Test DistinctAttributes --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "TestDialect.h"
+#include "mlir/IR/BuiltinAttributes.h"
+#include "mlir/Pass/Pass.h"
+
+using namespace mlir;
+
+namespace {
+/// This is a distinct attribute test pass that tests if distinct attributes can
+/// be created in parallel in a deterministic way.
+struct DistinctAttributesPass
+    : public PassWrapper<DistinctAttributesPass, OperationPass<func::FuncOp>> {
+  MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(DistinctAttributesPass)
+
+  StringRef getArgument() const final { return "test-distinct-attrs"; }
+  StringRef getDescription() const final {
+    return "Test parallel creation of distinct attributes";
+  }
+
+  void runOnOperation() override {
+    auto funcOp = getOperation();
+
+    /// Walk all operations and create a distinct output attribute given a
+    /// distinct input attribute.
+    funcOp->walk([](Operation *op) {
+      auto distinctAttr = op->getAttrOfType<DistinctAttr>("distinct.input");
+      if (!distinctAttr)
+        return;
+      op->setAttr("distinct.output",
+                  DistinctAttr::create(distinctAttr.getReferencedAttr()));
+    });
+  }
+};
+} // namespace
+
+namespace mlir {
+namespace test {
+void registerTestBuiltinDistinctAttributes() {
+  PassRegistration<DistinctAttributesPass>();
+}
+} // namespace test
+} // namespace mlir
diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp
index d75b54e8bd45a..4b7ae696e4777 100644
--- a/mlir/tools/mlir-opt/mlir-opt.cpp
+++ b/mlir/tools/mlir-opt/mlir-opt.cpp
@@ -73,6 +73,7 @@ void registerTestAffineLoopParametricTilingPass();
 void registerTestArithEmulateWideIntPass();
 void registerTestAliasAnalysisPass();
 void registerTestBuiltinAttributeInterfaces();
+void registerTestBuiltinDistinctAttributes();
 void registerTestCallGraphPass();
 void registerTestCfAssertPass();
 void registerTestConstantFold();
@@ -187,6 +188,7 @@ void registerTestPasses() {
   mlir::test::registerTestAliasAnalysisPass();
   mlir::test::registerTestArithEmulateWideIntPass();
   mlir::test::registerTestBuiltinAttributeInterfaces();
+  mlir::test::registerTestBuiltinDistinctAttributes();
   mlir::test::registerTestCallGraphPass();
   mlir::test::registerTestCfAssertPass();
   mlir::test::registerTestConstantFold();