Add a diagnostic kstat for obtaining pool status

This kstat output does not require taking the spa_namespace
lock, as in the case for 'zpool status'. It can be used for
investigations when pools are in a hung state while holding
global locks required for a traditional 'zpool status' to
proceed.

This kstat is not safe to use in conditions where pools are
in the process of configuration changes (i.e., adding/removing
devices). Therefore, this kstat is not intended to be a general
replacement or alternative to using 'zpool status'.

The newly introduced zfs_lockless_read_enabled module parameter
enables traversal of related kernel structures in cases where
the required read locks cannot be taken.

Sponsored-by: Klara, Inc.
Sponsored-by: Wasabi Technology, Inc.
Co-authored-by: Don Brady <[email protected]>
Co-authored-by: Umer Saleem <[email protected]>
Signed-off-by: Igor Ostapenko <[email protected]>

Author: 3 people

cmd/zfs/zfs_main.c

@@ -81,6 +81,7 @@
 #include "zfs_util.h"
 #include "zfs_comutil.h"
 #include "zfs_projectutil.h"
+#include "json.h"
 
 libzfs_handle_t *g_zfs;
 
@@ -1907,18 +1908,11 @@ is_recvd_column(zprop_get_cbdata_t *cbp)
 static nvlist_t *
 zfs_json_schema(int maj_v, int min_v)
 {
-	nvlist_t *sch = NULL;
-	nvlist_t *ov = NULL;
 	char cmd[MAX_CMD_LEN];
 	snprintf(cmd, MAX_CMD_LEN, "zfs %s", current_command->name);
 
-	sch = fnvlist_alloc();
-	ov = fnvlist_alloc();
-	fnvlist_add_string(ov, "command", cmd);
-	fnvlist_add_uint32(ov, "vers_major", maj_v);
-	fnvlist_add_uint32(ov, "vers_minor", min_v);
-	fnvlist_add_nvlist(sch, "output_version", ov);
-	fnvlist_free(ov);
+	nvlist_t *sch = fnvlist_alloc();
+	json_add_output_version(sch, cmd, maj_v, min_v);
 	return (sch);
 }
 

cmd/zpool/zpool_main.c

@@ -76,6 +76,8 @@
 #include "zfs_comutil.h"
 #include "zfeature_common.h"
 #include "zfs_valstr.h"
+#include "json.h"
+#include "literals.h"
 
 #include "statcommon.h"
 
@@ -309,41 +311,6 @@ static const char *checkpoint_state_str[] = {
 	"DISCARDING"
 };
 
-static const char *vdev_state_str[] = {
-	"UNKNOWN",
-	"CLOSED",
-	"OFFLINE",
-	"REMOVED",
-	"CANT_OPEN",
-	"FAULTED",
-	"DEGRADED",
-	"ONLINE"
-};
-
-static const char *vdev_aux_str[] = {
-	"NONE",
-	"OPEN_FAILED",
-	"CORRUPT_DATA",
-	"NO_REPLICAS",
-	"BAD_GUID_SUM",
-	"TOO_SMALL",
-	"BAD_LABEL",
-	"VERSION_NEWER",
-	"VERSION_OLDER",
-	"UNSUP_FEAT",
-	"SPARED",
-	"ERR_EXCEEDED",
-	"IO_FAILURE",
-	"BAD_LOG",
-	"EXTERNAL",
-	"SPLIT_POOL",
-	"BAD_ASHIFT",
-	"EXTERNAL_PERSIST",
-	"ACTIVE",
-	"CHILDREN_OFFLINE",
-	"ASHIFT_TOO_BIG"
-};
-
 static const char *vdev_init_state_str[] = {
 	"NONE",
 	"ACTIVE",
@@ -1089,15 +1056,10 @@ static nvlist_t *
 zpool_json_schema(int maj_v, int min_v)
 {
 	char cmd[MAX_CMD_LEN];
-	nvlist_t *sch = fnvlist_alloc();
-	nvlist_t *ov = fnvlist_alloc();
-
 	snprintf(cmd, MAX_CMD_LEN, "zpool %s", current_command->name);
-	fnvlist_add_string(ov, "command", cmd);
-	fnvlist_add_uint32(ov, "vers_major", maj_v);
-	fnvlist_add_uint32(ov, "vers_minor", min_v);
-	fnvlist_add_nvlist(sch, "output_version", ov);
-	fnvlist_free(ov);
+
+	nvlist_t *sch = fnvlist_alloc();
+	json_add_output_version(sch, cmd, maj_v, min_v);
 	return (sch);
 }
 
@@ -1247,7 +1209,7 @@ fill_vdev_info(nvlist_t *list, zpool_handle_t *zhp, char *name,
 		if (nvlist_lookup_uint64_array(nvdev, ZPOOL_CONFIG_VDEV_STATS,
 		    (uint64_t **)&vs, &c) == 0) {
 			fnvlist_add_string(list, "state",
-			    vdev_state_str[vs->vs_state]);
+			    vdev_state_string(vs->vs_state));
 		}
 	}
 }
@@ -9275,7 +9237,7 @@ vdev_stats_nvlist(zpool_handle_t *zhp, status_cbdata_t *cb, nvlist_t *nv,
 		fnvlist_add_string(vds, "was",
 		    fnvlist_lookup_string(nv, ZPOOL_CONFIG_PATH));
 	} else if (vs->vs_aux != VDEV_AUX_NONE) {
-		fnvlist_add_string(vds, "aux", vdev_aux_str[vs->vs_aux]);
+		fnvlist_add_string(vds, "aux", vdev_aux_string(vs->vs_aux));
 	} else if (children == 0 && !isspare &&
 	    getenv("ZPOOL_STATUS_NON_NATIVE_ASHIFT_IGNORE") == NULL &&
 	    VDEV_STAT_VALID(vs_physical_ashift, vsc) &&

include/Makefile.am

@@ -8,6 +8,8 @@ endif
 
 COMMON_H = \
 	cityhash.h \
+	json.h \
+	literals.h \
 	zfeature_common.h \
 	zfs_comutil.h \
 	zfs_deleg.h \
@@ -176,6 +178,7 @@ COMMON_H = \
 
 
 KERNEL_H = \
+	sys/spa_stats_json.h \
 	sys/zfs_ioctl.h \
 	sys/zfs_ioctl_impl.h \
 	sys/zfs_onexit.h \

include/json.h

@@ -0,0 +1,42 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or https://opensource.org/licenses/CDDL-1.0.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2025, Klara, Inc.
+ */
+
+#ifndef _JSON_H
+#define	_JSON_H extern __attribute__((visibility("default")))
+
+#include <sys/nvpair.h>
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+_JSON_H void json_add_output_version(nvlist_t *nvl, const char *cmd, int maj_v,
+    int min_v);
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _JSON_H */

include/literals.h

@@ -0,0 +1,42 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or https://opensource.org/licenses/CDDL-1.0.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2025, Klara, Inc.
+ */
+
+#ifndef _LITERALS_H
+#define	_LITERALS_H extern __attribute__((visibility("default")))
+
+#include <sys/fs/zfs.h>
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+_LITERALS_H const char *vdev_state_string(uint64_t state);
+_LITERALS_H const char *vdev_aux_string(uint64_t aux);
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif	/* _LITERALS_H */

include/sys/nvpair.h

@@ -412,6 +412,18 @@ _SYS_NVPAIR_H uint64_t fnvpair_value_uint64(const nvpair_t *nvp);
 _SYS_NVPAIR_H const char *fnvpair_value_string(const nvpair_t *nvp);
 _SYS_NVPAIR_H nvlist_t *fnvpair_value_nvlist(nvpair_t *nvp);
 
+/* JSON generation */
+
+typedef int (*nvjson_writer_t)(void *, const char *);
+typedef struct nvjson {
+	char *buf;
+	size_t size;
+	nvjson_writer_t writer;
+	void *writer_ctx;
+	int (*str_handler)(const char *, nvjson_writer_t, void *);
+} nvjson_t;
+_SYS_NVPAIR_H int nvlist_to_json(nvjson_t *, nvlist_t *);
+
 #ifdef	__cplusplus
 }
 #endif

include/sys/spa_impl.h

@@ -35,6 +35,7 @@
 #include <sys/spa.h>
 #include <sys/spa_checkpoint.h>
 #include <sys/spa_log_spacemap.h>
+#include <sys/spa_stats_json.h>
 #include <sys/vdev.h>
 #include <sys/vdev_rebuild.h>
 #include <sys/vdev_removal.h>
@@ -447,6 +448,7 @@ struct spa {
 	uint64_t	spa_autotrim;		/* automatic background trim? */
 	uint64_t	spa_errata;		/* errata issues detected */
 	spa_stats_t	spa_stats;		/* assorted spa statistics */
+	spa_stats_json_t	spa_stats_json;	/* diagnostic status in JSON */
 	spa_keystore_t	spa_keystore;		/* loaded crypto keys */
 
 	/* arc_memory_throttle() parameters during low memory condition */

include/sys/spa_stats_json.h

@@ -0,0 +1,47 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://opensource.org/licenses/CDDL-1.0.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2024, Klara, Inc.
+ */
+
+#ifndef _SYS_SPA_STATS_JSON_H
+#define	_SYS_SPA_STATS_JSON_H
+
+#include <sys/spa.h>
+#include <sys/kstat.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct spa_stats_json {
+	kmutex_t	ssj_lock;
+	kstat_t		*ssj_kstat;
+} spa_stats_json_t;
+
+extern int spa_stats_json_generate(spa_t *spa, char *buf, size_t size);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _SYS_SPA_STATS_JSON_H */

lib/libnvpair/Makefile.am

@@ -17,6 +17,7 @@ dist_libnvpair_la_SOURCES = \
 nodist_libnvpair_la_SOURCES = \
 	module/nvpair/nvpair_alloc_fixed.c \
 	module/nvpair/nvpair.c \
+	module/nvpair/nvpair_json.c \
 	module/nvpair/fnvpair.c
 
 libnvpair_la_LIBADD = \

lib/libnvpair/libnvpair.abi

@@ -196,6 +196,7 @@
     <elf-symbol name='nvlist_remove_all' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='nvlist_remove_nvpair' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='nvlist_size' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
+    <elf-symbol name='nvlist_to_json' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='nvlist_unpack' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='nvlist_xalloc' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
     <elf-symbol name='nvlist_xdup' type='func-type' binding='global-binding' visibility='default-visibility' is-defined='yes'/>
@@ -1879,6 +1880,25 @@
     <array-type-def dimensions='1' type-id='a84c031d' size-in-bits='32' id='8e0573fd'>
       <subrange length='4' type-id='7359adad' id='16fe7105'/>
     </array-type-def>
+    <typedef-decl name='nvjson_writer_t' type-id='b98b07cc' id='4e9d7f33'/>
+    <class-decl name='nvjson' size-in-bits='320' is-struct='yes' visibility='default' id='dcaf122b'>
+      <data-member access='public' layout-offset-in-bits='0'>
+        <var-decl name='buf' type-id='26a90f95' visibility='default'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='64'>
+        <var-decl name='size' type-id='b59d7dce' visibility='default'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='128'>
+        <var-decl name='writer' type-id='4e9d7f33' visibility='default'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='192'>
+        <var-decl name='writer_ctx' type-id='eaa32e2f' visibility='default'/>
+      </data-member>
+      <data-member access='public' layout-offset-in-bits='256'>
+        <var-decl name='str_handler' type-id='60a1a574' visibility='default'/>
+      </data-member>
+    </class-decl>
+    <typedef-decl name='nvjson_t' type-id='dcaf122b' id='25417009'/>
     <class-decl name='__mbstate_t' size-in-bits='64' is-struct='yes' naming-typedef-id='55e5b2b5' visibility='default' id='87447a2e'>
       <data-member access='public' layout-offset-in-bits='0'>
         <var-decl name='__count' type-id='95e97e5e' visibility='default'/>
@@ -1898,8 +1918,11 @@
     <typedef-decl name='__mbstate_t' type-id='87447a2e' id='55e5b2b5'/>
     <typedef-decl name='mbstate_t' type-id='55e5b2b5' id='3d7d8cbf'/>
     <typedef-decl name='wchar_t' type-id='95e97e5e' id='928221d2'/>
+    <pointer-type-def type-id='d3d2487a' size-in-bits='64' id='60a1a574'/>
+    <pointer-type-def type-id='fec2782a' size-in-bits='64' id='b98b07cc'/>
     <pointer-type-def type-id='3d7d8cbf' size-in-bits='64' id='a68021ce'/>
     <qualified-type-def type-id='a68021ce' restrict='yes' id='03aaab72'/>
+    <pointer-type-def type-id='25417009' size-in-bits='64' id='cd4de0be'/>
     <pointer-type-def type-id='928221d2' size-in-bits='64' id='323d93c1'/>
     <qualified-type-def type-id='323d93c1' restrict='yes' id='f1358bc3'/>
     <function-decl name='fnvpair_value_boolean_value' mangled-name='fnvpair_value_boolean_value' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='fnvpair_value_boolean_value'>
@@ -1973,6 +1996,17 @@
       <parameter type-id='5ce45b60' name='nvl'/>
       <return type-id='95e97e5e'/>
     </function-decl>
+    <function-type size-in-bits='64' id='d3d2487a'>
+      <parameter type-id='80f4b756'/>
+      <parameter type-id='4e9d7f33'/>
+      <parameter type-id='eaa32e2f'/>
+      <return type-id='95e97e5e'/>
+    </function-type>
+    <function-type size-in-bits='64' id='fec2782a'>
+      <parameter type-id='eaa32e2f'/>
+      <parameter type-id='80f4b756'/>
+      <return type-id='95e97e5e'/>
+    </function-type>
   </abi-instr>
   <abi-instr address-size='64' path='lib/libnvpair/nvpair_alloc_system.c' language='LANG_C99'>
     <class-decl name='__va_list_tag' size-in-bits='192' is-struct='yes' visibility='default' id='d5027220'>
@@ -3434,4 +3468,11 @@
     <qualified-type-def type-id='ee1d4944' const='yes' id='4b95388f'/>
     <var-decl name='nv_fixed_ops' type-id='4b95388f' mangled-name='nv_fixed_ops' visibility='default' elf-symbol-id='nv_fixed_ops'/>
   </abi-instr>
+  <abi-instr address-size='64' path='module/nvpair/nvpair_json.c' language='LANG_C99'>
+    <function-decl name='nvlist_to_json' mangled-name='nvlist_to_json' visibility='default' binding='global' size-in-bits='64' elf-symbol-id='nvlist_to_json'>
+      <parameter type-id='cd4de0be'/>
+      <parameter type-id='5ce45b60'/>
+      <return type-id='95e97e5e'/>
+    </function-decl>
+  </abi-instr>
 </abi-corpus>