CDRIVER-3775 mongoc_structured_log (#1795)

This is an initial implementation of "structured logging" for libmongoc.

* Partially addresses CDRIVER-3775 (Easier debugging with standardized logging)
* Partially addresses CDRIVER-4485 (Logging Specification); prose tests and truncation TBD
* Partially addresses CDRIVER-4486 (Command logging and monitoring); prose tests and truncation TBD
* Addresses CDRIVER-4535 (Misc updates to logging spec)
* Addresses CDRIVER-4553 (Command logging test covering unacknowledged write)
* Unified test runner support for: createEntities, observeLogMessages, waitForEvent, $$matchAsDocument, $$matchAsRoot
* Add serverDescriptionChangedEvent to the unified test runner and unify its two event serialization systems
* Private serialization utilities, *_append_contents_to_bson
* Sync unified tests from the command-logging-and-monitoring spec
* bson-dsl support for oid() values
* Private utilities for dealing with zero'ed oids

Author: mdbmes

src/common/src/bson-dsl.md

@@ -262,6 +262,12 @@ Generate a UTF-8 value from the given null-terminated character array beginning
 at `zstr`.
 
 
+#### `oid(const bson_oid_t* oid)`
+
+Generate an ObjectId value from the given C expression that evaluates to
+a bson_oid_t pointer.
+
+
 #### `utf8_w_len(const char* str, int len)`
 
 Generate a UTF-8 value from the character array beginning at `str`, with length

src/common/src/common-bson-dsl-private.h

@@ -246,6 +246,13 @@ BSON_IF_GNU_LIKE (_Pragma ("GCC diagnostic ignored \"-Wshadow\""))
 #define _bsonArrayOperation_boolean(X) _bsonArrayAppendValue (boolean (X))
 #define _bsonValueOperation_boolean(b) _bsonValueOperation_bool (b)
 
+#define _bsonValueOperation_oid(o)                                        \
+   if (!bson_append_oid (_bsonBuildAppendArgs, (o))) {                    \
+      bsonBuildError = "Error while appending oid(" _bsonDSL_str (o) ")"; \
+   } else                                                                 \
+      ((void) 0)
+#define _bsonArrayOperation_oid(X) _bsonArrayAppendValue (oid (X))
+
 #define _bsonValueOperation_null                       \
    if (!bson_append_null (_bsonBuildAppendArgs)) {     \
       bsonBuildError = "Error while appending a null"; \

src/common/src/common-oid-private.h

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2009-present MongoDB, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "common-prelude.h"
+
+#ifndef MONGO_C_DRIVER_COMMON_OID_PRIVATE_H
+#define MONGO_C_DRIVER_COMMON_OID_PRIVATE_H
+
+#include <bson/bson.h>
+
+BSON_BEGIN_DECLS
+
+extern const bson_oid_t kZeroObjectId;
+
+void
+mcommon_oid_set_zero (bson_oid_t *oid);
+
+bool
+mcommon_oid_is_zero (const bson_oid_t *oid);
+
+BSON_END_DECLS
+
+#endif /* MONGO_C_DRIVER_COMMON_OID_PRIVATE_H */

src/common/src/common-oid.c

@@ -0,0 +1,33 @@
+/*
+ * Copyright 2009-present MongoDB, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <common-oid-private.h>
+
+const bson_oid_t kZeroObjectId = {{0}};
+
+void
+mcommon_oid_set_zero (bson_oid_t *oid)
+{
+   BSON_ASSERT (oid);
+   memset (oid, 0, sizeof *oid);
+}
+
+bool
+mcommon_oid_is_zero (const bson_oid_t *oid)
+{
+   BSON_ASSERT (oid);
+   return bson_oid_equal_unsafe (oid, &kZeroObjectId);
+}

src/common/tests/test-common-oid.c

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2009-present MongoDB, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "TestSuite.h"
+
+#include <common-oid-private.h>
+
+static void
+test_mcommon_oid_zero (void)
+{
+   bson_oid_t oid;
+   bson_oid_init_from_string (&oid, "000000000000000000000000");
+   BSON_ASSERT (true == bson_oid_equal (&oid, &kZeroObjectId));
+   BSON_ASSERT (true == mcommon_oid_is_zero (&oid));
+   bson_oid_init_from_string (&oid, "010000000000000000000000");
+   BSON_ASSERT (false == mcommon_oid_is_zero (&oid));
+   bson_oid_init_from_string (&oid, "000000000000000000000001");
+   BSON_ASSERT (false == mcommon_oid_is_zero (&oid));
+   bson_oid_init_from_string (&oid, "ffffffffffffffffffffffff");
+   BSON_ASSERT (false == mcommon_oid_is_zero (&oid));
+   mcommon_oid_set_zero (&oid);
+   BSON_ASSERT (true == mcommon_oid_is_zero (&oid));
+}
+
+void
+test_mcommon_oid_install (TestSuite *suite)
+{
+   TestSuite_Add (suite, "/mcommon/oid/zero", test_mcommon_oid_zero);
+}

src/libbson/src/bson/bson-macros.h

@@ -22,6 +22,7 @@
 
 
 #include <stdio.h>
+#include <stdlib.h>
 
 #ifdef __cplusplus
 #include <algorithm>
@@ -190,25 +191,52 @@
 #define BSON_FUNC __func__
 #endif
 
-#define BSON_ASSERT(test)                                                                                         \
-   do {                                                                                                           \
-      if (!(BSON_LIKELY (test))) {                                                                                \
-         fprintf (stderr, "%s:%d %s(): precondition failed: %s\n", __FILE__, (int) (__LINE__), BSON_FUNC, #test); \
-         abort ();                                                                                                \
-      }                                                                                                           \
+
+#if defined(_MSC_VER)
+#define BSON_INLINE __inline
+#else
+#define BSON_INLINE __inline__
+#endif
+
+
+#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202311L
+#define BSON_NORETURN noreturn
+#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
+#define BSON_NORETURN _Noreturn
+#elif defined(__GNUC__) && 2 < __GNUC__ + (8 <= __GNUC_MINOR__)
+#define BSON_NORETURN __attribute__ ((__noreturn__))
+#else
+#define BSON_NORETURN
+#endif
+
+
+static BSON_INLINE BSON_NORETURN void
+_bson_assert_failed_on_line (const char *file, int line, const char *func, const char *test)
+{
+   fprintf (stderr, "%s:%d %s(): assertion failed: %s\n", file, line, func, test);
+   abort ();
+}
+
+static BSON_INLINE BSON_NORETURN void
+_bson_assert_failed_on_param (const char *param, const char *func)
+{
+   fprintf (stderr, "The parameter: %s, in function %s, cannot be NULL\n", param, func);
+   abort ();
+}
+
+#define BSON_ASSERT(test)                                                            \
+   do {                                                                              \
+      if (!(BSON_LIKELY (test))) {                                                   \
+         _bson_assert_failed_on_line (__FILE__, (int) (__LINE__), BSON_FUNC, #test); \
+      }                                                                              \
    } while (0)
 
 /**
  * @brief Assert the expression `Assertion`, and evaluates to `Value` on
  * success.
  */
 #define BSON_ASSERT_INLINE(Assertion, Value)                                                                           \
-   ((void) ((Assertion)                                                                                                \
-               ? (0)                                                                                                   \
-               : ((fprintf (                                                                                           \
-                      stderr, "%s:%d %s(): Assertion '%s' failed", __FILE__, (int) (__LINE__), BSON_FUNC, #Assertion), \
-                   abort ()),                                                                                          \
-                  0)),                                                                                                 \
+   ((void) ((Assertion) ? (0) : (_bson_assert_failed_on_line (__FILE__, (int) (__LINE__), BSON_FUNC, #Assertion), 0)), \
     Value)
 
 /**
@@ -225,12 +253,11 @@
 #define BSON_ASSERT_PTR_INLINE(Pointer) BSON_ASSERT_INLINE ((Pointer) != NULL, (Pointer))
 
 /* Used for asserting parameters to provide a more precise error message */
-#define BSON_ASSERT_PARAM(param)                                                                     \
-   do {                                                                                              \
-      if ((BSON_UNLIKELY (param == NULL))) {                                                         \
-         fprintf (stderr, "The parameter: %s, in function %s, cannot be NULL\n", #param, BSON_FUNC); \
-         abort ();                                                                                   \
-      }                                                                                              \
+#define BSON_ASSERT_PARAM(param)                           \
+   do {                                                    \
+      if ((BSON_UNLIKELY (param == NULL))) {               \
+         _bson_assert_failed_on_param (#param, BSON_FUNC); \
+      }                                                    \
    } while (0)
 
 // `BSON_OPTIONAL_PARAM` is a documentation-only macro to document X may be NULL.
@@ -294,13 +321,6 @@
 #endif
 
 
-#if defined(_MSC_VER)
-#define BSON_INLINE __inline
-#else
-#define BSON_INLINE __inline__
-#endif
-
-
 #ifdef _MSC_VER
 #define BSON_ENSURE_ARRAY_PARAM_SIZE(_n)
 #define BSON_TYPEOF decltype

src/libbson/src/bson/bson-timegm.c

@@ -31,14 +31,6 @@
 #define ATTRIBUTE_FORMAT(spec) /* empty */
 #endif
 
-#if !defined _Noreturn && (!defined(__STDC_VERSION__) || __STDC_VERSION__ < 201112)
-#if 2 < __GNUC__ + (8 <= __GNUC_MINOR__)
-#define _Noreturn __attribute__ ((__noreturn__))
-#else
-#define _Noreturn
-#endif
-#endif
-
 #if !defined(__STDC_VERSION__) && !defined restrict
 #define restrict /* empty */
 #endif

src/libmongoc/.gitignore

@@ -34,6 +34,7 @@ example-scram
 example-sdam-monitoring
 example-session
 example-start-at-optime
+example-structured-log
 example-transaction
 example-update
 fam

src/libmongoc/CMakeLists.txt

@@ -648,6 +648,7 @@ set (MONGOC_SOURCES
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-stream-gridfs-download.c
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-stream-gridfs-upload.c
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-stream-socket.c
+   ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-structured-log.c
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-timeout.c
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-topology.c
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-topology-background-monitoring.c
@@ -664,6 +665,7 @@ set (MONGOC_SOURCES
    ${mongo-c-driver_SOURCE_DIR}/src/common/src/common-atomic.c
    ${mongo-c-driver_SOURCE_DIR}/src/common/src/common-b64.c
    ${mongo-c-driver_SOURCE_DIR}/src/common/src/common-md5.c
+   ${mongo-c-driver_SOURCE_DIR}/src/common/src/common-oid.c
    ${mongo-c-driver_SOURCE_DIR}/src/common/src/common-thread.c
 
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-crypto.c
@@ -762,6 +764,7 @@ set (HEADERS
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-stream-file.h
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-stream-gridfs.h
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-stream-socket.h
+   ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-structured-log.h
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-topology-description.h
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-uri.h
    ${PROJECT_SOURCE_DIR}/src/mongoc/mongoc-version-functions.h
@@ -1031,6 +1034,7 @@ endif ()
 set (test-libmongoc-sources
    ${mongo-c-driver_SOURCE_DIR}/src/common/tests/test-common-atomic.c
    ${mongo-c-driver_SOURCE_DIR}/src/common/tests/test-common-cmp.c
+   ${mongo-c-driver_SOURCE_DIR}/src/common/tests/test-common-oid.c
    ${mongo-c-driver_SOURCE_DIR}/src/libbson/tests/corpus-test.c
    ${mongo-c-driver_SOURCE_DIR}/src/libbson/tests/corpus-test.h
    ${mongo-c-driver_SOURCE_DIR}/src/libbson/tests/test-atomic.c
@@ -1141,6 +1145,7 @@ set (test-libmongoc-sources
    ${PROJECT_SOURCE_DIR}/tests/test-mongoc-ssl.c
    ${PROJECT_SOURCE_DIR}/tests/test-mongoc-stream.c
    ${PROJECT_SOURCE_DIR}/tests/test-mongoc-streamable-hello.c
+   ${PROJECT_SOURCE_DIR}/tests/test-mongoc-structured-log.c
    ${PROJECT_SOURCE_DIR}/tests/test-mongoc-thread.c
    ${PROJECT_SOURCE_DIR}/tests/test-mongoc-timeout.c
    ${PROJECT_SOURCE_DIR}/tests/test-mongoc-topology-description.c
@@ -1292,7 +1297,9 @@ if (ENABLE_EXAMPLES AND ENABLE_SHARED)
    mongoc_add_example (example-gridfs ${PROJECT_SOURCE_DIR}/examples/example-gridfs.c)
    mongoc_add_example (example-gridfs-bucket ${PROJECT_SOURCE_DIR}/examples/example-gridfs-bucket.c)
    if (NOT WIN32 AND ENABLE_EXAMPLES)
+      # Examples that use pthreads
       mongoc_add_example (example-pool ${PROJECT_SOURCE_DIR}/examples/example-pool.c)
+      mongoc_add_example (example-structured-log ${PROJECT_SOURCE_DIR}/examples/example-structured-log.c)
    endif ()
    mongoc_add_example (example-scram ${PROJECT_SOURCE_DIR}/examples/example-scram.c)
    mongoc_add_example (example-sdam-monitoring ${PROJECT_SOURCE_DIR}/examples/example-sdam-monitoring.c)