Add documentation on logging formats

---------

Co-authored-by: Ryan McCormick <[email protected]>

Author: nnshah1

docs/protocol/extension_logging.md

@@ -69,7 +69,7 @@ the `$string` representation of the log setting and the “value” is a `$strin
 `$bool`, or `$number` representation of the setting value. Currently, the
 following log settings are defined:
 
-- "log_file" : a `$string` parameter defining the file where the log outputs will be saved. If an empty string is specified, log outputs will stream to the console.
+- "log_file" : a `$string` log file location where the log outputs will be saved. If empty, log outputs are streamed to the console.
 
 - "log_info" : a `$boolean` parameter that controls whether the Triton server logs INFO level messages.
 
@@ -117,8 +117,14 @@ $log_setting_request =
 }
 ```
 
-When a `$log_setting` JSON is received (defined above), only the specified
-settings will be updated.
+When a `$log_setting` JSON is received (defined above), only the
+specified settings will be updated. Currently, the following log
+settings (described above) can be updated:
+- "log_info"
+- "log_warning"
+- "log_error"
+- "log_verbose_level"
+- "log_format"
 
 ### Example Usage
 The logging protocol extension can be invoked using the curl library in the following manner (assuming
@@ -196,3 +202,74 @@ message LogSettingsResponse
   map<string, SettingValue> settings = 1;
 }
 ```
+
+## Logging Formats
+
+The logging extension offers two logging formats. The formats have a
+common set of fields but differ in how the timestamp for a log entry
+is represented. Messages are serialized according to JSON encoding
+rules by default. This behavior can be disabled by setting the
+environment variable TRITON_SERVER_ESCAPE_LOG_MESSAGES to "0" when
+launching the server but can not be changed through the logging
+extension.
+
+Log entries can be single-line or multi-line. Multi-line entries have
+a single optional heading followed by the structured representation of
+an object such as a table or protobuf message. Multi-line entries end
+when the next log entry begins.
+
+1. TRITONSERVER_LOG_DEFAULT
+
+### Single-line Entry
+```
+<level><month><day><hour>:<min>:<sec>.<usec> <pid> <file>:<line>] <message>
+```
+Example:
+```
+I0520 20:03:25.829575 3355 model_lifecycle.cc:441] "AsyncLoad() 'simple'"
+```
+### Multi-line Entry
+```
+<level><month><day><hour>:<min>:<sec>.<usec> <pid> <file>:<line>] <heading>
+<object>
+```
+Example:
+
+```
+I0520 20:03:25.912303 3355 server.cc:676]
++--------+---------+--------+
+| Model  | Version | Status |
++--------+---------+--------+
+| simple | 1       | READY  |
++--------+---------+--------+
+```
+
+
+2. TRITONSERVER_LOG_ISO8601
+
+### Single-line Entry
+```
+<year>-<month>-<day>T<hour>:<min>:<sec>Z <level> <pid> <file>:<line>] <message>
+```
+
+Example:
+```
+2024-05-20T20:03:26Z I 3415 model_lifecycle.cc:441] "AsyncLoad() 'simple'"
+```
+
+### Multi-line Entry
+```
+<year>-<month>-<day>T<hour>:<min>:<sec>Z <level> <pid> <file>:<line>] <heading>
+<object>
+```
+
+Example:
+
+```
+2024-05-20T20:03:26Z I 3415 server.cc:676]
++--------+---------+--------+
+| Model  | Version | Status |
++--------+---------+--------+
+| simple | 1       | READY  |
++--------+---------+--------+
+```

qa/L0_logging/log_format_test.py

@@ -280,6 +280,51 @@ def validate_table(table_rows):
 
 @validator
 def validate_message(message, escaped):
+    """message field validator
+
+    Messages can be single line or multi-line. In the multi-line case
+    messages have the form:
+
+    <heading>\n
+    <object>
+
+    Where heading is an optional string (escaped with normal escaping
+    rules) and object is a structured representation of an object such
+    as a table or protobuf. The only objects currently allowed are:
+
+    * Tables (triton::common::table_printer)
+
+    * Model config protobuf messages
+
+
+
+    Parameters
+    ----------
+    message : str
+        message portion of log record (may be multiple lines)
+    escaped : bool
+        whether the message is escaped
+
+    Raises
+    ------
+    Exception If message is expected to be escaped but is not
+    or object doesn't match formatting
+
+    Examples
+    --------
+
+    validate_message("foo",escaped=True) -> Exception
+    validate_message('"foo"', escaped=True) -> pass
+    validate_message('"foo"\nfoo',escaped=True) -> Exception
+    validate_message('"foo"\n+--------+---------+--------+\n' \
+                     '| Model  | Version | Status |\n' \
+                     '+--------+---------+--------+\n' \
+                     '| simple | 1       | READY  |\n' \
+                     '+--------+---------+--------+',
+                      escaped=True) -> pass
+
+    """
+
     split_message = message.split("\n")
     heading = split_message[0]
     obj = split_message[1:] if len(split_message) > 1 else []
@@ -300,7 +345,7 @@ def validate_message(message, escaped):
         elif escaped:
             validate_protobuf(obj)
         else:
-            # if not escaped we can't
+            # if not escaped and not table we can't
             # guarantee why type of object is present
             pass
 
@@ -315,9 +360,7 @@ def _setup(self, request):
         self._server_options["log-error"] = 1
         self._server_options["log-warning"] = 1
         self._server_options["log-format"] = "default"
-        self._server_options["model-repository"] = os.path.abspath(
-            os.path.join(module_directory, "log_models")
-        )
+        self._server_options["model-repository"] = test_model_directory
         self._server_process = None
         self._server_options["log-file"] = os.path.join(
             test_logs_directory, test_case_name + ".server.log"
@@ -436,11 +479,22 @@ def test_injection(self, log_format, format_regex, injected_record):
             triton_client = httpclient.InferenceServerClient(
                 url="localhost:8000", verbose=False
             )
-            while not triton_client.is_server_ready():
+
+            # TODO Refactor server launch, shutdown into reusable class
+            wait_time = 10
+
+            while wait_time and not triton_client.is_server_ready():
                 time.sleep(1)
+                wait_time -= 1
 
-            while not triton_client.is_model_ready("simple"):
+            while wait_time and not triton_client.is_model_ready("simple"):
                 time.sleep(1)
+                wait_time -= 1
+
+            if not triton_client.is_server_ready() or not triton_client.is_model_ready(
+                "simple"
+            ):
+                raise Exception("Model or Server not Ready")
 
         except Exception as e:
             self._shutdown_server()