implement launching dmesg with tail (#438)

* implement launching dmesg with tail

* more asan timeout

* add linux core directory detection; make next job launch delay depend on job parallelity; combine info prints to one

* ignore python caches

* fix merge issues, loop counting

* add flag to disable report tarball generation

* implement writing dmesg to report file

* increase timeout

* allow dmesg access

* python - no $

Author: dothebart

.gitignore

@@ -11,3 +11,5 @@ release
 .config
 .local
 .idea
+__pycache__
+

helper.linux.fish

@@ -1693,7 +1693,7 @@ function runInContainer
   # from a regular user. Therefore we have to do some Eiertanz to stop it
   # if we receive a TERM outside the container. Note that this does not
   # cover SIGINT, since this will directly abort the whole function.
-  set c (docker run -d --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \
+  set c (docker run -d --cap-add=SYS_PTRACE --privileged --security-opt seccomp=unconfined \
              -v $WORKDIR/work/:$INNERWORKDIR \
              -v $SSH_AUTH_SOCK:/ssh-agent \
              -v "$WORKDIR/jenkins/helper":"$WORKSPACE/jenkins/helper" \
@@ -1806,7 +1806,7 @@ function interactiveContainer
     set -l agentstarted ""
   end
 
-  docker run -it --rm --cap-add=SYS_PTRACE --security-opt seccomp=unconfined \
+  docker run -it --rm --cap-add=SYS_PTRACE --privileged --security-opt seccomp=unconfined \
     -v $WORKDIR/work:$INNERWORKDIR \
     -v $SSH_AUTH_SOCK:/ssh-agent \
     -v "$WORKDIR/jenkins/helper":"$WORKSPACE/jenkins/helper" \

jenkins/helper/async_client.py

@@ -52,6 +52,42 @@ def make_default_params(verbose):
         "identifier": ""
     }
 
+def tail_line_result(wait, line, params):
+    """
+    Keep the line, filter it for leading #,
+    if verbose print the line. else print progress.
+    """
+    # pylint: disable=pointless-statement
+    if params['skip_done']:
+        if isinstance(line, tuple):
+            print(params['prefix'] + str(line[0], 'utf-8').rstrip())
+            params['output'].write(line[0])
+        return True
+    now = datetime.now()
+    if now - params['last_read'] > timedelta(seconds=1):
+        params['skip_done'] = True
+        print(params['prefix'] + 'initial tail done, starting to output')
+    return True
+def make_tail_params(verbose, prefix, logfile):
+    """ create the structure to work with arrays to output the strings to """
+    return {
+        "trace_io": False,
+        "error": "",
+        "verbose": verbose,
+        "output": logfile.open("wb"),
+        "lfn": str(logfile),
+        "identifier": "",
+        "skip_done": False,
+        "prefix": prefix,
+        "last_read": datetime.now()
+    }
+def delete_tail_params(params):
+    """ teardown the structure to work with logfiles """
+    print(f"{params['identifier']} closing {params['lfn']}")
+    params['output'].flush()
+    params['output'].close()
+    print(f"{params['identifier']} {params['lfn']} closed")
+
 def make_logfile_params(verbose, logfile, trace):
     """ create the structure to work with logfiles """
     return {
@@ -214,6 +250,7 @@ def __init__(self, config, connect_instance, deadline_signal=-1):
         self.connect_instance = connect_instance
         self.cfg = config
         self.deadline_signal = deadline_signal
+        self.pid = None
         if self.deadline_signal == -1:
             # pylint: disable=no-member
             # yes, one is only there on the wintendo, the other one elsewhere.
@@ -315,6 +352,8 @@ def run_monitored(self,
             close_fds=ON_POSIX,
             cwd=self.cfg.test_data_dir.resolve(),
         ) as process:
+            # pylint: disable=consider-using-f-string
+            self.pid = process.pid
             queue = Queue()
             thread1 = Thread(
                 name=f"readIO {identifier}",
@@ -381,6 +420,7 @@ def run_monitored(self,
                 if datetime.now() > deadline:
                     have_deadline += 1
                 if have_deadline == 1:
+                    # pylint: disable=line-too-long
                     add_message_to_report(
                         params,
                         f"{identifier} Execution Deadline reached - will trigger signal {self.deadline_signal}!")
@@ -406,6 +446,7 @@ def run_monitored(self,
                         process.stdout.close()
                         break
                     except psutil.TimeoutExpired:
+                        # pylint: disable=line-too-long
                         deadline_grace_count += 1
                         print_log(f"{identifier} timeout waiting for exit {str(deadline_grace_count)}", params)
                         # if its not willing, use force: