macOS debuginfo: add support for using atos to get file:line info in backtraces. (#4291)

* macOS debuginfo: add support for using `atos` to get file:line info in backtraces.
Resolves issue #3864

* Return to LLVM default of emitting __debug_line section in __DWARF segment as a debug section (not regular)

* Re-enable test now that issue #3280 is fixed.

* Allow full path names in exception file:line debug output.

* Allow full path names in exception file:line debug output.

* Add changelog entry

Author: JohanEngelen

.github/workflows/supported_llvm_versions.yml

@@ -147,8 +147,6 @@ jobs:
           excludes='dmd-testsuite|lit-tests|ldc2-unittest'
           if [[ '${{ runner.os }}' == macOS ]]; then
             N=$(sysctl -n hw.logicalcpu)
-            # https://github.com/ldc-developers/ldc/issues/3280
-            excludes+='|druntime-test-exceptions-debug'
           else
             N=$(nproc)
           fi

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 #### Big news
 - Bit fields support. (#4015)
+- macOS on Apple M1: linking with `-g` is working again without unaligned pointer warnings/errors. This fixes file:line debug information in exception backtraces (requiring `atos`, a macOS development tool installed with Xcode), without the need to set MACOSX_DEPLOYMENT_TARGET=11 and using a modified LLVM. (#4291)
 
 #### Platform support
 - Initial ABI support for 64-bit RISC-V. (#4007)

driver/targetmachine.cpp

@@ -46,14 +46,15 @@
 // LDC-LLVM >= 6.0.1:
 // On Mac, emit __debug_line section in __DWARF segment as regular (non-debug)
 // section, like DMD, to enable file/line infos in backtraces. See
-// https://github.com/dlang/dmd/commit/2bf7d0db29416eacbb01a91e6502140e354ee0ef.
+// https://github.com/dlang/dmd/commit/2bf7d0db29416eacbb01a91e6502140e354ee0ef
+// https://github.com/ldc-developers/llvm-project/commit/110deda1bc1cf195983fea8c1107886057987955
 static llvm::cl::opt<bool, true> preserveDwarfLineSection(
     "preserve-dwarf-line-section",
     llvm::cl::desc("Mac: preserve DWARF line section during linking for "
                    "file/line infos in backtraces. Defaults to true."),
     llvm::cl::Hidden, llvm::cl::ZeroOrMore,
     llvm::cl::location(ldc::emitMachODwarfLineAsRegularSection),
-    llvm::cl::init(true));
+    llvm::cl::init(false));
 #endif
 
 static const char *getABI(const llvm::Triple &triple) {

runtime/druntime/src/core/internal/backtrace/dwarf.d

@@ -250,6 +250,11 @@ int processCallstack(Location[] locations, const(ubyte)[] debugLineSectionData,
 {
     if (debugLineSectionData)
         resolveAddresses(debugLineSectionData, locations, baseAddress);
+    version (Darwin)
+    {
+        if (!debugLineSectionData)
+            resolveAddressesWithAtos(locations);
+    }
 
     TraceInfoBuffer buffer;
     foreach (idx, const ref loc; locations)
@@ -268,6 +273,163 @@ int processCallstack(Location[] locations, const(ubyte)[] debugLineSectionData,
     return 0;
 }
 
+version (Darwin)
+{
+    /**
+     * Resolve the addresses of `locations` using `atos` (executable that ships with XCode)
+     *
+     * Spawns a child process that calls `atos`. Communication is through stdin/stdout pipes.
+     *
+     * After this function successfully completes, `locations` will contain
+     * file / lines informations.
+     *
+     * The lifetime of the `Location` data surpases function return (strndup is used).
+     *
+     * Params:
+     *   locations = The locations to resolve
+     */
+    private void resolveAddressesWithAtos(Location[] locations) @nogc nothrow
+    {
+        import core.stdc.stdio : fclose, fflush, fgets, fprintf, printf, snprintf;
+        import core.stdc.stdlib : exit;
+        import core.sys.posix.stdio : fdopen;
+        import core.sys.posix.unistd : close, dup2, execlp, fork, getpid, pipe;
+
+        // Create in/out pipes to communicate with the forked exec
+        int[2] dummy_pipes; // these dummy pipes are there to prevent funny issues when stdin/stdout is closed and pipe returns id 0 or 1
+        int[2] pipes_to_atos;
+        int[2] pipes_from_atos;
+        if ( pipe(dummy_pipes) < 0 || pipe(pipes_to_atos) < 0 || pipe(pipes_from_atos) < 0 ) {
+            printf("some pipe creation error!\n");
+            return;
+        }
+        close(dummy_pipes[0]);
+        close(dummy_pipes[1]);
+        auto write_to_atos = pipes_to_atos[1];
+        auto read_from_atos = pipes_from_atos[0];
+        auto atos_stdin = pipes_to_atos[0];
+        auto atos_stdout = pipes_from_atos[1];
+
+        auto self_pid = cast(int) getpid();
+
+        // Spawn a child process that calls atos, reads/writes from the pipes, and then exits.
+        auto child_id = fork();
+        if (child_id == -1)
+        {
+            printf("some fork error!\n");
+            return;
+        }
+        else if (child_id == 0)
+        {
+            // We are in the child process, spawn atos and link pipes
+
+            // Close unused read/write ends of pipes
+            close(write_to_atos);
+            close(read_from_atos);
+
+            // Link pipes to stdin/stdout
+            dup2(atos_stdin,  0);
+            close(atos_stdin);
+            dup2(atos_stdout, 1);
+            close(atos_stdout);
+
+            char[10] pid_str;
+            snprintf(pid_str.ptr, pid_str.sizeof, "%d", cast(int) self_pid);
+            const(char)* atos_executable = "atos";
+            const(char)* atos_p_arg = "-p";
+            const(char)* atos_fullpath_arg = "-fullPath";
+            execlp(atos_executable, atos_executable, atos_fullpath_arg, atos_p_arg, pid_str.ptr, null);
+
+            // If exec returns, an error occurred, need to exit the forked process here.
+            printf("some exec error!\n");
+            exit(0);
+        }
+        // Parent process just continues from here.
+
+        // Close unused pipes
+        close(atos_stdin);
+        close(atos_stdout);
+
+        auto to_atos = fdopen(write_to_atos, "w");
+        auto from_atos = fdopen(read_from_atos, "r");
+
+        // buffer for atos reading. Note that symbol names can be super large...
+        static char[16 * 1024] read_buffer = void;
+
+        char* status_ptr = null;
+        foreach (ref loc; locations)
+        {
+            fprintf(to_atos, "%p\n", loc.address);
+            fflush(to_atos);
+            read_buffer[0] = '\0';
+            status_ptr = fgets(read_buffer.ptr, read_buffer.sizeof, from_atos);
+            if (!status_ptr)
+                break;
+            Location parsed_loc = parseAtosLine(read_buffer.ptr);
+            if (parsed_loc.line != -1)
+            {
+                // Only update the file:line info, keep the procedure name as found before (preserving the standard truncation).
+                loc.file = parsed_loc.file;
+                loc.line = parsed_loc.line;
+            }
+        }
+
+        if (!status_ptr)
+            printf("\nDid not succeed in using 'atos' for extra debug information.\n");
+
+        fclose(to_atos);
+        fclose(from_atos);
+        close(write_to_atos);
+        close(read_from_atos);
+    }
+
+    private Location parseAtosLine(char* buffer) @nogc nothrow
+    {
+        // The line from `atos` is in one of these formats:
+        //   myfunction (in library.dylib) (sourcefile.c:17)
+        //   myfunction (in library.dylib) + 0x1fe
+        //   myfunction (in library.dylib) + 15
+        //   0xdeadbeef (in library.dylib) + 0x1fe
+        //   0xdeadbeef (in library.dylib) + 15
+        //   0xdeadbeef (in library.dylib)
+        //   0xdeadbeef
+
+        import core.stdc.stdlib : atoi;
+        import core.stdc.string : strchr, strstr;
+        import core.sys.posix.string : strndup;
+
+        Location loc;
+
+        if (!buffer)
+            return loc;
+        if (buffer[0] == '0' && buffer[1] == 'x')
+            // no named symbol found
+            return loc;
+
+        const symbolname_end = strstr(buffer, " (in ");
+        if (!symbolname_end)
+            return loc;
+        const symbolname_size = symbolname_end - buffer;
+        loc.procedure = strndup(buffer, symbolname_size)[0..symbolname_size];
+
+        const filename_start = strstr(symbolname_end, ") (") + 3;
+        if (cast(size_t)filename_start < 4)
+            return loc;
+        const colon_location = strchr(filename_start, ':');
+        if (!colon_location)
+            return loc;
+        const filename_size = colon_location - filename_start;
+        loc.file = strndup(filename_start, filename_size)[0..filename_size];
+
+        const final_paren = strchr(colon_location+1, ')');
+        if (!final_paren)
+            return loc;
+        loc.line = atoi(colon_location+1);
+
+        return loc;
+    }
+}
+
 /**
  * Resolve the addresses of `locations` using `debugLineSectionData`
  *

runtime/druntime/test/exceptions/Makefile

@@ -39,7 +39,7 @@ $(ROOT)/line_trace.done: $(ROOT)/line_trace
 	@echo Testing line_trace
 	$(QUIET)$(TIMELIMIT)$(ROOT)/line_trace $(RUN_ARGS) > $(ROOT)/line_trace.output
 	# Use sed to canonicalize line_trace.output and compare against expected output in line_trace.exp
-	$(QUIET)$(SED) "s/\[0x[0-9a-f]*\]/\[ADDR\]/g; s/scope //g; s/Nl//g" $(ROOT)/line_trace.output | $(DIFF) line_trace.exp -
+	$(QUIET)$(SED) "s|^.*/src/|src/|g; s/\[0x[0-9a-f]*\]/\[ADDR\]/g; s/scope //g; s/Nl//g" $(ROOT)/line_trace.output | $(DIFF) line_trace.exp -
 	@rm -f $(ROOT)/line_trace.output
 	@touch $@
 
@@ -49,15 +49,15 @@ $(ROOT)/line_trace_21656.done: $(ROOT)/line_trace
 	@mkdir -p $(ROOT)/line_trace_21656
 	@touch $(ROOT)/line_trace_21656/line_trace
 	$(QUIET)cd $(ROOT)/line_trace_21656 && PATH="..:$$PATH" $(TIMELIMIT)line_trace $(RUN_ARGS) > line_trace.output
-	$(QUIET)$(SED) "s/\[0x[0-9a-f]*\]/\[ADDR\]/g; s/scope //g; s/Nl//g" $(ROOT)/line_trace_21656/line_trace.output | $(DIFF) line_trace.exp -
+	$(QUIET)$(SED) "s|^.*/src/|src/|g; s/\[0x[0-9a-f]*\]/\[ADDR\]/g; s/scope //g; s/Nl//g" $(ROOT)/line_trace_21656/line_trace.output | $(DIFF) line_trace.exp -
 	@rm -rf $(ROOT)/line_trace_21656
 	@touch $@
 
 $(ROOT)/long_backtrace_trunc.done: $(ROOT)/long_backtrace_trunc
 	@echo Testing long_backtrace_trunc
 	$(QUIET)$(TIMELIMIT)$(ROOT)/long_backtrace_trunc $(RUN_ARGS) > $(ROOT)/long_backtrace_trunc.output
 	# Use sed to canonicalize long_backtrace_trunc.output and compare against expected output in long_backtrace_trunc.exp
-	$(QUIET)$(SED) "s/\[0x[0-9a-f]*\]/\[ADDR\]/g; s/scope //g; s/Nl//g" $(ROOT)/long_backtrace_trunc.output | $(DIFF) long_backtrace_trunc.exp -
+	$(QUIET)$(SED) "s|^.*/src/|src/|g; s/\[0x[0-9a-f]*\]/\[ADDR\]/g; s/scope //g; s/Nl//g; s|.F.*F.|.|g; s|.G.*G.|.|g" $(ROOT)/long_backtrace_trunc.output | $(DIFF) long_backtrace_trunc.exp -
 	@rm -f $(ROOT)/long_backtrace_trunc.output
 	@touch $@
 

runtime/druntime/test/exceptions/long_backtrace_trunc.exp

@@ -1,4 +1,4 @@
 object.Exception@src/long_backtrace_trunc.d(10): test
 ----------------
-src/long_backtrace_trunc.d:10 pure @safe int long_backtrace_trunc.AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA!(int).AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA.BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB.CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC.DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD.EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE.FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF.GGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG...
+src/long_backtrace_trunc.d:10 pure @safe int long_backtrace_trunc.AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA!(int).AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA.BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB.CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC.DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD.EEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEEE...
 src/long_backtrace_trunc.d:31 _Dmain [ADDR]