git: 0a2fa7bb9cbb - stable/14 - crashinfo: Print stack traces for all on-CPU threads

From: Mark Johnston <markj_at_FreeBSD.org>
Date: Fri, 09 Feb 2024 15:04:22 UTC
The branch stable/14 has been updated by markj:

URL: https://cgit.FreeBSD.org/src/commit/?id=0a2fa7bb9cbb788cfe07db11071712af7cdcea93

commit 0a2fa7bb9cbb788cfe07db11071712af7cdcea93
Author:     Mark Johnston <markj@FreeBSD.org>
AuthorDate: 2024-01-15 20:39:26 +0000
Commit:     Mark Johnston <markj@FreeBSD.org>
CommitDate: 2024-02-09 14:55:45 +0000

    crashinfo: Print stack traces for all on-CPU threads
    
    Add a python script which implements the bulk of this functionality.
    Over time, this would ideally evolve into a library of python routines
    which can be used to inspect kernel data structures and automate some
    debugging tasks, similar to jhb's out-of-tree scripts, but written in a
    somewhat nicer language and with better integration into the kgdb
    command prompt.
    
    Note that kgdb currently won't auto-load scripts in this directory.
    This should perhaps change in the future.  It probably also makes more
    sense to have a crashinfo.py which provides all the kgdb output that we
    want to include in core.txt, rather than having crashinfo.sh pipe in
    several commands.
    
    Reviewed by:    avg, imp
    Discussed with: jhb
    MFC after:      3 weeks
    Sponsored by:   The FreeBSD Foundation
    Differential Revision:  https://reviews.freebsd.org/D33817
    
    (cherry picked from commit 2524b7dfb0df7e37ea9651559ce6c027b006e2a9)
---
 etc/mtree/BSD.usr.dist          |  2 ++
 libexec/Makefile                |  1 +
 libexec/kgdb/Makefile           |  5 ++++
 libexec/kgdb/acttrace.py        | 63 +++++++++++++++++++++++++++++++++++++++++
 usr.sbin/crashinfo/crashinfo.sh |  8 ++++--
 5 files changed, 76 insertions(+), 3 deletions(-)

diff --git a/etc/mtree/BSD.usr.dist b/etc/mtree/BSD.usr.dist
index 83d8de034caf..a7738aaf6f78 100644
--- a/etc/mtree/BSD.usr.dist
+++ b/etc/mtree/BSD.usr.dist
@@ -161,6 +161,8 @@
         ..
         hyperv
         ..
+        kgdb
+        ..
         lpr
             ru
             ..
diff --git a/libexec/Makefile b/libexec/Makefile
index 795250dddfdb..bd1571edcfe4 100644
--- a/libexec/Makefile
+++ b/libexec/Makefile
@@ -12,6 +12,7 @@ SUBDIR=	${_atf} \
 	flua \
 	getty \
 	${_hyperv} \
+	kgdb \
 	${_mail.local} \
 	${_makewhatis.local} \
 	${_mknetid} \
diff --git a/libexec/kgdb/Makefile b/libexec/kgdb/Makefile
new file mode 100644
index 000000000000..f6b255ab4f60
--- /dev/null
+++ b/libexec/kgdb/Makefile
@@ -0,0 +1,5 @@
+FILESDIR?= /usr/libexec/kgdb
+
+FILES= acttrace.py
+
+.include <bsd.prog.mk>
diff --git a/libexec/kgdb/acttrace.py b/libexec/kgdb/acttrace.py
new file mode 100644
index 000000000000..3229ff708de1
--- /dev/null
+++ b/libexec/kgdb/acttrace.py
@@ -0,0 +1,63 @@
+#-
+# Copyright (c) 2022 The FreeBSD Foundation
+#
+# This software was developed by Mark Johnston under sponsorship from the
+# FreeBSD Foundation.
+#
+
+import gdb
+
+
+def symval(name):
+    return gdb.lookup_global_symbol(name).value()
+
+
+def tid_to_gdb_thread(tid):
+    for thread in gdb.inferiors()[0].threads():
+        if thread.ptid[2] == tid:
+            return thread
+    else:
+        return None
+
+
+def all_pcpus():
+    mp_maxid = symval("mp_maxid")
+    cpuid_to_pcpu = symval("cpuid_to_pcpu")
+
+    cpu = 0
+    while cpu <= mp_maxid:
+        pcpu = cpuid_to_pcpu[cpu]
+        if pcpu:
+            yield pcpu
+        cpu = cpu + 1
+
+
+class acttrace(gdb.Command):
+    def __init__(self):
+        super(acttrace, self).__init__("acttrace", gdb.COMMAND_USER)
+
+    def invoke(self, arg, from_tty):
+        # Save the current thread so that we can switch back after.
+        curthread = gdb.selected_thread()
+
+        for pcpu in all_pcpus():
+            td = pcpu['pc_curthread']
+            tid = td['td_tid']
+
+            gdb_thread = tid_to_gdb_thread(tid)
+            if gdb_thread is None:
+                print("failed to find GDB thread with TID {}".format(tid))
+            else:
+                gdb_thread.switch()
+
+                p = td['td_proc']
+                print("Tracing command {} pid {} tid {} (CPU {})".format(
+                      p['p_comm'], p['p_pid'], td['td_tid'], pcpu['pc_cpuid']))
+                gdb.execute("bt")
+                print()
+
+        curthread.switch()
+
+
+# Registers the command with gdb, doesn't do anything.
+acttrace()
diff --git a/usr.sbin/crashinfo/crashinfo.sh b/usr.sbin/crashinfo/crashinfo.sh
index 9a3d26020654..3bb1e1456462 100755
--- a/usr.sbin/crashinfo/crashinfo.sh
+++ b/usr.sbin/crashinfo/crashinfo.sh
@@ -215,13 +215,15 @@ echo
 sed -ne '/^  Panic String: /{s//panic: /;p;}' $INFO
 echo
 
-# XXX: /bin/sh on 7.0+ is broken so we can't simply pipe the commands to
-# kgdb via stdin and have to use a temporary file instead.
 file=`mktemp /tmp/crashinfo.XXXXXX`
 if [ $? -eq 0 ]; then
+	scriptdir=/usr/libexec/kgdb
+
 	echo "bt -full" >> $file
+	echo "source ${scriptdir}/acttrace.py" >> $file
+	echo "acttrace" >> $file
 	echo "quit" >> $file
-	${GDB%gdb}kgdb $KERNEL $VMCORE < $file
+	${GDB%gdb}kgdb -q $KERNEL $VMCORE < $file
 	rm -f $file
 	echo
 fi