Re: October 2026 stabilization week
- Reply: ShengYi Hung : "Re: October 2026 stabilization week"
- In reply to: Martin Cracauer : "Re: October 2026 stabilization week"
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Tue, 27 Oct 2026 13:09:52 UTC
On Tue, Oct 27, 2026 at 08:39:57AM -0400, Martin Cracauer wrote:
> Konstantin Belousov wrote on Tue, Oct 27, 2026 at 02:33:12PM +0200:
> > On Tue, Oct 27, 2026 at 03:23:28PM +0300, Dima Panov wrote:
> > >
> > >
> > > On 27.10.2026 15:05, Konstantin Belousov wrote:
> > > > On Tue, Oct 27, 2026 at 01:52:43PM +0300, Dima Panov wrote:
> > > > > Hello!
> > > > >
> > > > >
> > > > > Houston, we have a problem!
> > > > >
> > > > > When building latest stabweek world in poudriere on my aarch64 machine, I've got an error in libthr when pkg command is called:
> > > > >
> > > > >
> > > > > ===> Creating FreeBSD-kernel-generic-nodebug-16.snap20261026210619
> > > > > Fatal error 'mutex 0x41814480 own 0x8c935 is on list 0x0 0x409e0008' at line 154 in file /opt/poudriere/jails/16aarch64/usr/src/lib/libthr/thread/thr_mutex.c (errno = 0)
> > > > > Abort trap (core dumped)
> > > > >
> > > > > Host system was a 16.0-CURRENT #0 main-461da47e45e7: Sun Oct 18 02:22:28 MSK 2026
> > > > > Same error on host FreeBSD 16.0-CURRENT #0 main-5250c9890848: Tue Oct 27 00:49:17 MSK 2026
> > > > >
> > > > > But when I revert host to 16.0-CURRENT #0 main-17af098383ba: Fri Sep 18 23:23:19 MSK 2026
> > > > > all builds runs without any errors
> > > > >
> > > > > Looks like some of latest commits between Sept 18 and Oct18 in libthr causes this failure
> > > > >
> > > > > 4472a048cf95c9c1593a8b04655a59096b44b0c9
> > > > > 192781bf5d047cceff3a1947bfc754c8b5eb0835
> > > > > 1d21e67e720a08992e81d626cff9d76524a81f22
> > > > > 627e7d5b72cf967eae3cd9e9e0707ce0ff63a4da
> > > > > 99edcc3bd52614c6cf972ea8424a7807ed3fe1e3
> > > > > de2c0f157a9cce0b763da9ae97ada870fa4d1824
> > > > > 305c3044e0249ea24a9b4571dfa6ae319a5bcdec
> > > > > 5fb07bbcd5d5d9a865d9e7765fb93e9ba5edef1c
> > > > > a0e152973613e0d0d55257f4e1f614509b7ec4c9
> > > > > 894f70a2bd8c7f67f131b76b9ce122a752818af3
> > > >
> > > > What is the program that asserted?
> > >
> > > pkg when generates packages for recently build jail
> > >
> > > > Get the core dump and backtrace.
> > >
> > > /usr/obj/opt/poudriere/jails/16aarch64/usr/src/arm64.aarch64/packages/utilities$ sudo gdb pkg pkg.core
> > > GNU gdb (GDB) 15.1 [GDB v15.1 for FreeBSD]
> > > [skip]
> > > Type "apropos word" to search for commands related to "word"...
> > > Reading symbols from pkg...
> > > Reading symbols from /usr/lib/debug//usr/sbin/pkg.debug...
> > > [New LWP 170934]
> > > Core was generated by `pkg -o ABI=FreeBSD:16:aarch64 -o OSVERSION=1600029 -o ALLOW_BASE_SHLIBS=yes -o S'.
> > > Program terminated with signal SIGABRT, Aborted.
> > > Sent by kill() from pid 45354 and user 0.
> > > #0 0x0000000040fd52f4 in ?? ()
> > > (gdb) bt
> > > #0 0x0000000040fd52f4 in ?? ()
> > > #1 0x00000000002fb784 in ?? ()
> > > Backtrace stopped: previous frame identical to this frame (corrupt stack?)
> > > (gdb)
> >
> > You should build the system libraries with the debug symbols.
> > Otherwise the backtrace is not useful.
>
> I don't think that would have helped in this specific instance. It
> looks like a smashed stack.
Why?
I would believe that unwinder become confused because there is no
proper unwind info, but I am not sure about aarch64.
Anyway, I think the following patch would provide some minimal
useful information, but it is not a substitute for the full
backtrace.
commit cce9267d4c66eee4dd1be28024910944b6511939
Author: Konstantin Belousov <kib@FreeBSD.org>
Date: Tue Oct 27 15:05:59 2026 +0200
libthr/thr_mutex.c: make mutex asserts more useful
Print the line number of the caller where the assert fired.
Sponsored by: The FreeBSD Foundation
MFC after: 1 week
diff --git a/lib/libthr/thread/thr_mutex.c b/lib/libthr/thread/thr_mutex.c
index 9c0939aa6080..9c6f6d8c5dd9 100644
--- a/lib/libthr/thread/thr_mutex.c
+++ b/lib/libthr/thread/thr_mutex.c
@@ -131,41 +131,51 @@ mutex_init_link(struct pthread_mutex *m __unused)
#endif
}
-static void
-mutex_assert_is_owned(struct pthread_mutex *m __unused)
-{
-
#if defined(_PTHREADS_INVARIANTS)
- if (__predict_false(m->m_qe.tqe_prev == NULL))
- PANIC("mutex %p own %#x is not on list %p %p",
+static void
+mutex_assert_is_owned(struct pthread_mutex *m, u_int line)
+{
+ if (__predict_false(m->m_qe.tqe_prev == NULL)) {
+ PANIC("%s:%u: mutex %p own %#x is not on list %p %p",
+ __FILE__, line,
m, m->m_lock.m_owner, m->m_qe.tqe_prev, m->m_qe.tqe_next);
-#endif
+ }
}
static void
-mutex_assert_not_owned(struct pthread *curthread __unused,
- struct pthread_mutex *m __unused)
+mutex_assert_not_owned(struct pthread_mutex *m, u_int line)
{
+ struct pthread *curthread;
-#if defined(_PTHREADS_INVARIANTS)
+ curthread = _get_curthread();
if (__predict_false(m->m_qe.tqe_prev != NULL ||
- m->m_qe.tqe_next != NULL))
- PANIC("mutex %p own %#x is on list %p %p",
+ m->m_qe.tqe_next != NULL)) {
+ PANIC("%s:%u: mutex %p own %#x is on list %p %p",
+ __FILE__, line,
m, m->m_lock.m_owner, m->m_qe.tqe_prev, m->m_qe.tqe_next);
+ }
if (__predict_false(is_robust_mutex(m) &&
(m->m_lock.m_rb_lnk != 0 || m->m_rb_prev != NULL ||
(is_pshared_mutex(m) && curthread->robust_list ==
(uintptr_t)&m->m_lock) ||
(!is_pshared_mutex(m) && curthread->priv_robust_list ==
- (uintptr_t)&m->m_lock))))
+ (uintptr_t)&m->m_lock)))) {
PANIC(
- "mutex %p own %#x is on robust linkage %p %p head %p phead %p",
+ "%s:%u: mutex %p own %#x is on robust linkage %p %p head %p phead %p",
+ __FILE__, line,
m, m->m_lock.m_owner, (void *)m->m_lock.m_rb_lnk,
m->m_rb_prev, (void *)curthread->robust_list,
(void *)curthread->priv_robust_list);
-#endif
+ }
}
+#define MUTEX_ASSERT_IS_OWNED(m) mutex_assert_is_owned(m, __LINE__)
+#define MUTEX_ASSERT_NOT_OWNED(m) mutex_assert_not_owned(m, __LINE__)
+#else
+#define MUTEX_ASSERT_IS_OWNED(m) do {} while (0)
+#define MUTEX_ASSERT_NOT_OWNED(m) do {} while (0)
+#endif
+
static bool
is_pshared_mutex(struct pthread_mutex *m)
{
@@ -477,8 +487,7 @@ _thr_mutex_destroy(pthread_mutex_t *mutex)
if (m1 != NULL) {
if ((uint32_t)m1->m_lock.m_owner !=
UMUTEX_RB_OWNERDEAD) {
- mutex_assert_not_owned(
- _get_curthread(), m1);
+ MUTEX_ASSERT_NOT_OWNED(m1);
}
__thr_pshared_destroy(mutex);
}
@@ -490,7 +499,7 @@ _thr_mutex_destroy(pthread_mutex_t *mutex)
ret = EBUSY;
} else {
*mutex = THR_MUTEX_DESTROYED;
- mutex_assert_not_owned(_get_curthread(), m);
+ MUTEX_ASSERT_NOT_OWNED(m);
__thr_free(m);
ret = 0;
}
@@ -530,7 +539,7 @@ enqueue_mutex(struct pthread *curthread, struct pthread_mutex *m,
/* Add to the list of owned mutexes: */
if (error != EOWNERDEAD)
- mutex_assert_not_owned(curthread, m);
+ MUTEX_ASSERT_NOT_OWNED(m);
qidx = mutex_qidx(m);
TAILQ_INSERT_TAIL(&curthread->mq[qidx], m, m_qe);
if (!is_pshared_mutex(m))
@@ -558,7 +567,7 @@ dequeue_mutex(struct pthread *curthread, struct pthread_mutex *m)
struct pthread_mutex *mp, *mn;
int qidx;
- mutex_assert_is_owned(m);
+ MUTEX_ASSERT_IS_OWNED(m);
qidx = mutex_qidx(m);
if (is_robust_mutex(m)) {
mp = m->m_rb_prev;
@@ -1054,7 +1063,7 @@ _pthread_mutex_setprioceiling(pthread_mutex_t * __restrict mutex,
curthread = _get_curthread();
if (PMUTEX_OWNER_ID(m) == TID(curthread)) {
- mutex_assert_is_owned(m);
+ MUTEX_ASSERT_IS_OWNED(m);
m1 = TAILQ_PREV(m, mutex_queue, m_qe);
m2 = TAILQ_NEXT(m, m_qe);
if ((m1 != NULL && m1->m_lock.m_ceilings[0] > (u_int)ceiling) ||