Re: October 2026 stabilization week
- Reply: Dima Panov : "Re: October 2026 stabilization week"
- In reply to: Konstantin Belousov : "Re: October 2026 stabilization week"
- Go to: [ bottom of page ] [ top of archives ] [ this month ]
Date: Tue, 27 Oct 2026 16:47:56 UTC
I took a look at all my libthr changes and didn't observe any
possibility to lead this. A debug build is definitely helpful.
On 10/27/26 08:09, Konstantin Belousov wrote:
> On Tue, Oct 27, 2026 at 08:39:57AM -0400, Martin Cracauer wrote:
>> Konstantin Belousov wrote on Tue, Oct 27, 2026 at 02:33:12PM +0200:
>>> On Tue, Oct 27, 2026 at 03:23:28PM +0300, Dima Panov wrote:
>>>>
>>>> On 27.10.2026 15:05, Konstantin Belousov wrote:
>>>>> On Tue, Oct 27, 2026 at 01:52:43PM +0300, Dima Panov wrote:
>>>>>> Hello!
>>>>>>
>>>>>>
>>>>>> Houston, we have a problem!
>>>>>>
>>>>>> When building latest stabweek world in poudriere on my aarch64 machine, I've got an error in libthr when pkg command is called:
>>>>>>
>>>>>>
>>>>>> ===> Creating FreeBSD-kernel-generic-nodebug-16.snap20261026210619
>>>>>> Fatal error 'mutex 0x41814480 own 0x8c935 is on list 0x0 0x409e0008' at line 154 in file /opt/poudriere/jails/16aarch64/usr/src/lib/libthr/thread/thr_mutex.c (errno = 0)
>>>>>> Abort trap (core dumped)
>>>>>>
>>>>>> Host system was a 16.0-CURRENT #0 main-461da47e45e7: Sun Oct 18 02:22:28 MSK 2026
>>>>>> Same error on host FreeBSD 16.0-CURRENT #0 main-5250c9890848: Tue Oct 27 00:49:17 MSK 2026
>>>>>>
>>>>>> But when I revert host to 16.0-CURRENT #0 main-17af098383ba: Fri Sep 18 23:23:19 MSK 2026
>>>>>> all builds runs without any errors
>>>>>>
>>>>>> Looks like some of latest commits between Sept 18 and Oct18 in libthr causes this failure
>>>>>>
>>>>>> 4472a048cf95c9c1593a8b04655a59096b44b0c9
>>>>>> 192781bf5d047cceff3a1947bfc754c8b5eb0835
>>>>>> 1d21e67e720a08992e81d626cff9d76524a81f22
>>>>>> 627e7d5b72cf967eae3cd9e9e0707ce0ff63a4da
>>>>>> 99edcc3bd52614c6cf972ea8424a7807ed3fe1e3
>>>>>> de2c0f157a9cce0b763da9ae97ada870fa4d1824
>>>>>> 305c3044e0249ea24a9b4571dfa6ae319a5bcdec
>>>>>> 5fb07bbcd5d5d9a865d9e7765fb93e9ba5edef1c
>>>>>> a0e152973613e0d0d55257f4e1f614509b7ec4c9
>>>>>> 894f70a2bd8c7f67f131b76b9ce122a752818af3
>>>>> What is the program that asserted?
>>>> pkg when generates packages for recently build jail
>>>>
>>>>> Get the core dump and backtrace.
>>>> /usr/obj/opt/poudriere/jails/16aarch64/usr/src/arm64.aarch64/packages/utilities$ sudo gdb pkg pkg.core
>>>> GNU gdb (GDB) 15.1 [GDB v15.1 for FreeBSD]
>>>> [skip]
>>>> Type "apropos word" to search for commands related to "word"...
>>>> Reading symbols from pkg...
>>>> Reading symbols from /usr/lib/debug//usr/sbin/pkg.debug...
>>>> [New LWP 170934]
>>>> Core was generated by `pkg -o ABI=FreeBSD:16:aarch64 -o OSVERSION=1600029 -o ALLOW_BASE_SHLIBS=yes -o S'.
>>>> Program terminated with signal SIGABRT, Aborted.
>>>> Sent by kill() from pid 45354 and user 0.
>>>> #0 0x0000000040fd52f4 in ?? ()
>>>> (gdb) bt
>>>> #0 0x0000000040fd52f4 in ?? ()
>>>> #1 0x00000000002fb784 in ?? ()
>>>> Backtrace stopped: previous frame identical to this frame (corrupt stack?)
>>>> (gdb)
>>> You should build the system libraries with the debug symbols.
>>> Otherwise the backtrace is not useful.
>> I don't think that would have helped in this specific instance. It
>> looks like a smashed stack.
> Why?
> I would believe that unwinder become confused because there is no
> proper unwind info, but I am not sure about aarch64.
>
> Anyway, I think the following patch would provide some minimal
> useful information, but it is not a substitute for the full
> backtrace.
>
> commit cce9267d4c66eee4dd1be28024910944b6511939
> Author: Konstantin Belousov <kib@FreeBSD.org>
> Date: Tue Oct 27 15:05:59 2026 +0200
>
> libthr/thr_mutex.c: make mutex asserts more useful
>
> Print the line number of the caller where the assert fired.
>
> Sponsored by: The FreeBSD Foundation
> MFC after: 1 week
>
> diff --git a/lib/libthr/thread/thr_mutex.c b/lib/libthr/thread/thr_mutex.c
> index 9c0939aa6080..9c6f6d8c5dd9 100644
> --- a/lib/libthr/thread/thr_mutex.c
> +++ b/lib/libthr/thread/thr_mutex.c
> @@ -131,41 +131,51 @@ mutex_init_link(struct pthread_mutex *m __unused)
> #endif
> }
>
> -static void
> -mutex_assert_is_owned(struct pthread_mutex *m __unused)
> -{
> -
> #if defined(_PTHREADS_INVARIANTS)
> - if (__predict_false(m->m_qe.tqe_prev == NULL))
> - PANIC("mutex %p own %#x is not on list %p %p",
> +static void
> +mutex_assert_is_owned(struct pthread_mutex *m, u_int line)
> +{
> + if (__predict_false(m->m_qe.tqe_prev == NULL)) {
> + PANIC("%s:%u: mutex %p own %#x is not on list %p %p",
> + __FILE__, line,
> m, m->m_lock.m_owner, m->m_qe.tqe_prev, m->m_qe.tqe_next);
> -#endif
> + }
> }
>
> static void
> -mutex_assert_not_owned(struct pthread *curthread __unused,
> - struct pthread_mutex *m __unused)
> +mutex_assert_not_owned(struct pthread_mutex *m, u_int line)
> {
> + struct pthread *curthread;
>
> -#if defined(_PTHREADS_INVARIANTS)
> + curthread = _get_curthread();
> if (__predict_false(m->m_qe.tqe_prev != NULL ||
> - m->m_qe.tqe_next != NULL))
> - PANIC("mutex %p own %#x is on list %p %p",
> + m->m_qe.tqe_next != NULL)) {
> + PANIC("%s:%u: mutex %p own %#x is on list %p %p",
> + __FILE__, line,
> m, m->m_lock.m_owner, m->m_qe.tqe_prev, m->m_qe.tqe_next);
> + }
> if (__predict_false(is_robust_mutex(m) &&
> (m->m_lock.m_rb_lnk != 0 || m->m_rb_prev != NULL ||
> (is_pshared_mutex(m) && curthread->robust_list ==
> (uintptr_t)&m->m_lock) ||
> (!is_pshared_mutex(m) && curthread->priv_robust_list ==
> - (uintptr_t)&m->m_lock))))
> + (uintptr_t)&m->m_lock)))) {
> PANIC(
> - "mutex %p own %#x is on robust linkage %p %p head %p phead %p",
> + "%s:%u: mutex %p own %#x is on robust linkage %p %p head %p phead %p",
> + __FILE__, line,
> m, m->m_lock.m_owner, (void *)m->m_lock.m_rb_lnk,
> m->m_rb_prev, (void *)curthread->robust_list,
> (void *)curthread->priv_robust_list);
> -#endif
> + }
> }
>
> +#define MUTEX_ASSERT_IS_OWNED(m) mutex_assert_is_owned(m, __LINE__)
> +#define MUTEX_ASSERT_NOT_OWNED(m) mutex_assert_not_owned(m, __LINE__)
> +#else
> +#define MUTEX_ASSERT_IS_OWNED(m) do {} while (0)
> +#define MUTEX_ASSERT_NOT_OWNED(m) do {} while (0)
> +#endif
> +
> static bool
> is_pshared_mutex(struct pthread_mutex *m)
> {
> @@ -477,8 +487,7 @@ _thr_mutex_destroy(pthread_mutex_t *mutex)
> if (m1 != NULL) {
> if ((uint32_t)m1->m_lock.m_owner !=
> UMUTEX_RB_OWNERDEAD) {
> - mutex_assert_not_owned(
> - _get_curthread(), m1);
> + MUTEX_ASSERT_NOT_OWNED(m1);
> }
> __thr_pshared_destroy(mutex);
> }
> @@ -490,7 +499,7 @@ _thr_mutex_destroy(pthread_mutex_t *mutex)
> ret = EBUSY;
> } else {
> *mutex = THR_MUTEX_DESTROYED;
> - mutex_assert_not_owned(_get_curthread(), m);
> + MUTEX_ASSERT_NOT_OWNED(m);
> __thr_free(m);
> ret = 0;
> }
> @@ -530,7 +539,7 @@ enqueue_mutex(struct pthread *curthread, struct pthread_mutex *m,
>
> /* Add to the list of owned mutexes: */
> if (error != EOWNERDEAD)
> - mutex_assert_not_owned(curthread, m);
> + MUTEX_ASSERT_NOT_OWNED(m);
> qidx = mutex_qidx(m);
> TAILQ_INSERT_TAIL(&curthread->mq[qidx], m, m_qe);
> if (!is_pshared_mutex(m))
> @@ -558,7 +567,7 @@ dequeue_mutex(struct pthread *curthread, struct pthread_mutex *m)
> struct pthread_mutex *mp, *mn;
> int qidx;
>
> - mutex_assert_is_owned(m);
> + MUTEX_ASSERT_IS_OWNED(m);
> qidx = mutex_qidx(m);
> if (is_robust_mutex(m)) {
> mp = m->m_rb_prev;
> @@ -1054,7 +1063,7 @@ _pthread_mutex_setprioceiling(pthread_mutex_t * __restrict mutex,
>
> curthread = _get_curthread();
> if (PMUTEX_OWNER_ID(m) == TID(curthread)) {
> - mutex_assert_is_owned(m);
> + MUTEX_ASSERT_IS_OWNED(m);
> m1 = TAILQ_PREV(m, mutex_queue, m_qe);
> m2 = TAILQ_NEXT(m, m_qe);
> if ((m1 != NULL && m1->m_lock.m_ceilings[0] > (u_int)ceiling) ||
--
Best Regards
ShengYi Hung