Interesting process hang on head -r333079 aarch64 (Pine64+ 2GB)?

Mark Millard marklmi26-fbsd at yahoo.com
Sat Apr 28 20:11:51 UTC 2018


Using a new instance of top to watch and older one, the old
one shows (no longer updating):

up 0+13:47:14  09:58:51

and the new one shows for it:

  PID USERNAME    THR PRI NICE   SIZE    RES   SWAP STATE   C   TIME     CPU COMMAND
. . .
  834 root          1  20    0 13652K  1972K     0K select  3   2:21   0.00% top -CawSores
 
when the new one shows:

up 0+16:56:14  13:07:51

So, over 3 hours later.

I'll note that about 2 hours earlier it showed a larger RES figure, the same
figure the old top display shows:

  PID USERNAME    THR PRI NICE   SIZE    RES   SWAP STATE   C   TIME     CPU COMMAND
. . .
  834 root          1  20    0 13652K  2576K     0K select  3   2:21   0.00% top -CawSores

(gstat -pd also tends to stop updating, but I use top as the
example here.)

For reference the old (non-updating) top instance shows itself as:

  PID USERNAME    THR PRI NICE   SIZE    RES   SWAP STATE   C   TIME     CPU COMMAND
. . .
  834 root          1  20    0 13652K  2576K     0K CPU3    3   2:21   0.33% top -CawSores

The context is a Pine64+ 2GB building devel/llvm60 over the time,
via poudriere-devel, 1 builder but allowing it to use all 4 cores.
UFS context, not ZFS. devel/llvm60 started building 3:55:35 into
the poudriere run and has been in process for many hours.

The instances of top are via ssh logins over Ethernet. poudriere is
running on the console. (No non-poudriere console messages during
the poudriere run.)

Typing Control-L to the ssh session for old instance of top will cause
it to start updating again. (That is what I normally do when I notice
the lack of updates.)

# uname -apKU
FreeBSD pine64 12.0-CURRENT FreeBSD 12.0-CURRENT  r333079M  arm64 aarch64 1200062 1200062

This is built as a non-debug build with symbols and explicitly
causing -mcpu=cortex-a53 :

XCFLAGS+= -mcpu=cortex-a53
XCXXFLAGS+= -mcpu=cortex-a53
# There is no XCPPFLAGS but XCPP gets XCFLAGS content.
ACFLAGS.arm64cpuid.S+=  -mcpu=cortex-a53+crypto
ACFLAGS.aesv8-armx.S+=  -mcpu=cortex-a53+crypto
ACFLAGS.ghashv8-armx.S+=        -mcpu=cortex-a53+crypto


As for what vintage I'm building for /usr/ports :

# svnlite info /usr/ports/ | grep "Re[plv]"
Relative URL: ^/head
Repository Root: svn://svn.freebsd.org/ports
Repository UUID: 35697150-7ecd-e111-bb59-0022644237b5
Revision: 468505
Last Changed Rev: 468505

But it is a large /usr/ports jump, from something like early
January if I remember right. (FreeBSD was also from back then
before I started this update sequence.)


Context details (if you care):

# svnlite status /usr/src | sort
?       /usr/src/local_iperf3.sh
?       /usr/src/sys/amd64/conf/GENERIC-DBG
?       /usr/src/sys/amd64/conf/GENERIC-NODBG
?       /usr/src/sys/arm/conf/GENERIC-DBG
?       /usr/src/sys/arm/conf/GENERIC-NODBG
?       /usr/src/sys/arm64/conf/GENERIC-DBG
?       /usr/src/sys/arm64/conf/GENERIC-NODBG
?       /usr/src/sys/powerpc/conf/GENERIC64vtsc-DBG
?       /usr/src/sys/powerpc/conf/GENERIC64vtsc-NODBG
?       /usr/src/sys/powerpc/conf/GENERICvtsc-DBG
?       /usr/src/sys/powerpc/conf/GENERICvtsc-NODBG
M       /usr/src/contrib/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp
M       /usr/src/contrib/llvm/tools/lld/ELF/Arch/PPC64.cpp
M       /usr/src/crypto/openssl/crypto/armcap.c
M       /usr/src/lib/libkvm/kvm_powerpc.c
M       /usr/src/lib/libkvm/kvm_private.c
M       /usr/src/release/Makefile.vm
M       /usr/src/release/scripts/mk-vmimage.sh
M       /usr/src/release/tools/vmimage.subr
M       /usr/src/secure/lib/libcrypto/Makefile
M       /usr/src/stand/defs.mk
M       /usr/src/stand/powerpc/boot1.chrp/Makefile
M       /usr/src/stand/powerpc/kboot/Makefile
M       /usr/src/sys/arm64/arm64/identcpu.c
M       /usr/src/sys/conf/kmod.mk
M       /usr/src/sys/conf/ldscript.powerpc
M       /usr/src/sys/kern/subr_pcpu.c
M       /usr/src/sys/powerpc/aim/mmu_oea64.c
M       /usr/src/sys/powerpc/ofw/ofw_machdep.c
M       /usr/src/sys/powerpc/powerpc/interrupt.c
M       /usr/src/sys/powerpc/powerpc/mp_machdep.c
M       /usr/src/sys/powerpc/powerpc/trap.c
M       /usr/src/sys/vm/uma_core.c
M       /usr/src/usr.bin/top/machine.c

Most of the "M"s are non-aarch64 material. My GENERIC-*DBG's
include the matching GENERIC and then override various
debug-status points. Only powerpc family ones do somewhat
more.

# svnlite diff /usr/src/crypto/openssl/crypto/armcap.c
Index: /usr/src/crypto/openssl/crypto/armcap.c
===================================================================
--- /usr/src/crypto/openssl/crypto/armcap.c	(revision 333079)
+++ /usr/src/crypto/openssl/crypto/armcap.c	(working copy)
@@ -137,7 +137,9 @@
         }
     } else if (sigsetjmp(ill_jmp, 1) == 0) {
         _armv7_neon_probe();
+#ifndef DISABLE_HACK_THAT_AVOIDS_NEON
         OPENSSL_armcap_P |= ARMV7_NEON;
+#endif
         if (sigsetjmp(ill_jmp, 1) == 0) {
             _armv8_pmull_probe();
             OPENSSL_armcap_P |= ARMV8_PMULL | ARMV8_AES;

# svnlite diff /usr/src/sys/arm64/arm64/identcpu.c
Index: /usr/src/sys/arm64/arm64/identcpu.c
===================================================================
--- /usr/src/sys/arm64/arm64/identcpu.c	(revision 333079)
+++ /usr/src/sys/arm64/arm64/identcpu.c	(working copy)
@@ -1109,6 +1109,9 @@
 
 		/* Wake up the other CPUs */
 		atomic_store_rel_int(&ident_lock, 0);
-		__asm __volatile("sev" ::: "memory");
+		__asm __volatile(
+			"dsb ish	\n"
+			"sev		\n"
+			::: "memory");
 	}
 }

# svnlite diff /usr/src/sys/kern/subr_pcpu.c
Index: /usr/src/sys/kern/subr_pcpu.c
===================================================================
--- /usr/src/sys/kern/subr_pcpu.c	(revision 333079)
+++ /usr/src/sys/kern/subr_pcpu.c	(working copy)
@@ -279,6 +279,8 @@
 struct pcpu *
 pcpu_find(u_int cpuid)
 {
+if (mp_ncpus == 0 && cpuid == 0) {} // HACK: This combination is used in the late bootstrap
+//else if (mp_ncpus <= cpuid) { panic("pcpu_find: cpuid too large"); } // HACK
 
 	return (cpuid_to_pcpu[cpuid]);
 }

(The above I've sometimes uncommented the else for powerpc
family experiments.)

# svnlite diff /usr/src/sys/vm/uma_core.c
Index: /usr/src/sys/vm/uma_core.c
===================================================================
--- /usr/src/sys/vm/uma_core.c	(revision 333079)
+++ /usr/src/sys/vm/uma_core.c	(working copy)
@@ -3441,7 +3441,7 @@
 void
 uma_reclaim_wakeup(void)
 {
-
+printf("limit %lX, total %lX, needed %d\n", uma_kmem_limit, uma_kmem_total, uma_reclaim_needed);
 	if (atomic_fetchadd_int(&uma_reclaim_needed, 1) == 0)
 		wakeup(uma_reclaim);
 }


I've seen the behavior with an unmodified top but I add a
"K MaxObsUsed, " for swap normally:

# svnlite diff /usr/src/usr.bin/top/machine.c
Index: /usr/src/usr.bin/top/machine.c
===================================================================
--- /usr/src/usr.bin/top/machine.c	(revision 333079)
+++ /usr/src/usr.bin/top/machine.c	(working copy)
@@ -194,9 +194,9 @@
 	NULL
 };
 
-int swap_stats[7];
+int swap_stats[8];
 char *swapnames[] = {
-	"K Total, ", "K Used, ", "K Free, ", "% Inuse, ", "K In, ", "K Out",
+	"K Total, ", "K Used, ", "K MaxObsUsed, ", "K Free, ", "% Inuse, ", "K In, ", "K Out",
 	NULL
 };
 
@@ -550,14 +550,14 @@
 
 		/* first interval */
 		if (swappgsin < 0) {
-			swap_stats[4] = 0;
-			swap_stats[5] = 0;
+			swap_stats[4+1] = 0;
+			swap_stats[5+1] = 0;
 		}
 
 		/* compute differences between old and new swap statistic */
 		else {
-			swap_stats[4] = pagetok(((nspgsin - swappgsin)));
-			swap_stats[5] = pagetok(((nspgsout - swappgsout)));
+			swap_stats[4+1] = pagetok(((nspgsin - swappgsin)));
+			swap_stats[5+1] = pagetok(((nspgsout - swappgsout)));
 		}
 
 		swappgsin = nspgsin;
@@ -564,14 +564,16 @@
 		swappgsout = nspgsout;
 
 		/* call CPU heavy swapmode() only for changes */
-		if (swap_stats[4] > 0 || swap_stats[5] > 0 || swap_delay == 0) {
-			swap_stats[3] = swapmode(&swapavail, &swapfree);
+		if (swap_stats[4+1] > 0 || swap_stats[5+1] > 0 || swap_delay == 0) {
+			swap_stats[3+1] = swapmode(&swapavail, &swapfree);
 			swap_stats[0] = swapavail;
 			swap_stats[1] = swapavail - swapfree;
-			swap_stats[2] = swapfree;
+			if (swap_stats[2+0]<swap_stats[1])
+				swap_stats[2+0] = swap_stats[1];
+			swap_stats[2+1] = swapfree;
 		}
 		swap_delay = 1;
-		swap_stats[6] = -1;
+		swap_stats[6+1] = -1;
 	}
 
 	if (arc_enabled) {



As for /usr/ports:

# svnlite status /usr/ports/
M       /usr/ports/Mk/bsd.port.mk
M       /usr/ports/devel/libunwind/Makefile
M       /usr/ports/lang/gcc7/Makefile
M       /usr/ports/lang/qt5-qml/Makefile
M       /usr/ports/sysutils/cdrdao/Makefile
M       /usr/ports/textproc/qt5-xmlpatterns/Makefile
M       /usr/ports/x11-toolkits/qt5-gui/Makefile

I build ports with symbols and optimations,
a combination not officially supported. I've
also had problems at times with ${UNAME} -s
and ${UNAME} -r not working (empty strings
resulting) and have used explicit echo's
instead as a work around. (Elsewhere, such as
cdrdao, ${UNAME} -m too.)

# svnlite diff /usr/ports/Mk/bsd.port.mk
Index: /usr/ports/Mk/bsd.port.mk
===================================================================
--- /usr/ports/Mk/bsd.port.mk	(revision 468505)
+++ /usr/ports/Mk/bsd.port.mk	(working copy)
@@ -1148,12 +1148,12 @@
 
 # Get the operating system type
 .if !defined(OPSYS)
-OPSYS!=	${UNAME} -s
+OPSYS!=	echo FreeBSD
 .endif
 _EXPORTED_VARS+=	OPSYS
 
 .if !defined(_OSRELEASE)
-_OSRELEASE!=	${UNAME} -r
+_OSRELEASE!=	echo 12.0-CURRENT
 .endif
 _EXPORTED_VARS+=	_OSRELEASE
 
@@ -1756,7 +1756,11 @@
 STRIP_CMD=	${TRUE}
 .endif
 DEBUG_FLAGS?=	-g
+.if defined(ALLOW_OPTIMIZATIONS_FOR_WITH_DEBUG)
+CFLAGS:=		${CFLAGS} ${DEBUG_FLAGS}
+.else
 CFLAGS:=		${CFLAGS:N-O*:N-fno-strict*} ${DEBUG_FLAGS}
+.endif
 .if defined(INSTALL_TARGET)
 INSTALL_TARGET:=	${INSTALL_TARGET:S/^install-strip$/install/g}
 .endif




===
Mark Millard
marklmi at yahoo.com
( dsl-only.net went
away in early 2018-Mar)



More information about the freebsd-arm mailing list