svn commit: r365674 - user/cperciva/freebsd-update-build

[Gordon Tetlow]

Author: gordon
Date: Sat Sep 12 23:04:13 2020
New Revision: 365674
URL: https://svnweb.freebsd.org/changeset/base/365674

Log:
  Add parallelization.
  
  Use a couple of simple shell constructs to monitor the number of jobs
  running at any given time and if we exceed the maximum (based on CPUs
  since this is a CPU bound process), we sleep and recheck if any of our
  children have finished.

Modified:
  user/cperciva/freebsd-update-build/build-upgrade-patches.sh

Modified: user/cperciva/freebsd-update-build/build-upgrade-patches.sh
==============================================================================
--- user/cperciva/freebsd-update-build/build-upgrade-patches.sh	Sat Sep 12 21:50:25 2020	(r365673)
+++ user/cperciva/freebsd-update-build/build-upgrade-patches.sh	Sat Sep 12 23:04:13 2020	(r365674)
@@ -12,6 +12,21 @@ OLDRELS=$3
 
 WWWDIR=${BASEDIR}/pub
 
+MAXJOBS=$(( $(sysctl -n hw.ncpu) - 2 ))
+if [ $MAXJOBS -lt 1 ]; then
+	MAXJOBS=1
+fi
+
+genpatch() {
+	tempdir=$(mktemp -d -t genpatch)
+
+	gunzip < ${WWWDIR}/${OR}/${ARCH}/f/${OH}.gz > ${tempdir}/${OH}
+	gunzip < ${WWWDIR}/${NR}/${ARCH}/f/${NH}.gz > ${tempdir}/${NH}
+	bsdiff ${tempdir}/${OH} ${tempdir}/${NH} ${WWWDIR}/to-${TARGETREL}/${ARCH}/bp/${OH}-${NH}
+	rm ${tempdir}/${OH} ${tempdir}/${NH}
+	rmdir ${tempdir}
+}
+
 mkdir -p ${WWWDIR}/to-${TARGETREL}/${ARCH}/bp/
 for V in ${OLDRELS}; do
 	zcat ${WWWDIR}/${V}/${ARCH}/m/* |
@@ -43,10 +58,17 @@ zcat ${WWWDIR}/${TARGETREL}/${ARCH}/m/* |
 	if [ ${OH} = ${NH} ]; then
 		continue
 	fi
-	gunzip < ${WWWDIR}/${OR}/${ARCH}/f/${OH}.gz > ${OH}
-	gunzip < ${WWWDIR}/${NR}/${ARCH}/f/${NH}.gz > ${NH}
-	bsdiff ${OH} ${NH} ${WWWDIR}/to-${TARGETREL}/${ARCH}/bp/${OH}-${NH}
-	rm ${OH} ${NH}
+
+	genpatch &
+
+	jobs="$(jobs)"
+	numjobs=$(echo "$jobs" | wc -l)
+	while [ $numjobs -ge $MAXJOBS ]; do
+		sleep 2
+
+		jobs="$(jobs)"
+		numjobs=$(echo "$jobs" | wc -l)
+	done
     done
 
 rm hashtab