INDEX user-friendliness and SMP speed-up patch

[Kris Kennaway]

The attached patch does a few things to 'make index':

* Fixes the bsd.port.subdir.mk code that is supposed to report index
  breakage (the fallback code wasn't actually being run because make
  would halt immediately following the error).  This should help with
  INDEX error reports because it will immediately show the cause of
  failure, so we won't have to pull teeth to extract it from the
  submitter.

* Streamlines the 'make describe' code a bit.

* Provide some basic instructions to the user when an index build
  fails, on when and how to report index build failures (turn this off
  with INDEX_QUIET=1)

* Removed INDEX_NOSORT, because I couldn't imagine it to be very useful.

* Don't prevent INDEX builds from seeing the local host environment.
  Since a lot of users are using 'make index' thesedays they should get
  an index that reflects their local settings and installed ports.  If
  you want to build a 'default' index that isn't influenced by local
  settings (e.g. for release builds), set the INDEX_PRISTINE variable.

* Allows parallel INDEX builds (using make -j).  The most obvious way
  of doing this doesn't work, because I/O from child makes is broken up
  into 2k chunks, and output lines from 'make describe' that exceed this
  length (*cough* GNOME *cough*) will be intertwined with the output of
  other makes, leading to a corrupted INDEX.  Instead, I collect the
  output from the child make processes in temporary files and
  recombine them at the end.

The number of concurrent make processes to spawn can be set using
INDEX_JOBS.  By default this is set to 2, which seems to be a sweet
spot for both single and dual-processor systems.  On my tests I do not
see any significant performance changes on UP, but on a dual 4.x
system the build time drops by 47% (6 minute index builds on one test
machine!).  Depending on your disk and CPU hardware you might see
further gains with INDEX_JOBS=4 or higher, so you might like to
experiment to see what works best.  On a dual 5.x system the
performance gains do not seem to be as great (20-30%), but this is
still a significant net win.  I'd be interested to see benchmarks from
/usr/bin/time on various systems - make sure that the system is idle
while you're building index, because it is sensitive to disk loads.

Please test, and provide feedback.

Kris



-------------- next part --------------
Index: Makefile
===================================================================
RCS file: /home/ncvs/ports/Makefile,v
retrieving revision 1.81
diff -u -r1.81 Makefile
--- Makefile	23 Feb 2004 04:41:00 -0000	1.81
+++ Makefile	15 Mar 2004 09:52:23 -0000
@@ -70,20 +70,39 @@
 	@rm -f ${.CURDIR}/${INDEXFILE}
 	@cd ${.CURDIR} && make ${.CURDIR}/${INDEXFILE}
 
+INDEX_JOBS?=	2
+
 ${.CURDIR}/${INDEXFILE}:
 	@echo -n "Generating ${INDEXFILE} - please wait.."; \
-	export LOCALBASE=/nonexistentlocal; \
-	export X11BASE=/nonexistentx; \
-	cd ${.CURDIR} && make describe ECHO_MSG="echo > /dev/null" | \
-		perl ${.CURDIR}/Tools/make_index | \
-	sed -e 's/  */ /g' -e 's/|  */|/g' -e 's/  *|/|/g' -e "s,$${LOCALBASE},/usr/local," -e "s,$${X11BASE},/usr/X11R6," > ${INDEXFILE}
-.if !defined(INDEX_NOSORT)
-	@sed -e 's./.
.g' ${.CURDIR}/${INDEXFILE} | \
+	if [ "${INDEX_PRISTINE}" != "" ]; then \
+		export LOCALBASE=/nonexistentlocal; \
+		export X11BASE=/nonexistentx; \
+	fi; \
+	tmpdir=`/usr/bin/mktemp -d -t index` || exit 1; \
+	trap "rm -rf $${tmpdir}; exit 1" 1 2 3 5 10 13 15; \
+	( cd ${.CURDIR} && make -j${INDEX_JOBS} INDEX_TMPDIR=$${tmpdir} BUILDING_INDEX=1 \
+		ECHO_MSG="echo > /dev/null" describe ) || \
+		(rm -rf $${tmpdir} ; \
+		if [ "${INDEX_QUIET}" = "" ]; then \
+			echo; \
+			echo "********************************************************************"; \
+			echo "Before reporting this error, verify that you are running a supported"; \
+			echo "version of FreeBSD (see http://www.FreeBSD.org/ports/) and that you"; \
+			echo "have a complete and up-to-date ports collection.  If so, then report"; \
+			echo "the failure to ports at FreeBSD.org together with relevant details of"; \
+			echo "your ports configuration (including FreeBSD version, environment and"; \
+			echo "/etc/make.conf settings)."; \
+			echo "********************************************************************"; \
+			echo; \
+		fi; \
+ 		exit 1); \
+	cat $${tmpdir}/${INDEXFILE}.desc.* | perl ${.CURDIR}/Tools/make_index | \
+		sed -e 's/  */ /g' -e 's/|  */|/g' -e 's/  *|/|/g' -e "s,$${LOCALBASE},/usr/local," \
+			-e "s,$${X11BASE},/usr/X11R6," -e 's./.
.g' | \
 		sort -t '|' +1 -2 | \
-		sed -e 's.
./.g' > ${.CURDIR}/${INDEXFILE}.tmp; \
-	mv -f ${.CURDIR}/${INDEXFILE}.tmp ${.CURDIR}/${INDEXFILE}
-.endif
-	@echo " Done."
+		sed -e 's.
./.g' > ${.CURDIR}/${INDEXFILE}; \
+	rm -rf $${tmpdir}; \
+	echo " Done."
 
 print-index:	${.CURDIR}/${INDEXFILE}
 	@awk -F\| '{ printf("Port:\t%s\nPath:\t%s\nInfo:\t%s\nMaint:\t%s\nIndex:\t%s\nB-deps:\t%s\nR-deps:\t%s\n\n", $$1, $$2, $$4, $$6, $$7, $$8, $$9); }' < ${.CURDIR}/${INDEXFILE}
Index: Mk/bsd.port.subdir.mk
===================================================================
RCS file: /home/ncvs/ports/Mk/bsd.port.subdir.mk,v
retrieving revision 1.52
diff -u -r1.52 bsd.port.subdir.mk
--- Mk/bsd.port.subdir.mk	4 Feb 2004 04:27:04 -0000	1.52
+++ Mk/bsd.port.subdir.mk	15 Mar 2004 01:00:30 -0000
@@ -198,38 +195,35 @@
 
 .if !target(describe)
 .if defined(PORTSTOP)
+# This is a bit convoluted to deal with the fact that make will overlap I/O from child make processes
+# if they write more than 2k.  This will corrupt the INDEX file, so we have to make sure that each
+# child make writes to their own file, which we will combine at the end.  This gives substantial
+# performance benefits over doing a make -j1
+
+.if defined(BUILDING_INDEX)
+describe: ${SUBDIR:S/^/describe./}
+
+.for i in ${SUBDIR}
+describe.$i:
+	@${MAKE} -B ${i:S/^/_/:S/$/.describe/} > ${INDEX_TMPDIR}/${INDEXFILE}.desc.${i}
+.endfor
+.else
 describe: ${SUBDIR:S/^/_/:S/$/.describe/}
+.endif
 .else
 describe:
-	@TMPFILE=`mktemp -q /tmp/describe.XXXXXX` || exit 1; \
-	for sub in ${SUBDIR}; do \
-	OK=""; \
-	for dud in $$DUDS; do \
-		if [ $${dud} = $$sub ]; then \
-			OK="false"; \
-			${ECHO_MSG} "===> ${DIRPRFX}$$sub skipped"; \
-		fi; \
-	done; \
-	if test -d ${.CURDIR}/$${sub}.${MACHINE_ARCH}; then \
-		edir=$${sub}.${MACHINE_ARCH}; \
-	elif test -d ${.CURDIR}/$${sub}; then \
-		edir=$${sub}; \
+	@for sub in ${SUBDIR}; do \
+	if test -d ${.CURDIR}/$${sub}; then \
+		${ECHO_MSG} "===> ${DIRPRFX}$${sub}"; \
+		cd ${.CURDIR}/$${sub}; \
+		${MAKE} -B describe 2> /dev/null || \
+			(echo "===> ${DIRPRFX}$${sub} failed:" >&2 ; \
+			cd ${.CURDIR}/$${sub}; ${MAKE} -B describe >&2; \
+			exit 1) ;\
 	else \
-		OK="false"; \
 		${ECHO_MSG} "===> ${DIRPRFX}$${sub} non-existent"; \
 	fi; \
-	if [ "$$OK" = "" ]; then \
-		${ECHO_MSG} "===> ${DIRPRFX}$${edir}"; \
-		cd ${.CURDIR}/$${edir}; \
-		${MAKE} -B describe 2>$${TMPFILE}; \
-		if [ -s $${TMPFILE} ]; then \
-			echo "===> ${DIRPRFX}$${sub} failed:" >&2; \
-			cat $${TMPFILE} >&2; \
-			echo -n >$${TMPFILE}; \
-		fi; \
-	fi; \
-	done; \
-	rm -f $${TMPFILE}
+	done
 .endif
 .endif
 
-------------- next part --------------
A non-text attachment was scrubbed...
Name: not available
Type: application/pgp-signature
Size: 187 bytes
Desc: not available
Url : http://lists.freebsd.org/pipermail/freebsd-ports/attachments/20040315/45074064/attachment.bin