ports/152877: Patch to add locking to ports make operations

Tue Dec 7 01:20:12 UTC 2010

>Number:         152877
>Category:       ports
>Synopsis:       Patch to add locking to ports make operations
>Confidential:   no
>Severity:       non-critical
>Priority:       low
>Responsible:    freebsd-ports-bugs
>State:          open
>Quarter:        
>Keywords:       
>Date-Required:
>Class:          change-request
>Submitter-Id:   current-users
>Arrival-Date:   Tue Dec 07 01:20:11 UTC 2010
>Closed-Date:
>Last-Modified:
>Originator:     Ivan Brawley
>Release:        8.1STABLE 20101104
>Organization:
>Environment:
FreeBSD bang 8.1-STABLE FreeBSD 8.1-STABLE #3: Thu Nov  4 18:21:39 CST 2010     root at bang:/usr/obj/usr/src/sys/BANG  amd64

>Description:
With the use of machines with multiple cores, being able to use "make -j" for ports becomes more and more desirable.

Work has been done to allow for individual ports to be built with "make -j", however making multiple ports at once can cause problems. Problems arise when multiple ports try to build/install the same dependency at the same time.

So I've worked on a patch to Mk/bsd.port.mk that will use the lockf(1) command to lock the port's Makefile before doing any of the seven major targets (fetch, extract, patch, configure, build, install and package).

This method will cause the 2nd make (or 3rd, etc) to block it's make while another is running. It does cause the two to swap between being blocked and doing the work, but at least it's safe in that only one of them is doing the work.

It adds "WITH_PORT_LOCKING" and "WITHOUT_PORT_LOCKING" variables to control whether to do locking and "LOCKF_CMD" that does the actual locking. I've chosen the port's Makefile to control the locking of the port as every port has one and it allows for multiple logins/xterms to do makes with locking.
>How-To-Repeat:

>Fix:
The attached patch to Mk/bsd.port.mk contains the magic for port locking.

Include it if its decided it's good enough.

Patch attached with submission follows:

--- Mk/bsd.port.mk.orig	2010-12-07 10:46:25.550003334 +1030
+++ Mk/bsd.port.mk	2010-12-07 10:56:22.700237577 +1030
@@ -961,6 +961,10 @@
 #				  that are explicitly marked MAKE_JOBS_UNSAFE.  User settable.
 # MAKE_JOBS_NUMBER
 #				- Override the number of make jobs to be used.  User settable.
+# WITH_PORT_LOCKING
+#				- Acquire a "lock" when doing fetch, extract, patch, etc.
+#				  This will prevent multiple ports trying to make the same
+#				  dependency and tripping over each other.
 #
 # For install:
 #
@@ -2367,6 +2371,10 @@
 
 TAR?=	/usr/bin/tar
 
+.if defined(WITH_PORT_LOCKING) && !defined(WITHOUT_PORT_LOCKING)
+LOCKF_CMD?=	lockf -k -s ./Makefile
+.endif
+
 # EXTRACT_SUFX is defined in .pre.mk section
 .if defined(USE_ZIP)
 EXTRACT_CMD?=		${UNZIP_CMD}
@@ -4398,29 +4406,39 @@
 .else
 _CHROOT_SEQ=
 .endif
-_SANITY_SEQ=	${_CHROOT_SEQ} pre-everything check-makefile \
+_FETCH_DEP=		check-sanity
+_EXTRACT_DEP=		fetch
+_PATCH_DEP=		extract
+_CONFIGURE_DEP=		patch
+_BUILD_DEP=		configure
+_INSTALL_DEP=		build
+_PACKAGE_DEP=		install
+_SANITY_SEQ=		${_CHROOT_SEQ} pre-everything check-makefile \
 				check-categories check-makevars check-desktop-entries \
 				check-depends identify-install-conflicts check-deprecated \
 				check-vulnerable check-license buildanyway-message \
 				options-message
-_FETCH_DEP=		check-sanity
+.if defined(WITH_PORT_LOCKING) && !defined(WITHOUT_PORT_LOCKING) && !defined(_HAVE_PORT_LOCK_${PKGCATEGORY}_${PORTDIRNAME})
+_FETCH_SEQ=		lock-fetch
+_EXTRACT_SEQ=		lock-extract
+_PATCH_SEQ=		lock-patch
+_CONFIGURE_SEQ=		lock-configure
+_BUILD_SEQ=		lock-build
+_INSTALL_SEQ=		lock-install
+_PACKAGE_SEQ=		lock-package
+.else
 _FETCH_SEQ=		fetch-depends pre-fetch pre-fetch-script \
 				do-fetch post-fetch post-fetch-script
-_EXTRACT_DEP=	fetch
 _EXTRACT_SEQ=	check-build-conflicts extract-message checksum extract-depends \
 				pre-extract pre-extract-script do-extract \
 				post-extract post-extract-script
-_PATCH_DEP=		extract
 _PATCH_SEQ=		ask-license patch-message patch-depends patch-dos2unix pre-patch \
 				pre-patch-script do-patch post-patch post-patch-script
-_CONFIGURE_DEP=	patch
 _CONFIGURE_SEQ=	build-depends lib-depends configure-message \
 				pre-configure pre-configure-script \
 				run-autotools do-configure post-configure post-configure-script
-_BUILD_DEP=		configure
 _BUILD_SEQ=		build-message pre-build pre-build-script do-build \
 				post-build post-build-script
-_INSTALL_DEP=	build
 _INSTALL_SEQ=	install-message check-install-conflicts run-depends lib-depends apply-slist pre-install \
 				pre-install-script generate-plist check-already-installed
 _INSTALL_SUSEQ= check-umask install-mtree pre-su-install \
@@ -4430,9 +4448,19 @@
 				add-plist-docs add-plist-examples add-plist-data \
 				add-plist-post install-rc-script compress-man \
 				install-ldconfig-file fake-pkg security-check
-_PACKAGE_DEP=	install
 _PACKAGE_SEQ=	package-message pre-package pre-package-script \
 				do-package post-package-script
+.endif
+
+.if defined(WITH_PORT_LOCKING) && !defined(WITHOUT_PORT_LOCKING)
+.for target in fetch extract patch configure build install package
+.if !target(lock-${target})
+lock-${target}:
+	@${ECHO_MSG} "===>  Doing \"make ${target}\" under a lock for ${PKGNAME}"
+	@cd ${.CURDIR} && ${LOCKF_CMD} ${MAKE} -D_HAVE_PORT_LOCK_${PKGCATEGORY}_${PORTDIRNAME} ${__softMAKEFLAGS} ${target}
+.endif
+.endfor
+.endif
 
 .if !target(post-chroot)
 post-chroot:
@@ -5105,10 +5133,15 @@
 		if [ X${USE_PACKAGE_DEPENDS} != "X" ]; then \
 			subpkgfile=`(cd $$dir; ${MAKE} $$depends_args -V PKGFILE)`; \
 			if [ -r "$${subpkgfile}" -a "$$target" = "${DEPENDS_TARGET}" ]; then \
-				${ECHO_MSG} "===>   Installing existing package $${subpkgfile}"; \
-				${PKG_ADD} $${subpkgfile}; \
+				if [ X"${WITH_PORT_LOCKING}" = "X" -o X"${WITHOUT_PORT_LOCKING}" != "X" ]; then \
+					${ECHO_MSG} "===>   Installing existing package $${subpkgfile}"; \
+					${PKG_ADD} $${subpkgfile}; \
+				else \
+					${ECHO_MSG} "===>   Installing existing package (under a lock) $${subpkgfile}"; \
+					(cd $$dir; ${LOCKF_CMD} ${PKG_ADD} $${subpkgfile}) ; \
+				fi; \
 			else \
-			  (cd $$dir; ${MAKE} -DINSTALLS_DEPENDS $$target $$depends_args) ; \
+				(cd $$dir; ${MAKE} -DINSTALLS_DEPENDS $$target $$depends_args) ; \
 			fi; \
 		else \
 			(cd $$dir; ${MAKE} -DINSTALLS_DEPENDS $$target $$depends_args) ; \
@@ -5207,6 +5240,11 @@
 			else \
 				${_INSTALL_DEPENDS} \
 			fi; \
+		else \
+			if [ -d "$$dir" -a X"${WITH_PORT_LOCKING}" != "X" -a X"${WITHOUT_PORT_LOCKING}" = "X" ]; then \
+				${ECHO_MSG} "===>    Testing for lock on $$prog in $$dir"; \
+				(cd $$dir; ${LOCKF_CMD} ${ECHO_MSG} "     => No lock on $$prog. Continuing..") ; \
+			fi; \
 		fi; \
 	done
 .endif
@@ -5252,6 +5290,11 @@
 					${FALSE}; \
 				fi; \
 			fi; \
+		else \
+			if [ -d "$$dir" -a X"${WITH_PORT_LOCKING}" != "X" -a X"${WITHOUT_PORT_LOCKING" = "X" ]; then \
+				${ECHO_MSG} "===>    Testing for lock on $$lib in $$dir"; \
+				(cd $$dir; ${LOCKF_CMD} ${ECHO_MSG} "     => No lock on $$lib. Continuing..") ; \
+			fi; \
 		fi; \
 	done
 .endif


>Release-Note:
>Audit-Trail:
>Unformatted:

