socsvn commit: r237211 - soc2012/scher/par_ports/head/Mk
scher at FreeBSD.org
scher at FreeBSD.org
Wed Jun 6 13:17:57 UTC 2012
Author: scher
Date: Wed Jun 6 13:17:53 2012
New Revision: 237211
URL: http://svnweb.FreeBSD.org/socsvn/?view=rev&rev=237211
Log:
[new_feature] bsd.port.mk - parallel non-blocking build of port's
dependecies as background jobs in XXX-depends targets.
${_PROCESS_ACTIVE_BUILDS}: new script that covers most of
non-blocking parallel dependency build magic.
Controls all spawned background jobs by PIDs which are stored in $${active_builds} array
Controls jobs exit codes.
Acts accordingly on various jobs exit codes (0, 1, ${_parv_MAKE_LOCK_EXIT_STATUS}).
Recheck correct install of lib dependency.
Considers parallel dep builds limit as set by ${_parv_PARALLEL_BUILDS_NUMBER}.
Waits for a while if it is impossible to spawn new dependency build.
Makes attempts to spawn another dependency build, if it is possible.
-------------------------------------
Changes in old targets (XXX-depends):
Reimplementation of loops in XXX-depends targets for non-blocking purpose:
Outer "while" loop replaced "for" loop.
Some kind of "round-robin" implementation is used to process port's deps.
If some dependency (dep) is locked then it is moved to the end of the deps queue.
If some dep is already installed then it is deleted from the deps queue.
If some dependency is not found - it's build starts as background process and
it's PID is added to $${active_builds} array to control it's evaluation.
--------------------------------------
All the above mentioned is actual for non-parallel builds,
with the following exception:
${_parv_PARALLEL_BUILDS_NUMBER} is set to 1 in non-parallel case.
This perevents from spawning more than one background job.
Thus processing of port's deps will be blocked until this job will be ready.
Hence we have one background job per time for non-parallel build.
Modified:
soc2012/scher/par_ports/head/Mk/bsd.port.mk
Modified: soc2012/scher/par_ports/head/Mk/bsd.port.mk
==============================================================================
--- soc2012/scher/par_ports/head/Mk/bsd.port.mk Wed Jun 6 12:45:48 2012 (r237210)
+++ soc2012/scher/par_ports/head/Mk/bsd.port.mk Wed Jun 6 13:17:53 2012 (r237211)
@@ -5028,8 +5028,13 @@
.endif
############### PAR_PORTS SPECIFIC COMMENT LINE ###############
-# _INSTALL_DEPENDS script is surrounded by while loop.
-# When the port is installed it is necessary to break most inner while loop.
+# $${active_builds} - a list of "pid:path:dir[:target]" or "pid:lib:dir[:target]"
+# tuples of all currently processed ports, spawned by this make process
+# $${notfound} codes : 0 - dep found
+# 1 - dep not found
+# ${_parv_ON_LOCK_EXIT_STATUS} - dep is locked
+# $${spawned} - sets to background process PID if any process was ran
+# in background.
############### END OF PAR_PORTS SPECIFIC COMMENT LINE ###############
_INSTALL_DEPENDS= \
@@ -5048,32 +5053,83 @@
${PKG_ADD} $${subpkgfile}; \
fi; \
else \
- (cd $$dir; ${MAKE} -DINSTALLS_DEPENDS $$target $$depends_args) && status=$$? || status=$$?; \
+ (cd $$dir; ${MAKE} -DINSTALLS_DEPENDS $$target $$depends_args) & spawned=$$!; \
fi; \
else \
- (cd $$dir; ${MAKE} -DINSTALLS_DEPENDS $$target $$depends_args) && status=$$? || status=$$?; \
+ (cd $$dir; ${MAKE} -DINSTALLS_DEPENDS $$target $$depends_args) & spawned=$$!; \
fi; \
- if [ ! ${_parv_WANT_PARALLEL_BUILD} ] && [ $${status} != 0 ]; then \
- exit $${status}; \
+
+_PROCESS_ACTIVE_BUILDS= \
+ echo "debugging_out_here: --- ENTER _PROCESS_ACTIVE_BUILDS SECTION " > ${PORTSDIR}/tmp/log; \
+ while true; do \
+ builds_num=$$( ${ECHO_CMD} $${active_builds} | wc -w ); \
+ if [ $${builds_num} -lt ${_parv_PARALLEL_BUILDS_NUMBER} ] && [ $${\#depends} -ne 0 ]; then \
+ echo "debugging_out_here: WE CAN START ANOTHER PAR BUILD " > ${PORTSDIR}/tmp/log; \
+ break; \
fi; \
- if [ ${_parv_WANT_PARALLEL_BUILD} ]; then \
- if [ $${status} -eq 0 ]; then \
- _parv_next_dep=1; \
- elif [ $${status} -eq ${_parv_MAKE_LOCK_EXIT_STATUS} ]; then \
- ${ECHO_CMD} $${prog} may be installing now.; \
- sleep 2; \
- continue; \
- else \
- exit $${status}; \
- fi; \
+ if [ $${builds_num} -eq 0 ] && [ $${\#depends} -eq 0 ]; then \
+ ${ECHO_MSG} "===> Returning to build of ${PKGNAME}"; \
+ break; \
fi; \
- ${ECHO_MSG} "===> Returning to build of ${PKGNAME}";
+ for build in $$( ${ECHO_CMD} "$${active_builds}" ); do \
+ pid=$${build%%:*}; \
+ dep=$${build\#*:}; \
+ ps -p $${pid} > /dev/null || { \
+ wait $${pid} && status=$$? || status=$$?; \
+ if [ $${status} -eq 0 ]; then \
+ echo "debugging_out_here: SOME DEP IS INSTALLED ($${dep}) " > ${PORTSDIR}/tmp/log; \
+ echo "debugging_out_here: WAS _$${active_builds}_ " > ${PORTSDIR}/tmp/log; \
+ active_builds="$${active_builds%%$${build}*} $${active_builds\#\#*$${build}}"; \
+ active_builds=$$( echo "$${active_builds}" | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$$//' ); \
+ builds_num=$$(( $${builds_num} - 1 )); \
+ echo "debugging_out_here: NOW _$${active_builds}_ " > ${PORTSDIR}/tmp/log; \
+ if [ ${.TARGET} = "lib-depends" ]; then \
+ lib=$${dep%%:*}; \
+ pattern="`${ECHO_CMD} $$lib | ${SED} -E -e 's/\./\\\\./g' -e 's/(\\\\)?\+/\\\\+/g'`"; \
+ if ! ${LDCONFIG} ${_LDCONFIG_FLAGS} -r | ${GREP} -vwF -e "${PKGCOMPATDIR}" | ${GREP} -qwE -e "-l$$pattern"; then \
+ ${ECHO_MSG} "Error: shared library \"$$lib\" does not exist"; \
+ ${FALSE}; \
+ fi; \
+ fi; \
+ elif [ $${status} -eq ${_parv_MAKE_LOCK_EXIT_STATUS} ]; then \
+ echo "debugging_out_here: SOME DEP IS LOCKED ($${dep}) " > ${PORTSDIR}/tmp/log; \
+ echo "debugging_out_here: WAS _$${active_builds}_ " > ${PORTSDIR}/tmp/log; \
+ active_builds="$${active_builds%%$${build}*} $${active_builds\#\#*$${build}}"; \
+ active_builds=$$( echo "$${active_builds}" | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$$//' ); \
+ builds_num=$$(( $${builds_num} - 1 )); \
+ echo "debugging_out_here: NOW _$${active_builds}_ " > ${PORTSDIR}/tmp/log; \
+ echo "debugging_out_here: ADD IT BACK DO DEPENDS LIST " > ${PORTSDIR}/tmp/log; \
+ echo "debugging_out_here: WAS _$${depends}_ " > ${PORTSDIR}/tmp/log; \
+ depends="$${depends} $${dep}"; \
+ depends=$$( echo "$${depends}" | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$$//' ); \
+ echo "debugging_out_here: NOW _$${depends}_ " > ${PORTSDIR}/tmp/log; \
+ else \
+ ${ECHO_CMD} "Errors occured while building a dependency."; \
+ ${ECHO_CMD} "Terminating..."; \
+ exit 1; \
+ fi; \
+ }; \
+ done; \
+ if { [ $${builds_num} -eq ${_parv_PARALLEL_BUILDS_NUMBER} ] || \
+ ( [ $${builds_num} -gt 0 ] && [ $${\#depends} -eq 0 ] ); }; then \
+ echo "debugging_out_here: CAN'T SPAWN MORE DEPS! SLEEP... " > ${PORTSDIR}/tmp/log; \
+ sleep 2; \
+ fi; \
+ done; \
+ echo "debugging_out_here: --- LEAVE _PROCESS_ACTIVE_BUILDS SECTION " > ${PORTSDIR}/tmp/log
.for deptype in PKG EXTRACT PATCH FETCH BUILD RUN
${deptype:L}-depends:
.if defined(${deptype}_DEPENDS)
.if !defined(NO_DEPENDS)
- @for i in `${ECHO_CMD} "${${deptype}_DEPENDS}"`; do \
+ @depends=`${ECHO_CMD} "${${deptype}_DEPENDS}"`; \
+ active_builds=""; \
+ echo "debugging_out_here: ------------------------- " > ${PORTSDIR}/tmp/log; \
+ echo "debugging_out_here: ALL DEPS - $${depends}" > ${PORTSDIR}/tmp/log; \
+ while [ $${#depends} -ne 0 ]; do \
+ i=$${depends%% *}; \
+ echo "debugging_out_here: ----- NEW STEP -----" >> ${PORTSDIR}/tmp/log; \
+ echo "debugging_out_here CURRENT DEP $${i}" >> ${PORTSDIR}/tmp/log; \
prog=$${i%%:*}; \
if [ -z "$$prog" ]; then \
${ECHO_MSG} "Error: there is an empty port dependency in ${deptype}_DEPENDS."; \
@@ -5095,8 +5151,6 @@
target="${DEPENDS_TARGET}"; \
depends_args="${DEPENDS_ARGS}"; \
fi; \
- _parv_next_dep=0; \
- while [ $${_parv_next_dep} -eq 0 ]; do \
if [ ${_parv_WANT_PARALLEL_BUILD} ]; then \
( attempts=-1; ${_parv_PKG_DBDIR_LOCK_LOOP} ); \
fi; \
@@ -5163,10 +5217,9 @@
( cd $${dir}; ${MAKE} check-lock ) || { \
status=$$?; \
if [ $${status} -eq ${_parv_MAKE_LOCK_EXIT_STATUS} ]; then \
- ( ${_parv_PKG_DBDIR_DO_UNLOCK} ); \
+ echo "debugging_out_here: LOCKED $${i}" >> ${PORTSDIR}/tmp/log; \
${ECHO_CMD} $${prog} may be installing now.; \
- sleep 2; \
- continue; \
+ notfound=${_parv_ON_LOCK_EXIT_STATUS}; \
else \
exit $${status}; \
fi; \
@@ -5175,17 +5228,33 @@
if [ ${_parv_WANT_PARALLEL_BUILD} ]; then \
( ${_parv_PKG_DBDIR_DO_UNLOCK} ); \
fi; \
- if [ $$notfound != 0 ]; then \
+ if [ $$notfound -eq 1 ]; then \
${ECHO_MSG} "===> Verifying $$target for $$prog in $$dir"; \
if [ ! -d "$$dir" ]; then \
${ECHO_MSG} " => No directory for $$prog. Skipping.."; \
else \
${_INSTALL_DEPENDS} \
+ if [ $${spawned} ]; then \
+ echo "debugging_out_here: SPAWNED PID $${spawned}" > ${PORTSDIR}/tmp/log; \
+ active_builds="$${active_builds} $${spawned}:$${i}"; \
+ depends="$${depends%%$${i}*} $${depends##*$${i}}"; \
+ depends=$$( echo "$${depends}" | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$$//' ); \
+ spawned=""; \
+ echo "debugging_out_here: AFTER TRANSFER TO ACTIVE BUILDS _$${depends}_" >> ${PORTSDIR}/tmp/log; \
+ fi; \
+ fi; \
+ elif [ $${notfound} -eq 0 ]; then \
+ depends="$${depends%%$${i}*} $${depends##*$${i}}"; \
+ depends=$$( echo "$${depends}" | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$$//' ); \
+ echo "debugging_out_here: AFTER DELETE _$${depends}_" >> ${PORTSDIR}/tmp/log; \
+ elif [ $${notfound} -eq ${_parv_ON_LOCK_EXIT_STATUS} ]; then \
+ if [ $$( ${ECHO_CMD} $${depends} | wc -w ) -gt 1 ]; then \
+ depends="$${depends#* } $${depends%% *}"; \
+ depends=$$( echo "$${depends}" | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$$//' ); \
+ echo "debugging_out_here: AFTER CHANGE $${depends}" >> ${PORTSDIR}/tmp/log; \
fi; \
fi; \
- if [ $${notfound} -eq 0 ]; then _parv_next_dep=1; fi; \
- if [ ! ${_parv_WANT_PARALLEL_BUILD} ]; then _parv_next_dep=1; fi; \
- done; \
+ ${_PROCESS_ACTIVE_BUILDS}; \
done
.endif
.else
@@ -5195,7 +5264,14 @@
lib-depends:
.if defined(LIB_DEPENDS) && !defined(NO_DEPENDS)
- @for i in ${LIB_DEPENDS}; do \
+ @depends=`${ECHO_CMD} "${LIB_DEPENDS}"`; \
+ active_builds=""; \
+ echo "debugging_out_here: ------------------------- " > ${PORTSDIR}/tmp/log; \
+ echo "debugging_out_here: ALL DEPS - $${depends}" > ${PORTSDIR}/tmp/log; \
+ while [ $${#depends} -ne 0 ]; do \
+ i=$${depends%% *}; \
+ echo "debugging_out_here: ----- NEW STEP -----" >> ${PORTSDIR}/tmp/log; \
+ echo "debugging_out_here CURRENT DEP $${i}" >> ${PORTSDIR}/tmp/log; \
lib=$${i%%:*}; \
pattern="`${ECHO_CMD} $$lib | ${SED} -E -e 's/\./\\\\./g' -e 's/(\\\\)?\+/\\\\+/g'`"\
dir=$${i#*:}; \
@@ -5206,12 +5282,10 @@
else \
dir=$${dir%%:*}; \
fi; \
- ${ECHO_MSG} -n "===> ${PKGNAME} depends on shared library: $$lib"; \
- _parv_next_dep=0; \
- while [ $${_parv_next_dep} -eq 0 ]; do \
if [ ${_parv_WANT_PARALLEL_BUILD} ]; then \
( attempts=-1; ${_parv_PKG_DBDIR_LOCK_LOOP} ); \
fi; \
+ ${ECHO_MSG} -n "===> ${PKGNAME} depends on shared library: $$lib"; \
if ${LDCONFIG} ${_LDCONFIG_FLAGS} -r | ${GREP} -vwF -e "${PKGCOMPATDIR}" | ${GREP} -qwE -e "-l$$pattern"; then \
${ECHO_MSG} " - found"; \
if [ ${_DEPEND_ALWAYS} = 1 ]; then \
@@ -5228,10 +5302,9 @@
( cd $${dir}; ${MAKE} check-lock ) || { \
status=$$?; \
if [ $${status} -eq ${_parv_MAKE_LOCK_EXIT_STATUS} ]; then \
- ( ${_parv_PKG_DBDIR_DO_UNLOCK} ); \
- ${ECHO_CMD} $${prog} may be installing now.; \
- sleep 2; \
- continue; \
+ echo "debugging_out_here: LOCKED $${i}" >> ${PORTSDIR}/tmp/log; \
+ ${ECHO_CMD} $${lib} may be installing now.; \
+ notfound=${_parv_ON_LOCK_EXIT_STATUS}; \
else \
exit $${status}; \
fi; \
@@ -5240,21 +5313,33 @@
if [ ${_parv_WANT_PARALLEL_BUILD} ]; then \
( ${_parv_PKG_DBDIR_DO_UNLOCK} ); \
fi; \
- if [ $$notfound != 0 ]; then \
+ if [ $$notfound -eq 1 ]; then \
${ECHO_MSG} "===> Verifying $$target for $$lib in $$dir"; \
if [ ! -d "$$dir" ]; then \
${ECHO_MSG} " => No directory for $$lib. Skipping.."; \
else \
${_INSTALL_DEPENDS} \
- if ! ${LDCONFIG} ${_LDCONFIG_FLAGS} -r | ${GREP} -vwF -e "${PKGCOMPATDIR}" | ${GREP} -qwE -e "-l$$pattern"; then \
- ${ECHO_MSG} "Error: shared library \"$$lib\" does not exist"; \
- ${FALSE}; \
+ if [ $${spawned} ]; then \
+ echo "debugging_out_here: SPAWNED PID $${spawned}" > ${PORTSDIR}/tmp/log; \
+ active_builds="$${active_builds} $${spawned}:$${i}"; \
+ depends="$${depends%%$${i}*} $${depends##*$${i}}"; \
+ depends=$$( echo "$${depends}" | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$$//' ); \
+ spawned=""; \
+ echo "debugging_out_here: AFTER TRANSFER TO ACTIVE BUILDS _$${depends}_" >> ${PORTSDIR}/tmp/log; \
fi; \
fi; \
- fi; \
- if [ $${notfound} -eq 0 ]; then _parv_next_dep=1; fi; \
- if [ ! ${_parv_WANT_PARALLEL_BUILD} ]; then _parv_next_dep=1; fi; \
- done; \
+ elif [ $${notfound} -eq 0 ]; then \
+ depends="$${depends%%$${i}*} $${depends##*$${i}}"; \
+ depends=$$( echo "$${depends}" | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$$//' ); \
+ echo "debugging_out_here: AFTER DELETE _$${depends}_" >> ${PORTSDIR}/tmp/log; \
+ elif [ $${notfound} -eq ${_parv_ON_LOCK_EXIT_STATUS} ]; then \
+ if [ $$( ${ECHO_CMD} $${depends} | wc -w ) -gt 1 ]; then \
+ depends="$${depends#* } $${depends%% *}"; \
+ depends=$$( echo "$${depends}" | sed 's/^[[:space:]]*//' | sed 's/[[:space:]]*$$//' ); \
+ echo "debugging_out_here: AFTER CHANGE $${depends}" >> ${PORTSDIR}/tmp/log; \
+ fi; \
+ fi; \
+ ${_PROCESS_ACTIVE_BUILDS}; \
done
.endif
More information about the svn-soc-all
mailing list