git: 076687302bff - main - Mk: fix checksum/fetch/makesum with large distfiles set
Date: Wed, 16 Sep 2026 13:08:27 UTC
The branch main has been updated by mat:
URL: https://cgit.FreeBSD.org/ports/commit/?id=076687302bffe156fb9b379c2eb622f0ac2e711e
commit 076687302bffe156fb9b379c2eb622f0ac2e711e
Author: Mathieu Arnold <mat@FreeBSD.org>
AuthorDate: 2026-09-14 13:28:08 +0000
Commit: Mathieu Arnold <mat@FreeBSD.org>
CommitDate: 2026-09-16 13:06:39 +0000
Mk: fix checksum/fetch/makesum with large distfiles set
When a port has a very large set of DISTFILES, makesum/fetch dies with
an obscure message such as:
> /tmp/makeQEBorT: /usr/bin/env: Argument list too long
The interface from make(1) to sh(1) passes the list of MASTER_SITES and
DISTFILES as arguments on the command line. At one point, it reaches
ARG_MAX and fails with the previous message. This changes the interface
to use temporary files in WRKDIR.
PR: 295029
Differential Revision: https://reviews.freebsd.org/D59475
---
Mk/Scripts/checksum.sh | 10 ++---
Mk/Scripts/do-fetch.sh | 29 ++++++++-----
Mk/Scripts/makesum.sh | 8 ++--
Mk/bsd.port.mk | 109 +++++++++++++++++++++++++------------------------
4 files changed, 83 insertions(+), 73 deletions(-)
diff --git a/Mk/Scripts/checksum.sh b/Mk/Scripts/checksum.sh
index d1ca174acf59..46cd449209fb 100644
--- a/Mk/Scripts/checksum.sh
+++ b/Mk/Scripts/checksum.sh
@@ -7,9 +7,9 @@ set -o pipefail
. "${dp_SCRIPTSDIR}/functions.sh"
-validate_env dp_CHECKSUM_ALGORITHMS dp_CURDIR dp_DISTDIR dp_DISTINFO_FILE \
- dp_DIST_SUBDIR dp_ECHO_MSG dp_FETCH_REGET dp_MAKE dp_MAKEFLAGS \
- dp_DISABLE_SIZE dp_NO_CHECKSUM
+validate_env dp_CHECKSUM_ALGORITHMS dp_CKSUMFILES_FILE dp_CURDIR dp_DISTDIR \
+ dp_DISTINFO_FILE dp_DIST_SUBDIR dp_ECHO_MSG dp_FETCH_REGET dp_MAKE \
+ dp_MAKEFLAGS dp_DISABLE_SIZE dp_NO_CHECKSUM
[ -n "${DEBUG_MK_SCRIPTS}" -o -n "${DEBUG_MK_SCRIPTS_CHECKSUM}" ] && set -x
@@ -21,7 +21,7 @@ if [ -f "${dp_DISTINFO_FILE}" ]; then
cd "${dp_DISTDIR}"
OK=
refetchlist=
- for file in "${@}"; do
+ while read -r file ; do
ignored="true"
for alg in ${dp_CHECKSUM_ALGORITHMS}; do
ignore="false"
@@ -69,7 +69,7 @@ if [ -f "${dp_DISTINFO_FILE}" ]; then
${dp_ECHO_MSG} "=> No suitable checksum found for $file."
OK=false
fi
- done
+ done < ${dp_CKSUMFILES_FILE}
if [ "${OK:=true}" = "retry" ] && [ "${dp_FETCH_REGET}" -gt 0 ]; then
${dp_ECHO_MSG} "===> Refetch for ${dp_FETCH_REGET} more times files: $refetchlist"
diff --git a/Mk/Scripts/do-fetch.sh b/Mk/Scripts/do-fetch.sh
index da42191cf533..810587a355bb 100644
--- a/Mk/Scripts/do-fetch.sh
+++ b/Mk/Scripts/do-fetch.sh
@@ -8,16 +8,21 @@ set -o pipefail
. "${dp_SCRIPTSDIR}/functions.sh"
validate_env dp_DEVELOPER dp_DISABLE_SIZE dp_DISTDIR dp_DISTINFO_FILE \
- dp_DIST_SUBDIR dp_ECHO_MSG dp_FETCH_AFTER_ARGS dp_FETCH_BEFORE_ARGS \
- dp_FETCH_CMD dp_FETCH_ENV dp_FORCE_FETCH_ALL dp_FORCE_FETCH_LIST \
- dp_MASTER_SITE_BACKUP dp_MASTER_SITE_OVERRIDE dp_MASTER_SORT_AWK \
- dp_NO_CHECKSUM dp_RANDOMIZE_SITES dp_SITE_FLAVOR dp_TARGET
+ dp_DIST_SUBDIR dp_DISTFILES_FILE dp_ECHO_MSG dp_FETCH_AFTER_ARGS \
+ dp_FETCH_BEFORE_ARGS dp_FETCH_CMD dp_FETCH_ENV dp_FORCE_FETCH_ALL \
+ dp_FORCE_FETCH_LIST dp_MASTER_SITE_BACKUP dp_MASTER_SITES_FILE \
+ dp_MASTER_SITE_OVERRIDE dp_MASTER_SORT_AWK dp_NO_CHECKSUM \
+ dp_RANDOMIZE_SITES dp_SITE_FLAVOR dp_TARGET
[ -n "${DEBUG_MK_SCRIPTS}" -o -n "${DEBUG_MK_SCRIPTS_DO_FETCH}" ] && set -x
set -u
case ${dp_TARGET} in
+ makesum-fetch)
+ dp_NO_CHECKSUM=yes
+ dp_DISABLE_SIZE=yes
+ ;&
do-fetch|makesum)
if [ ! -d "${dp_DISTDIR}" ]; then
mkdir -p "${dp_DISTDIR}"
@@ -26,7 +31,11 @@ case ${dp_TARGET} in
;;
esac
-for _file in "${@}"; do
+. $dp_MASTER_SITES_FILE
+
+# Read the list of files to fetch from stdin, one per line, instead of
+# from the command line to workatound ARG_MAX limits.
+while IFS= read -r _file; do
file=${_file%%:*}
# If this files has groups
@@ -68,7 +77,7 @@ for _file in "${@}"; do
fi
fi
case ${dp_TARGET} in
- do-fetch|makesum)
+ do-fetch|makesum|makesum-fetch)
${dp_ECHO_MSG} "=> $file doesn't seem to exist in ${dp_DISTDIR}."
if [ ! -w "${dp_DISTDIR}" ]; then
${dp_ECHO_MSG} "=> ${dp_DISTDIR} is not writable by you; cannot fetch."
@@ -87,7 +96,7 @@ for _file in "${@}"; do
__MASTER_SITES_TMP="${__MASTER_SITES_TMP} ${___MASTER_SITES_TMP}"
else
case ${dp_TARGET} in
- do-fetch|makesum)
+ do-fetch|makesum|makesum-fetch)
if [ -n "${dp_DEVELOPER}" ]; then
${dp_ECHO_MSG} "===> /!\\ Error /!\\"
else
@@ -147,7 +156,7 @@ for _file in "${@}"; do
fi
_fetch_cmd="${_fetch_cmd} ${args} ${dp_FETCH_AFTER_ARGS}"
case ${dp_TARGET} in
- do-fetch|makesum)
+ do-fetch|makesum|makesum-fetch)
${dp_ECHO_MSG} "=> Attempting to fetch ${site}${file}"
if env -S "${dp_FETCH_ENV}" ${_fetch_cmd}; then
actual_size=$(stat -f %z "${file}")
@@ -171,7 +180,7 @@ for _file in "${@}"; do
esac
done
case ${dp_TARGET} in
- do-fetch|makesum)
+ do-fetch|makesum|makesum-fetch)
${dp_ECHO_MSG} "=> Couldn't fetch it - please try to retrieve this"
${dp_ECHO_MSG} "=> port manually into ${dp_DISTDIR} and try again."
exit 1
@@ -180,5 +189,5 @@ for _file in "${@}"; do
echo "echo \"${file}\" not fetched; }"
;;
esac
-done
+done < $dp_DISTFILES_FILE
diff --git a/Mk/Scripts/makesum.sh b/Mk/Scripts/makesum.sh
index c9f50e56bb5d..e3781f8abfc2 100644
--- a/Mk/Scripts/makesum.sh
+++ b/Mk/Scripts/makesum.sh
@@ -7,8 +7,8 @@ set -o pipefail
. "${dp_SCRIPTSDIR}/functions.sh"
-validate_env dp_CHECKSUM_ALGORITHMS dp_CKSUMFILES dp_DISTDIR dp_DISTINFO_FILE \
- dp_ECHO_MSG
+validate_env dp_CHECKSUM_ALGORITHMS dp_CKSUMFILES_FILE dp_DISTDIR \
+ dp_DISTINFO_FILE dp_ECHO_MSG
[ -n "${DEBUG_MK_SCRIPTS}" -o -n "${DEBUG_MK_SCRIPTS_MAKESUM}" ] && set -x
@@ -31,7 +31,7 @@ if [ -f "${dp_DISTINFO_FILE}" ] && grep -q "^TIMESTAMP " ${dp_DISTINFO_FILE}; th
grep -v "^TIMESTAMP " ${dp_DISTINFO_FILE} > ${DISTINFO_OLD} || true
fi
-for file in ${dp_CKSUMFILES}; do
+while read -r file ; do
for alg in ${dp_CHECKSUM_ALGORITHMS}; do
eval "alg_executable=\$dp_$alg"
@@ -40,7 +40,7 @@ for file in ${dp_CKSUMFILES}; do
fi
done
echo "SIZE ($file) = $(stat -f %z "$file")" >> "${DISTINFO_NEW}"
-done
+done < ${dp_CKSUMFILES_FILE}
# Now, we generate the distinfo file in two cases:
# - If the saved file is empty, it means there was no TIMESTAMP in it, so we
diff --git a/Mk/bsd.port.mk b/Mk/bsd.port.mk
index 48782c59fa7e..b0edcb14b53b 100644
--- a/Mk/bsd.port.mk
+++ b/Mk/bsd.port.mk
@@ -3111,57 +3111,50 @@ _DO_FETCH_ENV+= dp_DEVELOPER=
# Fetch
-. if !target(do-fetch)
-do-fetch:
-. if !empty(DISTFILES)
- @${SETENV} \
- ${_DO_FETCH_ENV} ${_MASTER_SITES_ENV} \
- dp_SITE_FLAVOR=MASTER \
- ${SH} ${SCRIPTSDIR}/do-fetch.sh ${DISTFILES:C/.*/'&'/}
-. endif
-. if defined(PATCHFILES) && !empty(PATCHFILES)
- @${SETENV} \
- ${_DO_FETCH_ENV} ${_PATCH_SITES_ENV} \
- dp_SITE_FLAVOR=PATCH \
- ${SH} ${SCRIPTSDIR}/do-fetch.sh ${PATCHFILES:C/:-p[0-9]//:C/.*/'&'/}
-. endif
-. endif
-#
-# Prints out a list of files to fetch (useful to do a batch fetch)
-
-. if !target(fetch-list)
-fetch-list:
-. if !empty(DISTFILES)
- @${SETENV} \
- ${_DO_FETCH_ENV} ${_MASTER_SITES_ENV} \
- dp_SITE_FLAVOR=MASTER \
- ${SH} ${SCRIPTSDIR}/do-fetch.sh ${DISTFILES:C/.*/'&'/}
-. endif
-. if defined(PATCHFILES) && !empty(PATCHFILES)
- @${SETENV} \
- ${_DO_FETCH_ENV} ${_PATCH_SITES_ENV} \
- dp_SITE_FLAVOR=PATCH \
- ${SH} ${SCRIPTSDIR}/do-fetch.sh ${PATCHFILES:C/:-p[0-9]//:C/.*/'&'/}
-. endif
-. endif
-
-# Used by fetch-urlall-list and fetch-url-list
-
-. if !target(fetch-url-list-int)
-fetch-url-list-int:
-. if !empty(DISTFILES)
+_MASTER_SITES_FILE=${WRKDIR}/.master_sites
+_DISTFILES_FILE=${WRKDIR}/.distfiles
+_PATCH_SITES_FILE=${WRKDIR}/.patch_sites
+_PATCHFILES_FILE=${WRKDIR}/.patchfiles
+
+# do-fetch does the fetching
+# fetch-list Prints out a list of files to fetch (useful to do a batch fetch)
+# fetch-url-list-int Used by fetch-urlall-list and fetch-url-list
+. for _target in do-fetch fetch-list fetch-url-list-int makesum-fetch
+. if !target(${_target})
+${_target}:
+ @${MKDIR} ${WRKDIR}
+. if !empty(DISTFILES)
+ @${RM} ${_MASTER_SITES_FILE} ${_DISTFILES_FILE}
+. for site in ${_MASTER_SITES_ENV}
+ @printf '%s\n' "${site}" >> ${_MASTER_SITES_FILE}
+. endfor
+. for file in ${DISTFILES}
+ @printf '%s\n' "${file}" >> ${_DISTFILES_FILE}
+. endfor
@${SETENV} \
- ${_DO_FETCH_ENV} ${_MASTER_SITES_ENV} \
+ ${_DO_FETCH_ENV} \
+ dp_MASTER_SITES_FILE=${_MASTER_SITES_FILE} \
+ dp_DISTFILES_FILE=${_DISTFILES_FILE} \
dp_SITE_FLAVOR=MASTER \
- ${SH} ${SCRIPTSDIR}/do-fetch.sh ${DISTFILES:C/.*/'&'/}
-. endif
-. if defined(PATCHFILES) && !empty(PATCHFILES)
+ ${SH} ${SCRIPTSDIR}/do-fetch.sh
+. endif
+. if defined(PATCHFILES) && !empty(PATCHFILES)
+ @${RM} ${_PATCH_SITES_FILE} ${_PATCHFILES_FILE}
+. for site in ${_PATCH_SITES_ENV}
+ @printf '%s\n' "${site}" >> ${_PATCH_SITES_FILE}
+. endfor
+. for file in ${PATCHFILES}
+ @printf '%s\n' "${file}" >> ${_PATCHFILES_FILE}
+. endfor
@${SETENV} \
- ${_DO_FETCH_ENV} ${_PATCH_SITES_ENV} \
+ ${_DO_FETCH_ENV} \
+ dp_PATCH_SITES_FILE=${_PATCH_SITES_FILE} \
+ dp_PATCHFILES_FILE=${_PATCHFILES_FILE} \
dp_SITE_FLAVOR=PATCH \
- ${SH} ${SCRIPTSDIR}/do-fetch.sh ${PATCHFILES:C/:-p[0-9]//:C/.*/'&'/}
+ ${SH} ${SCRIPTSDIR}/do-fetch.sh
+. endif
. endif
-. endif
+. endfor
. if !target(fetch-url-recursive-list-int)
fetch-url-recursive-list-int: fetch-url-list-int
@@ -3936,6 +3929,7 @@ _CHECKSUM_INIT_ENV= \
dp_SHA256=${SHA256}
. if !target(makesum)
+_CKSUMFILES_FILE=${WRKDIR}/.cksumfiles
# Some port change the options with OPTIONS_*_FORCE when make(makesum) to be
# able to add all distfiles in one go.
# For this to work, we need to call the do-fetch script directly here so that
@@ -3943,28 +3937,35 @@ _CHECKSUM_INIT_ENV= \
# As we're fetching new distfiles, that are not in the distinfo file, disable
# checksum and sizes checks.
makesum: check-sanity
- @cd ${.CURDIR} && ${MAKE} fetch NO_CHECKSUM=yes \
- DISABLE_SIZE=yes DISTFILES="${DISTFILES}" \
- MASTER_SITES="${MASTER_SITES}" \
- MASTER_SITE_SUBDIR="${MASTER_SITE_SUBDIR}" \
- PATCH_SITES="${PATCH_SITES}"
+ @cd ${.CURDIR} && ${MAKE} makesum-fetch
+ @${MKDIR} ${WRKDIR}
+ @${RM} ${_CKSUMFILES_FILE}
+. for file in ${_CKSUMFILES}
+ @printf '%s\n' "${file}" >> ${_CKSUMFILES_FILE}
+. endfor
@${SETENV} \
${_CHECKSUM_INIT_ENV} \
dp_CHECKSUM_ALGORITHMS='${CHECKSUM_ALGORITHMS:tu}' \
- dp_CKSUMFILES='${_CKSUMFILES}' \
+ dp_CKSUMFILES_FILE='${_CKSUMFILES_FILE}' \
dp_DISTDIR='${DISTDIR}' \
dp_DISTINFO_FILE='${DISTINFO_FILE}' \
dp_ECHO_MSG='${ECHO_MSG}' \
dp_SCRIPTSDIR='${SCRIPTSDIR}' \
- ${SH} ${SCRIPTSDIR}/makesum.sh ${DISTFILES:C/.*/'&'/}
+ ${SH} ${SCRIPTSDIR}/makesum.sh
. endif
. if !target(checksum)
checksum: fetch
. if !empty(_CKSUMFILES) && !defined(NO_CHECKSUM)
+ @${MKDIR} ${WRKDIR}
+ @${RM} ${_CKSUMFILES_FILE}
+. for file in ${_CKSUMFILES}
+ @printf '%s\n' "${file}" >> ${_CKSUMFILES_FILE}
+. endfor
@${SETENV} \
${_CHECKSUM_INIT_ENV} \
dp_CHECKSUM_ALGORITHMS='${CHECKSUM_ALGORITHMS:tu}' \
+ dp_CKSUMFILES_FILE='${_CKSUMFILES_FILE}' \
dp_CURDIR='${.CURDIR}' \
dp_DISTDIR='${DISTDIR}' \
dp_DISTINFO_FILE='${DISTINFO_FILE}' \
@@ -3976,7 +3977,7 @@ checksum: fetch
dp_SCRIPTSDIR='${SCRIPTSDIR}' \
dp_DISABLE_SIZE='${DISABLE_SIZE}' \
dp_NO_CHECKSUM='${NO_CHECKSUM}' \
- ${SH} ${SCRIPTSDIR}/checksum.sh ${_CKSUMFILES:C/.*/'&'/}
+ ${SH} ${SCRIPTSDIR}/checksum.sh
. endif
. endif