svn commit: r238127 - user/dougb/portmaster
Doug Barton
dougb at FreeBSD.org
Thu Jul 5 09:00:33 UTC 2012
Author: dougb
Date: Thu Jul 5 09:00:32 2012
New Revision: 238127
URL: http://svn.freebsd.org/changeset/base/238127
Log:
Major updates/fixes to the distfile handling code
=================================================
* Make all parts of the code SUBDIR-safe (most were already)
- This involves using $port_subdir globally to refer to just the
directory name under DISTDIR, and add a $full_port_subdir to contain
the full path to the file for the places where that was used.
This also allows simplifying a few places where both were needed.
- Fix a few places where the relativity of the cwd was not respected,
most often resulting in files not being deleted that should be
* When checking DI_FILES, anchor the grep pattern with a ^ to
avoid false positives
* Move the storage of the distfile info back to /var/db/pkg where it
should have been all along. Mostly for correctness, but also to deal
with the problem of non-UNIQUENAMEs causing the wrong files to be
deleted with -d.
* Clean up the old /var/db/ports/*/distfiles files after reading them,
and delete the directory if it's empty. Since we are doing this
unconditionally now, there is no longer a need for delete_dist_list().
The new ones will live and die with the rest of the package directory.
* Update some of the messages printed out for distfile-related things
to make it more clear what's going on.
* Check DI_FILES to see if the distfile could be valid for another port
in more places, so add a function to deal with the common ones.
* Add 2 more functions, make_distfiles() and make_port_subdir() to bring
that common code (3 places each, ugh) under the same umbrella. Improve
both implementations to use fewer forks.
Other changes
=============
* Add -e to the echo for pm_sv so that we can easily add a newline
* Take advantage of the above, and other adjustments to the whitespace
to try to fit with the general theme of ws after a section
* Change the last 'find -d foo' to 'find foo -depth'
* Use 'while read' for the installed distfiles file, less forks
Modified:
user/dougb/portmaster/portmaster
Modified: user/dougb/portmaster/portmaster
==============================================================================
--- user/dougb/portmaster/portmaster Thu Jul 5 07:20:16 2012 (r238126)
+++ user/dougb/portmaster/portmaster Thu Jul 5 09:00:32 2012 (r238127)
@@ -144,7 +144,7 @@ parent_exit () {
fi
fi
if [ -z "$BACKUP" -a -z "$NO_BACKUP" -a -n "$NB_DELETE" ]; then
- pm_sv 'Deleting safety packages for successful installs'
+ pm_sv 'Deleting safety packages for successful installs\n'
pm_cd $pbu || fail "Cannot cd to $pbu"
pm_rm_s $NB_DELETE
fi
@@ -166,7 +166,7 @@ parent_exit () {
done
if [ -n "$PM_WRKDIRPREFIX" -a -z "$1" ]; then
- pm_v ; count=0
+ count=0
while : ; do
ps axww | grep '[m]ake clean NOCLEANDEPENDS=ncd' >/dev/null || break
count=$(( $count + 1 ))
@@ -225,7 +225,6 @@ parent_exit () {
fi
[ -n "$PM_LOG" ] && date >> $PM_LOG
fi
- [ -n "$PM_SU_VERBOSE" ] && echo ''
case "$show_list" in
all) echo "===>>> The following actions were performed:"
echo -e $INSTALLED_LIST
@@ -335,7 +334,7 @@ pm_rmdir_s () { $PM_SU_CMD /bin/r
pm_unlink_s () { [ -e "$1" ] && $PM_SU_CMD /bin/unlink $1; }
pm_v () { [ -n "$PM_VERBOSE" ] && echo "$*"; }
-pm_sv () { [ -n "$PM_SU_VERBOSE" ] && echo "===>>> SU $*"; }
+pm_sv () { [ -n "$PM_SU_VERBOSE" ] && echo -e "===>>> SU $*"; }
# Do this here so we can have a reasonably good guess.
# May be modified below.
@@ -1266,7 +1265,7 @@ delete_empty_dist_subdirs () {
# Get back to somewhere safe so we do not
# delete our CWD out from under ourselves
pm_cd $DISTDIR || fail "Cannot cd into $DISTDIR"
- find -d $DISTDIR -type d \( -empty -and ! -path \*\.zfs/\* \) -delete
+ find $DISTDIR -depth -type d \( -empty -and ! -path \*\.zfs/\* \) -delete
}
init_packages_var () {
@@ -1343,8 +1342,8 @@ if [ -n "$CLEAN_DISTFILES" ]; then
echo "===>>> Checking for stale distfiles" ; echo ''
for df in `find $DISTDIR -type f | sort`; do
f=${df#$DISTDIR}
- if ! grep -ql $f $DI_FILES; then
- get_answer_yn n "\t===>>> Delete stale file: ${f}"
+ if ! grep -ql ^$f $DI_FILES; then
+ get_answer_yn n "\t===>>> Delete stale file: $f"
case "$?" in
0) echo " Deleting $f" ; echo ''
/bin/unlink $df ;;
@@ -1778,57 +1777,66 @@ pm_pkg_create () {
find_dl_distfiles () {
# Global: dist_list dist_list_files
- local file
# We need to define these for use in the deletion/update process.
- if pm_cd $pd/$1; then
- dist_list=`pm_make_b -V OPTIONSFILE`
- dist_list="${dist_list%options}distfiles"
-
- [ -s "$dist_list" ] || { unset dist_list ; return 0; }
- [ -n "$DONT_SCRUB_DISTFILES" ] && return 0
-
- # The grep is needed to allow for comments, etc.
- for file in `grep ^DISTFILE $dist_list`; do
- file=${file#DISTFILE:} ; file=${file%%:*}
- dist_list_files="${dist_list_files}${file#*/} "
- done
+ if [ -s "${pdb}/${upg_port}/distfiles" ]; then
+ dist_list="${pdb}/${upg_port}/distfiles"
else
- # The port might have moved, etc.; so take a stab at it,
- # but do not take a chance with a possibly wrong answer
- dist_list="$port_dbdir/${1##*/}/distfiles"
-
- [ -s "$dist_list" ] || { unset dist_list ; return 0; }
- [ -n "$DONT_SCRUB_DISTFILES" ] && return 0
-
- # Do not strip the subdir, we will use it in delete_all
- for file in `grep ^DISTFILE $dist_list`; do
- file=${file#DISTFILE:} ; file=${file%%:*}
- dist_list_files="${dist_list_files}${file} "
- done
+ # Old method, for now
+ if pm_cd $pd/$1; then
+ dist_list=`pm_make_b -V OPTIONSFILE`
+ dist_list="${dist_list%options}distfiles"
+ else
+ # The port might have moved, etc.; so take a stab at it,
+ # but do not take a chance with a possibly wrong answer
+ dist_list="$port_dbdir/${1##*/}/distfiles"
+ fi
fi
-}
-delete_dist_list () {
- [ -n "$dist_list" ] || return 0
+ [ -s "$dist_list" ] || { unset dist_list ; return 0; }
+ [ -n "$DONT_SCRUB_DISTFILES" ] && return 0
+
+ local line
+ while read line; do
+ case "$line" in
+ DISTFILE*) line=${line#DISTFILE:} ; line=${line%%:*}
+ dist_list_files="${dist_list_files}${line#*/} " ;;
+ esac
+ done < $dist_list
+
+ # Clean up after the old method
+ case "$dist_list" in
+ ${port_dbdir}*)
+ pm_sv Deleting $dist_list
+ pm_unlink_s $dist_list
+
+ local dir=`find ${dist_list%/distfiles} -type d -empty 2>/dev/null`
+ if [ -d "$dir" ]; then
+ pm_sv Deleting empty $dir directory
+ pm_rmdir_s $dir
+ fi
- local dir
+ unset dist_list ;;
+ esac
+}
- pm_sv Deleting $dist_list
- pm_unlink_s $dist_list
+check_di_files () {
+ # Global: distfiles_checked
- dir=`find ${dist_list%/distfiles} -type d -empty 2>/dev/null`
- if [ -d "$dir" ]; then
- pm_sv Deleting the $dir directory
- pm_rmdir_s $dir
+ if [ -s "$DI_FILES" ]; then
+ if grep -ql ^${port_subdir}${1} $DI_FILES; then
+ distfiles_checked="${distfiles_checked}${1}:"
+ pm_v "===>>> Keeping distfile, valid for another port: $1"
+ return 0
+ fi
fi
+
+ return 1
}
find_and_delete_distfiles () {
- # Global: port_subdir DISTDIR distfiles distfiles_checked delete_all
- local ps file answer
-
- ps=${port_subdir#$DISTDIR}
+ # Global: distfiles_checked
+ local file answer
for file in ${1}*; do
# This generally means the pattern did not match
@@ -1844,23 +1852,17 @@ find_and_delete_distfiles () {
case "$distfiles" in
*" ${file} "*)
distfiles_checked="${distfiles_checked}${file}:"
- pm_v "===>>> Keeping current distfile: $file"
+ pm_v "===>>> Keeping current distfile: ${port_subdir}${file}"
continue ;; # Do not delete current version
- *) if [ -s "$DI_FILES" ]; then
- if grep -ql ${ps}$file $DI_FILES; then
- distfiles_checked="${distfiles_checked}${file}:"
- pm_v "===>>> Keeping current distfile: $file"
- continue # Do not delete current version
- fi
- fi
+ *) check_di_files $file && continue
if [ -n "$ALWAYS_SCRUB_DISTFILES" -o -n "$delete_all" ]; then
- echo "===>>> Deleting stale distfile: $file"
+ echo "===>>> Deleting stale distfile: ${port_subdir}${file}"
pm_unlink $file
continue
fi
- get_answer_g n y "\n===>>> Delete $file? y/n"
+ get_answer_g n y "\n===>>> Delete ${port_subdir}${file}? y/n"
case "$?" in
1) pm_unlink $file ;;
0) distfiles_checked="${distfiles_checked}${file}:" ;;
@@ -1869,9 +1871,27 @@ find_and_delete_distfiles () {
done
}
-set_distfiles_and_subdir () {
- # Global: dist_list_files distfiles port_subdir DISTDIR
+make_distfiles () {
+ # Global: distfiles
+
+ # Even if there are no distfiles there will be a space returned
+ distfiles=`pm_make -V ALLFILES` ; distfiles=${distfiles%% }
+ if [ -n "$distfiles" ]; then
+ distfiles=" $distfiles "
+ else
+ unset distfiles
+ fi
+}
+
+make_port_subdir () {
+ # Global: port_subdir full_port_subdir
+ port_subdir=`pm_make -V DIST_SUBDIR`
+ port_subdir="${port_subdir:+${port_subdir}/}"
+ full_port_subdir="${DISTDIR}${port_subdir}"
+}
+
+set_distfiles_and_subdir () {
[ -z "$dist_list_files" ] && find_dl_distfiles $1
if [ -d "$pd/$1" ]; then
@@ -1880,26 +1900,18 @@ set_distfiles_and_subdir () {
return 1
fi
- if [ -z "$distfiles" ]; then
- distfiles=`pm_make -V ALLFILES | sed -e 's# *$##g'`
- [ -n "$distfile" ] && distfiles=" ${distfiles} "
- fi
+ [ -n "$distfiles" ] || make_distfiles
+
# Ports may have no distfiles
[ -z "$dist_list_files" -a -z "$distfiles" ] && return 2
- if [ -z "$port_subdir" ]; then
- port_subdir=`pm_make -V DIST_SUBDIR`
- if [ -n "$port_subdir" ]; then
- port_subdir="${DISTDIR}${port_subdir}/"
- else
- port_subdir=$DISTDIR
- fi
- fi
- if [ -d "$port_subdir" ]; then
- pm_cd $port_subdir || fail "cd to $port_subdir failed!"
+ [ -n "$full_port_subdir" ] || make_port_subdir
+
+ if [ -d "$full_port_subdir" ]; then
+ pm_cd $full_port_subdir || fail "cd to $full_port_subdir failed!"
else
echo ''
- echo "===>>> $port_subdir does not exist, therefore we"
+ echo "===>>> $full_port_subdir does not exist, therefore we"
echo ' will assume that all relevant distfiles are gone.'
echo ''
echo " Try ${0##*/} [-y] --clean-distfiles for a full cleanup"
@@ -1910,13 +1922,13 @@ set_distfiles_and_subdir () {
}
delete_stale_distfiles () {
- # Global: distfiles port_subdir distfiles_checked delete_all
+ # Global: distfiles_checked
local file answer
set_distfiles_and_subdir $1 || return 0
distfiles_checked=':'
- # If these two match, it means that the distfiles in the +CONTENTS
+ # If these two match, it means that the distfiles in the
# file are the current set, so do not delete them.
if [ ! " $dist_list_files" = "$distfiles" ]; then
for file in $dist_list_files; do
@@ -1925,17 +1937,19 @@ delete_stale_distfiles () {
case "$distfiles" in
*" ${file} "*)
distfiles_checked="${distfiles_checked}${file}:"
- pm_v "===>>> Keeping current distfile: $file"
+ pm_v "===>>> Keeping current distfile: ${port_subdir}${file}"
continue ;; # Do not delete current version
esac
+ check_di_files $file && continue
+
if [ -n "$ALWAYS_SCRUB_DISTFILES" -o -n "$delete_all" ]; then
- echo "===>>> Deleting stale distfile: $file"
+ echo "===>>> Deleting stale distfile: ${port_subdir}${file}"
pm_unlink $file
continue
fi
- get_answer_g n y "\n===>>> Delete $file? y/n"
+ get_answer_g n y "\n===>>> Delete ${port_subdir}${file}? y/n"
case "$?" in
1) pm_unlink $file ;;
0) distfiles_checked="${distfiles_checked}${file}:" ;;
@@ -1944,7 +1958,7 @@ delete_stale_distfiles () {
fi
# Eventually we will hide this behind an "aggressive distfile purge"
- # flag, but until the DISTFILE stuff is well populated in PORT_DBDIR,
+ # flag, but until the DISTFILE stuff is well populated
# keep doing it both ways.
for file in $distfiles $dist_list_files; do
find_and_delete_distfiles ${file%[-]*}-[0-9]
@@ -1972,16 +1986,16 @@ delete_all_distfiles () {
answer=y
if [ -z "$ALWAYS_SCRUB_DISTFILES" ]; then
- echo "===>>> However, the list of files in $dist_list"
+ echo "===>>> However, the list of files from $dist_list"
get_answer_g n y " should be current. Delete the files on this list? y/n"
case "$?" in 0) answer=n ;; esac
fi
case "$answer" in
[yY]) for f in $dist_list_files; do
- if [ -f "${DISTDIR}${f}" ]; then
- echo " Deleting ${DISTDIR}${f}"
- /bin/unlink ${DISTDIR}${f}
+ if [ -f "$f" ]; then
+ echo " Deleting ${port_subdir}${f}"
+ /bin/unlink $f
fi
done ;;
esac
@@ -2094,7 +2108,7 @@ if [ -n "$EXPUNGE" ]; then
fi
[ -n "$BACKUP" ] && { init_packages ; pm_pkg_create $pbu $EXPUNGE; }
- [ -z "$DONT_SCRUB_DISTFILES" ] && { delete_all_distfiles $origin; delete_dist_list; }
+ [ -z "$DONT_SCRUB_DISTFILES" ] && delete_all_distfiles $origin
echo "===>>> Running pkg_delete -f $EXPUNGE"
pm_pkg_delete_s -f $EXPUNGE || fail 'pkg_delete failed'
@@ -2126,12 +2140,12 @@ if [ -n "$CLEAN_STALE" ]; then
continue
fi
- pkg_info $iport
+ echo '' ; pkg_info $iport
get_answer_yn n "\t===>>> ${iport} is no longer depended on, delete"
case "$?" in
0) [ -n "$BACKUP" ] && { init_packages ; pm_pkg_create $pbu $iport; }
- [ -z "$DONT_SCRUB_DISTFILES" ] && { delete_all_distfiles $origin; delete_dist_list; }
+ [ -z "$DONT_SCRUB_DISTFILES" ] && delete_all_distfiles $origin
echo "===>>> Running pkg_delete -f $iport"
pm_pkg_delete_s -f $iport || fail 'pkg_delete failed'
@@ -2566,7 +2580,7 @@ dependency_check () {
if [ -n "$SHOW_WORK" ]; then
safe_exit
elif [ -n "$PM_FIRST_PASS" -a -z "$PM_PACKAGES" ]; then
- echo "===>>> Initial dependency check complete for $portdir"
+ echo -e "===>>> Initial dependency check complete for $portdir\n"
else
echo "===>>> Dependency check complete for $portdir"
local deps ; deps="(${dep_of_deps}/${num_of_deps})"
@@ -3266,8 +3280,7 @@ if [ -z "$PM_INDEX_ONLY" -a -z "$PM_BUIL
# PATCHFILES may get added after the first pass, but we want to
# do as much of this as we can, as early as we can, and
# patch files are usually small anyway.
- distfiles=`pm_make -V ALLFILES | sed -e 's# *$##g'`
- [ -n "$distfiles" ] && distfiles=" ${distfiles} "
+ make_distfiles
# Make sure that different ports using the same distfiles
# do not clobber each other's fetchs
@@ -3728,6 +3741,7 @@ if [ -n "$upg_port" -o -n "$ro_upg_port"
pm_pkg_delete_s -f $upg_port
fi
+ echo ''
pm_cd_pd $portdir
fi
@@ -3845,23 +3859,14 @@ fi
unset temp
# This will serve for *delete*distfiles() as well
-[ -z "$use_package" ] && distfiles=`pm_make -V ALLFILES | sed -e 's# *$##g'`
+[ -z "$use_package" ] && make_distfiles
if [ -n "$distfiles" ]; then
- distfiles=" ${distfiles} "
-
# Implement storage of distfile information in the way that
# it will (hopefully, soon?) be implemented in bsd.port.mk
# See http://www.freebsd.org/cgi/query-pr.cgi?pr=106483
- dist_list=`pm_make_b -V OPTIONSFILE`
- dist_list="${dist_list%options}distfiles"
- if [ ! -d "${dist_list%/distfiles}" ]; then
- pm_sv Creating ${dist_list%/distfiles}
- pm_mkdir_s ${dist_list%/distfiles}
- fi
+ dist_list="${pdb}/${new_port}/distfiles"
- ds=`pm_make -V DIST_SUBDIR`
- [ -n "$ds" ] && ds="${ds}/"
- port_subdir="${DISTDIR}${ds}" # Also for *delete*distfiles()
+ make_port_subdir
if [ -s distinfo ]; then
distinfo=distinfo
@@ -3873,18 +3878,25 @@ if [ -n "$distfiles" ]; then
pm_mktemp dist_list ; dist_list_temp=$pm_mktemp_file
echo '# Added by portmaster' > $dist_list_temp
for file in $distfiles; do
- size=`grep "^SIZE (${ds}${file})" $distinfo`
- sha256=`grep "^SHA256 (${ds}${file})" $distinfo`
- echo "DISTFILE:${ds}${file}:SIZE=${size##* }:SHA256=${sha256##* }" \
- >> $dist_list_temp
+ while read line ; do
+ case "$line" in
+ SHA256\ \(${port_subdir}${file}\)*) sha256=${line##* } ;;
+ SIZE\ \(${port_subdir}${file}\)*)
+ [ -n "$sha256" ] || fail "$distinfo is out of order"
+ echo "DISTFILE:${port_subdir}${file}:SIZE=${line##* }:SHA256=${sha256}" \
+ >> $dist_list_temp ; break ;;
+ esac
+ done < $distinfo
+ unset sha256
+
# Make sure any new distfiles get added to the list
[ -n "$DI_FILES" -a ! "$$" -eq "$PM_PARENT_PID" ] &&
- echo "${ds}$file" >> $DI_FILES
+ echo "${port_subdir}${file}" >> $DI_FILES
done
- pm_sv Installing $dist_list
+ pm_sv "Installing $dist_list\n"
pm_install_s $dist_list_temp $dist_list
- /bin/unlink $dist_list_temp ; unset ds dist_list_temp
+ /bin/unlink $dist_list_temp ; unset distinfo dist_list_temp file line
fi
if [ -n "$use_package" ]; then
@@ -3924,7 +3936,7 @@ fi
check_dependency_files $portdir $new_port
if [ -s "$grep_deps" ]; then
- echo "===>>> Updating dependency entry for $new_port in each dependent port"; pm_v
+ echo -e "===>>> Updating dependency entry for $new_port in each dependent port\n"
while read d_port; do
pm_v "===>>> $d_port"
dp_cont=$pdb/$d_port/+CONTENTS
@@ -3942,7 +3954,7 @@ if [ -s "$grep_deps" ]; then
done < $grep_deps
unset d_port dp_cont do_update
- update_required_by $new_port
+ update_required_by $new_port ; pm_v
fi
if [ -n "$upg_port" ]; then
@@ -3973,7 +3985,6 @@ if [ -z "$DONT_SCRUB_DISTFILES" ]; then
delete_stale_distfiles $portdir
if [ -n "$ro_opd" ]; then
delete_all_distfiles $ro_opd
- delete_dist_list
fi
fi
More information about the svn-src-user
mailing list