bin/96288: portsnap: servers failover
Auster Vl.
lrou at x.ua
Mon Apr 24 22:10:19 UTC 2006
>Number: 96288
>Category: bin
>Synopsis: portsnap: servers failover
>Confidential: no
>Severity: non-critical
>Priority: low
>Responsible: freebsd-bugs
>State: open
>Quarter:
>Keywords:
>Date-Required:
>Class: change-request
>Submitter-Id: current-users
>Arrival-Date: Mon Apr 24 22:10:17 GMT 2006
>Closed-Date:
>Last-Modified:
>Originator: Auster Vl.
>Release: FreeBSD 6.0-RELEASE-p7 i386
>Organization:
>Environment:
System: FreeBSD 6.0-RELEASE-p7 i386
>Description:
You can get the error indicates then connection broken and return,
when first portsnap server goes down.
If the first server goes down, portsnap will automatically switch
to the next server for failover.
>How-To-Repeat:
Example with broken portsnap2.freebsd.org lines:
# /usr/sbin/portsnap fetch
Looking up portsnap.FreeBSD.org mirrors... using portsnap2.FreeBSD.org.
Fetching snapshot tag... fetch: http://portsnap2.FreeBSD.org./latest.ssl: Operation timed out
failed.
#
# /tmp/portsnap fetch
Looking up portsnap.FreeBSD.org mirrors... using portsnap2.FreeBSD.org portsnap1.FreeBSD.org
Fetching snapshot tag from portsnap2.FreeBSD.org ... failed.
Fetching snapshot tag from portsnap1.FreeBSD.org ... done.
Fetching snapshot metadata from portsnap1.FreeBSD.org ... done.
Updating from ÐÏÎÅÄÅÌØÎÉË, 24 ÁÐÒÅÌÑ 2006 Ç. 06:14:30 (EEST) to ÐÏÎÅÄÅÌØÎÉË, 24 ÁÐÒÅÌÑ 2006 Ç. 22:27:53 (EEST).
Fetching 4 metadata patches from portsnap1.FreeBSD.org.. done.
Applying metadata patches... done.
Fetching 69 patches from portsnap1.FreeBSD.org....10....20....30....40....50....60.... done.
Applying patches... done.
Fetching 1 new ports or files from portsnap1.FreeBSD.org ... done.
#
[restore previous db/portsnap]
# /tmp/portsnap -r fetch
Looking up portsnap.FreeBSD.org mirrors... using portsnap2.FreeBSD.org portsnap1.FreeBSD.org
Fetching snapshot tag from portsnap2.FreeBSD.org ... failed.
Fetching snapshot tag from portsnap1.FreeBSD.org ... done.
Fetching snapshot metadata from portsnap2.FreeBSD.org ... failed.
Fetching snapshot metadata from portsnap1.FreeBSD.org ... done.
Updating from ÐÏÎÅÄÅÌØÎÉË, 24 ÁÐÒÅÌÑ 2006 Ç. 06:14:30 (EEST) to ÐÏÎÅÄÅÌØÎÉË, 24 ÁÐÒÅÌÑ 2006 Ç. 22:27:53 (EEST).
Fetching 4 metadata patches from portsnap2.FreeBSD.org failed.
Fetching 4 metadata patches from portsnap1.FreeBSD.org.. done.
Applying metadata patches... done.
Fetching 69 patches from portsnap2.FreeBSD.org failed.
Fetching 69 patches from portsnap1.FreeBSD.org....10....20....30....40....50....60.... done.
Applying patches... done.
Fetching 1 new ports or files from portsnap2.FreeBSD.org ... failed.
Fetching 1 new ports or files from portsnap1.FreeBSD.org ... done.
#
>Fix:
for example, please recheck.
% diff -u /usr/sbin/portsnap /tmp/portsnap
--- /usr/sbin/portsnap
+++ /tmp/portsnap
@@ -45,6 +45,8 @@
-k KEY -- Trust an RSA key with SHA256 hash of KEY
-p portsdir -- Location of uncompressed ports tree
(default: /usr/ports/)
+ -r -- Do not reorder servernames.
+ (default: true)
-s server -- Server from which to fetch updates.
(default: portsnap.FreeBSD.org)
path -- Extract only parts of the tree starting with the given
@@ -81,6 +83,8 @@
DDSTATS=""
INDEXONLY=""
SERVERNAME=""
+ SERVERLIST=""
+ REORDER_SERVERLIST="YES"
}
# Parse the command line
@@ -132,6 +136,9 @@
if [ ! -z "${SERVERNAME}" ]; then usage; fi
shift; SERVERNAME="$1"
;;
+ -r)
+ REORDER_SERVERLIST=""
+ ;;
cron | extract | fetch | update)
COMMANDS="${COMMANDS} $1"
;;
@@ -286,6 +293,16 @@
}
+reorder_serverlist() {
+ [ "$REORDER_SERVERLIST" != "YES" ] && return
+ while [ $# -gt 0 ]; do
+ _item="$1"; shift
+ SERVERLIST=`echo $SERVERLIST | sed "s,$_item,,"`
+ SERVERLIST="$SERVERLIST $_item"
+ SERVERLIST="${SERVERLIST# }"
+ done
+}
+
#### Core functionality -- the actual work gets done here
# Use an SRV query to pick a server. If the SRV query doesn't provide
@@ -350,16 +367,17 @@
SRV_W=`echo $X | cut -f 2 -d ' '`
SRV_W=$(($SRV_W + $SRV_W_ADD))
if [ $SRV_RND -le $SRV_W ]; then
- SERVERNAME=`echo $X | cut -f 3 -d ' '`
- break
+ SERVERLIST="`echo ${X%.} | cut -f 3 -d ' '` ${SERVERLIST}"
else
SRV_RND=$(($SRV_RND - $SRV_W))
+ SERVERLIST="${SERVERLIST} `echo $X | cut -f 3 -d ' '`"
fi
;;
esac
done < serverlist
- echo " using ${SERVERNAME}"
+ SERVERLIST="${SERVERLIST# }"
+ echo " using ${SERVERLIST}"
}
# Check that we have a public key with an appropriate hash, or
@@ -369,12 +387,16 @@
return
fi
- echo -n "Fetching public key... "
- rm -f pub.ssl
- fetch ${QUIETFLAG} http://${SERVERNAME}/pub.ssl \
- 2>${QUIETREDIR} || true
- if ! [ -r pub.ssl ]; then
+ for X in ${SERVERLIST}; do
+ rm -f pub.ssl
+ echo -n "Fetching public key from ${X} ... "
+ fetch ${QUIETFLAG} http://${X}/pub.ssl \
+ 2>${QUIETREDIR} && break
echo "failed."
+ reorder_serverlist "${X}"
+ done
+ if ! [ -r pub.ssl ]; then
+ echo "Fetching public key failed."
return 1
fi
if ! [ `${SHA256} -q pub.ssl` = ${KEYPRINT} ]; then
@@ -387,13 +409,16 @@
# Fetch a snapshot tag
fetch_tag() {
- rm -f snapshot.ssl tag.new
-
- echo ${NDEBUG} "Fetching snapshot tag... "
- fetch ${QUIETFLAG} http://${SERVERNAME}/$1.ssl
- 2>${QUIETREDIR} || true
- if ! [ -r $1.ssl ]; then
+ for X in ${SERVERLIST}; do
+ rm -f snapshot.ssl tag.new
+ echo ${NDEBUG} "Fetching snapshot tag from ${X} ... "
+ fetch ${QUIETFLAG} http://${X}/$1.ssl \
+ 2>${QUIETREDIR} && break
echo "failed."
+ reorder_serverlist "${X}"
+ done
+ if ! [ -r $1.ssl ]; then
+ echo "Fetching snapshot tag failed."
return 1
fi
@@ -466,11 +491,19 @@
# Fetch snapshot metadata file
fetch_metadata() {
- rm -f ${SNAPSHOTHASH} tINDEX.new
- echo ${NDEBUG} "Fetching snapshot metadata... "
- fetch ${QUIETFLAG} http://${SERVERNAME}/t/${SNAPSHOTHASH}
- 2>${QUIETREDIR} || return
+ for X in ${SERVERLIST}; do
+ rm -f ${SNAPSHOTHASH} tINDEX.new
+ echo ${NDEBUG} "Fetching snapshot metadata from ${X} ... "
+ fetch ${QUIETFLAG} http://${X}/t/${SNAPSHOTHASH} \
+ 2>${QUIETREDIR} && break
+ echo "failed."
+ reorder_serverlist "${X}"
+ done
+ if ! [ -r ${SNAPSHOTHASH} ]; then
+ echo ${NDEBUG} "Fetching snapshot metadata failed"
+ return
+ fi
if [ `${SHA256} -q ${SNAPSHOTHASH}` != ${SNAPSHOTHASH} ]; then
echo "snapshot metadata corrupt."
return 1
@@ -554,14 +587,23 @@
fetch_metadata || return 1
fetch_metadata_sanity || return 1
- rm -f ${SNAPSHOTHASH}.tgz
- rm -rf snap/
-
# Don't ask fetch(1) to be quiet -- downloading a snapshot of ~ 35MB will
# probably take a while, so the progrees reports that fetch(1) generates
# will be useful for keeping the users' attention from drifting.
- echo "Fetching snapshot generated at `date -r ${SNAPSHOTDATE}`:"
- fetch http://${SERVERNAME}/s/${SNAPSHOTHASH}.tgz || return 1
+ for X in ${SERVERLIST}; do
+ rm -f ${SNAPSHOTHASH}.tgz
+ rm -rf snap/
+
+ echo "Fetching snapshot generated at `date -r ${SNAPSHOTDATE}` from ${X}:"
+ fetch http://${X}/s/${SNAPSHOTHASH}.tgz && break
+ echo "failed."
+ reorder_serverlist "${X}"
+ done
+
+ if ! [ -r ${SNAPSHOTHASH}.tgz ]; then
+ echo "Fetching snapshot generated at `date -r ${SNAPSHOTDATE}` failed."
+ return 1
+ fi
echo -n "Extracting snapshot... "
tar -xzf ${SNAPSHOTHASH}.tgz snap/ || return 1
@@ -612,13 +654,30 @@
fetch_make_patchlist > patchlist
# Attempt to fetch metadata patches
- echo -n "Fetching `wc -l < patchlist | tr -d ' '` "
- echo ${NDEBUG} "metadata patches.${DDSTATS}"
- tr '|' '-' < patchlist |
- lam -s "tp/" - -s ".gz" |
- xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \
- 2>${STATSREDIR} | fetch_progress
- echo "done."
+ patchlist_l=`wc -l < patchlist | tr -d ' '`
+ [ ${patchlist_l} -ne 0 ] && {
+ for X in ${SERVERLIST}; do
+ echo -n "Fetching ${patchlist_l} "
+ echo ${NDEBUG} "metadata patches${DDSTATS} from ${X}"
+ tr '|' '-' < patchlist |
+ lam -s "tp/" - -s ".gz" |
+ xargs ${XARGST} ${PHTTPGET} ${X} \
+ 2>${STATSREDIR} | fetch_progress
+
+ br_flag=no
+ while read LINE; do
+ A=`echo ${LINE} | cut -f 1 -d '|'`
+ B=`echo ${LINE} | cut -f 2 -d '|'`
+ [ -f "${A}-${B}.gz" ] && continue
+ br_flag="YES"
+ echo "failed."
+ reorder_serverlist "${X}"
+ break
+ done < patchlist 2>${QUIETREDIR}
+ [ "$br_flag" = "YES" ] && continue
+ echo "done."
+ break
+ done
# Attempt to apply metadata patches
echo -n "Applying metadata patches... "
@@ -638,6 +697,7 @@
rm -f diff OLD NEW ${X}-${Y}.gz ptmp
done < patchlist 2>${QUIETREDIR}
echo "done."
+ }
# Update metadata without patches
join -t '|' -v 2 tINDEX tINDEX.new |
@@ -647,11 +707,18 @@
echo ${Y};
fi
done > filelist
- echo -n "Fetching `wc -l < filelist | tr -d ' '` "
- echo ${NDEBUG} "metadata files... "
- lam -s "f/" - -s ".gz" < filelist |
- xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \
- 2>${QUIETREDIR}
+
+ filelist_l=`wc -l < filelist | tr -d ' '`
+ [ ${filelist_l} -ne 0 ] && {
+ for X in ${SERVERLIST}; do
+ echo -n "Fetching ${filelist_l} "
+ echo ${NDEBUG} "metadata files from ${X} ... "
+ lam -s "f/" - -s ".gz" < filelist |
+ xargs ${XARGST} ${PHTTPGET} ${X} \
+ 2>${QUIETREDIR} && break
+ echo "failed."
+ reorder_serverlist "${X}"
+ done
while read Y; do
if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then
@@ -662,6 +729,7 @@
fi
done < filelist
echo "done."
+ }
# Extract the index
gunzip -c files/`look INDEX tINDEX.new |
@@ -673,12 +741,29 @@
fetch_make_patchlist > patchlist
# Attempt to fetch ports patches
- echo -n "Fetching `wc -l < patchlist | tr -d ' '` "
- echo ${NDEBUG} "patches.${DDSTATS}"
- tr '|' '-' < patchlist | lam -s "bp/" - |
- xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \
- 2>${STATSREDIR} | fetch_progress
- echo "done."
+ patchlist_l=`wc -l < patchlist | tr -d ' '`
+ [ ${patchlist_l} -ne 0 ] && {
+ for X in ${SERVERLIST}; do
+ echo -n "Fetching ${patchlist_l} "
+ echo ${NDEBUG} "patches${DDSTATS} from ${X}"
+ tr '|' '-' < patchlist | lam -s "bp/" - |
+ xargs ${XARGST} ${PHTTPGET} ${X} \
+ 2>${STATSREDIR} | fetch_progress
+
+ br_flag=no
+ while read LINE; do
+ A=`echo ${LINE} | cut -f 1 -d '|'`
+ B=`echo ${LINE} | cut -f 2 -d '|'`
+ [ -f "${A}-${B}" ] && continue
+ br_flag="YES"
+ echo "failed."
+ reorder_serverlist "${X}"
+ break
+ done < patchlist 2>${QUIETREDIR}
+ [ "$br_flag" = "YES" ] && continue
+ echo "done."
+ break
+ done
# Attempt to apply ports patches
echo -n "Applying patches... "
@@ -695,6 +780,7 @@
rm -f diff OLD NEW ${X}-${Y}
done < patchlist 2>${QUIETREDIR}
echo "done."
+ }
# Update ports without patches
join -t '|' -v 2 INDEX INDEX.new |
@@ -704,11 +790,18 @@
echo ${Y};
fi
done > filelist
- echo -n "Fetching `wc -l < filelist | tr -d ' '` "
- echo ${NDEBUG} "new ports or files... "
- lam -s "f/" - -s ".gz" < filelist |
- xargs ${XARGST} ${PHTTPGET} ${SERVERNAME} \
- 2>${QUIETREDIR}
+
+ filelist_l=`wc -l < filelist | tr -d ' '`
+ [ ${filelist_l} -ne 0 ] && {
+ for X in ${SERVERLIST}; do
+ echo -n "Fetching ${filelist_l} "
+ echo ${NDEBUG} "new ports or files from ${X} ... "
+ lam -s "f/" - -s ".gz" < filelist |
+ xargs ${XARGST} ${PHTTPGET} ${X} \
+ 2>${QUIETREDIR} && break
+ echo "failed."
+ reorder_serverlist "${X}"
+ done
while read Y; do
if [ `gunzip -c < ${Y}.gz | ${SHA256} -q` = ${Y} ]; then
@@ -719,6 +812,7 @@
fi
done < filelist
echo "done."
+ }
# Remove files which are no longer needed
cut -f 2 -d '|' tINDEX INDEX | sort > oldfiles
>Release-Note:
>Audit-Trail:
>Unformatted:
More information about the freebsd-bugs
mailing list