svn commit: r247750 - in user/cperciva: . portsnap-build portsnap-build/s
Colin Percival
cperciva at FreeBSD.org
Mon Mar 4 01:56:04 UTC 2013
Author: cperciva
Date: Mon Mar 4 01:56:00 2013
New Revision: 247750
URL: http://svnweb.freebsd.org/changeset/base/247750
Log:
Add Portsnap build code.
To set up portsnap builds:
1. Edit build.conf,
2. Run setup.sh and follow the instructions it prints,
3. Run keygen.sh and follow the instructions it prints.
After that,
# build.sh snap
does a portsnap build and generates a snapshot tarball (these are used to
provide a faster bootstrap than downloading 23000+ individual files),
# build.sh update
does a portsnap build without a snapshot tarball, and
# upload.sh
uploads files over SSH to the target host.
Alternatively,
# loop.sh &
will do a continuous loop of builds and uploads with one snapshot per day.
Added:
user/cperciva/
user/cperciva/portsnap-build/
user/cperciva/portsnap-build/build.conf (contents, props changed)
user/cperciva/portsnap-build/build.sh (contents, props changed)
user/cperciva/portsnap-build/keygen.sh (contents, props changed)
user/cperciva/portsnap-build/loop.sh (contents, props changed)
user/cperciva/portsnap-build/releasesnap.sh (contents, props changed)
user/cperciva/portsnap-build/s/
user/cperciva/portsnap-build/s/alias-all.sh (contents, props changed)
user/cperciva/portsnap-build/s/alias-index.sh (contents, props changed)
user/cperciva/portsnap-build/s/describes-err.sh (contents, props changed)
user/cperciva/portsnap-build/s/describes-fallback.sh (contents, props changed)
user/cperciva/portsnap-build/s/describes-icbm.sh (contents, props changed)
user/cperciva/portsnap-build/s/describes-run.sh (contents, props changed)
user/cperciva/portsnap-build/s/describes-warn.sh (contents, props changed)
user/cperciva/portsnap-build/s/fileset-addfiles.sh (contents, props changed)
user/cperciva/portsnap-build/s/fileset-findnew-cleanup.sh (contents, props changed)
user/cperciva/portsnap-build/s/fileset-findnew.sh (contents, props changed)
user/cperciva/portsnap-build/s/fileset-mergedb.sh (contents, props changed)
user/cperciva/portsnap-build/s/fileset-mirrorlists.sh (contents, props changed)
user/cperciva/portsnap-build/s/fileset-mkpatches.sh (contents, props changed)
user/cperciva/portsnap-build/s/fileset-prune.sh (contents, props changed)
user/cperciva/portsnap-build/s/fileset-publish-tindex.sh (contents, props changed)
user/cperciva/portsnap-build/s/fileset-publish.sh (contents, props changed)
user/cperciva/portsnap-build/s/fileset-snap.sh (contents, props changed)
user/cperciva/portsnap-build/s/svn-getrev.sh (contents, props changed)
user/cperciva/portsnap-build/s/treesnap-build.sh (contents, props changed)
user/cperciva/portsnap-build/s/treesnap-index.sh (contents, props changed)
user/cperciva/portsnap-build/s/treesnap-mktars-all.sh (contents, props changed)
user/cperciva/portsnap-build/s/treesnap-mktars.sh (contents, props changed)
user/cperciva/portsnap-build/s/treesnap-publishsigs.sh (contents, props changed)
user/cperciva/portsnap-build/s/treesnap-sign.sh (contents, props changed)
user/cperciva/portsnap-build/setup.sh (contents, props changed)
user/cperciva/portsnap-build/upload.sh (contents, props changed)
Added: user/cperciva/portsnap-build/build.conf
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/portsnap-build/build.conf Mon Mar 4 01:56:00 2013 (r247750)
@@ -0,0 +1,68 @@
+# SVN repository
+export REPO=svn://svn.freebsd.org/ports
+
+# tar(1) utility for building tarballs
+export TAR=`which tar`
+
+# sizes of memory disks
+export PORTSMDSIZE=500M
+export SNAPMDSIZE=300M
+export JAILMDSIZE=100M
+export TMPMDSIZE=100M
+
+# number of jobs to run in parallel
+export JNUM=16
+
+# world tarball to use for INDEX building
+export WORLDTAR=`pwd`/world.tar
+
+# file listing "aliases" for tarballs to allow widespread changes (e.g.,
+# switching from cvs to svn, or between versions of tar) to be reverted.
+# NOTE: This is used by freebsd.org portsnap builds but is unlikely to
+# be needed by anyone else.
+# export ALIASFILE=`pwd`/ALIAS
+
+# internal work directory
+export STATEDIR=`pwd`/state
+
+# From: and To: for INDEX-breakage notifications
+export INDEXMAIL_FROM="your at email.address"
+export INDEXMAIL_TO="your at email.address"
+
+# From: and To: for build status emails
+export BUILDMAIL_FROM="your at email.address"
+export BUILDMAIL_TO="your at email.address"
+
+# Duration to keep port tarballs and generate binary patches
+export MAXAGE_DATA=2592000
+
+# Duration to keep metadata files on mirrors and generate patches
+export MAXAGE_META=691200
+
+# Duration to keep "extra" files (tags and snapshot tarballs) around
+export MAXAGE_EXTRA=172800
+
+# Describes files to build (on HEAD)
+export DESCRIBES_BUILD="7 8 9"
+
+# Describes files to publish (on HEAD)
+export DESCRIBES_PUBLISH="4 5 6 7 8 9"
+
+# Keys to sign builds with (get this from keygen.sh)
+export SIGNKEYS="XXXX"
+
+# Key to generate backwards-compatible signature with (until new
+# multi-signature functionality is added to portsnap in mid-2013).
+export SIGNKEYS_PRIMARY="XXXX"
+
+# SSH key for uploading bits
+UPLOAD_KEY=/root/id_dsa_upload
+
+# Account to ssh into for uploading bits
+UPLOAD_ACCT=ssh-user at ssh-host
+
+# Directory (relative to the home directory) to extract uploaded bits into
+UPLOAD_DIR=htmldocs
+
+# keep this here
+export PORTSNAP_BUILD_CONF_READ=1
Added: user/cperciva/portsnap-build/build.sh
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/portsnap-build/build.sh Mon Mar 4 01:56:00 2013 (r247750)
@@ -0,0 +1,98 @@
+#!/bin/sh -e
+set -e
+
+# usage: sh -e build.sh MODE
+MODE=$1
+
+# Sanity-check input
+if ! [ "${MODE}" = "update" ] && ! [ "${MODE}" = "snap" ]; then
+ echo "usage: sh -e build.sh (update|snap)"
+ exit 1
+fi
+
+# Load configuration
+. build.conf
+
+# Create working direcories
+WORKDIR=${STATEDIR}/work
+SNAPDIR=${WORKDIR}/snap
+TMPDIR=${WORKDIR}/tmp
+SIGDIR=${WORKDIR}/sigs
+mkdir ${WORKDIR} ${SNAPDIR} ${TMPDIR} ${SIGDIR}
+
+# Record when we're starting
+SNAPDATE=`date "+%s"`
+
+# Get the latest revision # on the tree
+NEWREV=`sh -e s/svn-getrev.sh head`
+
+# Create a memory disk for holding the snapshot files.
+SNAPMD=`mdconfig -a -t swap -s ${SNAPMDSIZE} -n`
+newfs -O 1 -n /dev/md${SNAPMD} >/dev/null
+mount /dev/md${SNAPMD} ${SNAPDIR}
+
+# Build a snapshot
+sh -e s/treesnap-build.sh head@${NEWREV} "${DESCRIBES_BUILD}" \
+ ${TMPDIR} ${SNAPDIR}
+
+# Replace tarballs with "aliased" tarballs
+if ! [ -z ${ALIASFILE} ]; then
+ if [ ${MODE} = "snap" ]; then
+ sh -e s/alias-all.sh ${SNAPDIR} ${STATEDIR}/fileset/oldfiles \
+ ${ALIASFILE} ${WORKDIR}
+ else
+ sh -e s/alias-index.sh ${SNAPDIR} ${STATEDIR}/fileset/oldfiles \
+ ${ALIASFILE} ${WORKDIR}
+ fi
+fi
+
+# Send emails if INDEX was broken or fixed
+sh -e s/describes-warn.sh ${SNAPDIR} ${NEWREV} ${STATEDIR}/describes \
+ "${DESCRIBES_BUILD}"
+
+# Use old DESCRIBE files if the latest ones didn't build
+sh -e s/describes-fallback.sh ${SNAPDIR} ${STATEDIR}/describes \
+ "${DESCRIBES_PUBLISH}"
+
+# Collect metadata
+sh -e s/treesnap-index.sh ${SNAPDIR} ${WORKDIR}/INDEX ${WORKDIR}/tINDEX
+
+# Add these files to our (overlapping) set of snapshots
+sh -e s/fileset-snap.sh ${SNAPDATE} ${STATEDIR}/fileset ${SNAPDIR} \
+ ${WORKDIR}/INDEX ${WORKDIR}/tINDEX ${STATEDIR}/stage ${TMPDIR}
+
+# Sign the tree
+sh -e s/treesnap-sign.sh ${SNAPDATE} ${WORKDIR}/tINDEX ${SIGDIR}
+
+# Publish signatures as the "latest" build
+sh -e s/treesnap-publishsigs.sh ${SIGDIR} ${STATEDIR}/stage latest
+
+# Build a snapshot tarball if necessary
+if [ ${MODE} = "snap" ]; then
+ echo "`date`: Building snapshot tarball"
+ SNAPSHOTHASH=`sha256 -q ${WORKDIR}/tINDEX`
+ tar -czf ${STATEDIR}/stage/s/${SNAPSHOTHASH}.tgz -C ${WORKDIR} snap
+ echo "${SNAPDATE}|s/${SNAPSHOTHASH}.tgz" \
+ >> ${STATEDIR}/fileset/extradb
+ sh -e s/treesnap-publishsigs.sh ${SIGDIR} ${STATEDIR}/stage snapshot
+fi
+
+# Delete signatures
+rm ${SIGDIR}/*.ssl
+
+# Unmount and delete the snapshot disk
+umount /dev/md${SNAPMD}
+mdconfig -d -u ${SNAPMD}
+
+# Delete indexes
+rm ${WORKDIR}/INDEX ${WORKDIR}/tINDEX
+
+# Delete temporary directories
+rmdir ${SIGDIR} ${TMPDIR} ${SNAPDIR} ${WORKDIR}
+
+# Publish file lists for mirroring
+sh -e s/fileset-mirrorlists.sh ${STATEDIR}/fileset ${STATEDIR}/stage
+
+# Report sucess
+echo "`date`: Finished"
+
Added: user/cperciva/portsnap-build/keygen.sh
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/portsnap-build/keygen.sh Mon Mar 4 01:56:00 2013 (r247750)
@@ -0,0 +1,37 @@
+#!/bin/sh -e
+set -e
+
+# usage: sh -e keygen.sh
+
+# Load configuration
+. build.conf
+
+# Create temporary working space
+mkdir ${STATEDIR}/work
+
+# Create key
+openssl genrsa -F4 2048 > ${STATEDIR}/work/priv.ssl
+openssl rsa -in ${STATEDIR}/work/priv.ssl -pubout > ${STATEDIR}/work/pub.ssl
+
+# Compute key hash
+KEYHASH=`sha256 -q ${STATEDIR}/work/pub.ssl`
+
+# Move files into their permanent location
+mv ${STATEDIR}/work/priv.ssl ${STATEDIR}/keys/priv-${KEYHASH}.ssl
+mv ${STATEDIR}/work/pub.ssl ${STATEDIR}/keys/pub-${KEYHASH}.ssl
+
+# Announce the public key hash
+cat <<- EOF
+
+ A key has been generated with hash:
+ ${KEYHASH}
+
+EOF
+xargs -s 80 <<- EOF
+ Add this to the SIGNKEYS list in build.conf to start signing builds
+ with it; and set it as the KEYPRINT value in /etc/portsnap.conf on
+ systems which need to use these updates.
+EOF
+
+# Remove our temporary working directory
+rmdir ${STATEDIR}/work
Added: user/cperciva/portsnap-build/loop.sh
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/portsnap-build/loop.sh Mon Mar 4 01:56:00 2013 (r247750)
@@ -0,0 +1,47 @@
+#!/bin/sh
+set -e
+
+# Load configuration
+. build.conf
+
+# Loop until we fail or the looping is stopped administratively
+while ! [ -f failed ] && ! [ -f adminlock ]; do
+ # Figure out if this is the first build of a new day
+ touch -t `date "+%Y%m%d0000"` ${STATEDIR}/midnight
+ if ! [ ${STATEDIR}/lastsnap -nt ${STATEDIR}/midnight ]; then
+ BUILDTYPE=snap
+ touch ${STATEDIR}/lastsnap
+ else
+ BUILDTYPE=update
+ fi
+ rm ${STATEDIR}/midnight
+
+ # Send an email
+ (
+ echo "From: ${BUILDMAIL_FROM}"
+ for ADDR in ${BUILDMAIL_TO}; do
+ echo "To: ${ADDR}"
+ done
+ echo "Subject: `hostname` Portsnap build.sh ${BUILDTYPE}"
+ echo
+
+ # Do the build
+ if ! sh -e build.sh ${BUILDTYPE} 2>&1; then
+ touch failed;
+ else
+ # Upload if the build succeeded
+ sh upload.sh 2>&1 || true
+ fi
+
+ # Once a day, clean up portsnap-master
+ if [ ${BUILDTYPE} = "snap" ]; then
+ echo "`date`: Cleaning bits on portsnap-master"
+ ssh -i ${UPLOAD_KEY} ${UPLOAD_ACCT} \
+ sh portsnap-clean.sh 2>&1
+ fi
+ ) | sendmail -t
+done
+
+# Send a warning if builds stop running
+echo "Subject: `hostname` Portsnap builds no longer running!" |
+ sendmail ${BUILDMAIL_TO}
Added: user/cperciva/portsnap-build/releasesnap.sh
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/portsnap-build/releasesnap.sh Mon Mar 4 01:56:00 2013 (r247750)
@@ -0,0 +1,69 @@
+#!/bin/sh -e
+set -e
+
+# usage: sh -e releasesnap.sh TREE DESCRIBES TARBALL
+# e.g., sh -e releasesnap.sh tags/RELEASE_9_1_0 "7 8 9" portsnap.tgz
+TREE="$1"
+DESCRIBES="$2"
+TARBALL="$3"
+
+# Sanity-check input
+if [ -z "${TREE}" ] || [ -z "${DESCRIBES}" ] || [ -z "${TARBALL}" ]; then
+ echo "usage: sh -e releasesnap.sh TREE DESCRIBES TARBALL"
+ exit 1
+fi
+
+# Load configuration
+. build.conf
+
+# Get the latest revision # on the tree
+NEWREV=`sh -e s/svn-getrev.sh ${TREE}`
+
+# Create a memory disk for holding everything which will end up in
+# /var/db/portsnap. Note that for normal (head) builds we mount the disk
+# on ${SNAPDIR}; we can't do that here because we want everything to be on
+# a single filesystem so that hardlinks work.
+SNAPMD=`mdconfig -a -t swap -s ${SNAPMDSIZE} -n`
+newfs -O 1 -n /dev/md${SNAPMD} >/dev/null
+
+# Mount the memory disk
+WORKDIR=${STATEDIR}/work
+mkdir ${WORKDIR}
+mount /dev/md${SNAPMD} ${WORKDIR}
+
+# Build a snapshot
+SNAPDIR=${STATEDIR}/work/files
+mkdir ${SNAPDIR}
+sh -e s/treesnap-build.sh ${TREE}@${NEWREV} "${DESCRIBES}" ${WORKDIR} ${SNAPDIR}
+
+# Replace tarballs with "aliased" tarballs
+if ! [ -z ${ALIASFILE} ]; then
+ sh -e s/alias-all.sh ${SNAPDIR} ${STATEDIR}/fileset/oldfiles \
+ ${ALIASFILE} ${WORKDIR}
+fi
+
+# Make sure we have the required describe files
+sh -e s/describes-err.sh ${SNAPDIR} "${DESCRIBES}"
+
+# Collect metadata
+sh -e s/treesnap-index.sh ${SNAPDIR} ${WORKDIR}/INDEX ${WORKDIR}/tINDEX
+
+# Hard-link compressed index file and remove the uncompressed file
+ln ${SNAPDIR}/`sha256 -q ${WORKDIR}/INDEX`.gz ${WORKDIR}/INDEX.gz
+rm ${WORKDIR}/INDEX
+
+# Create tag file
+echo "portsnap|`date "+%s"`|`sha256 -q ${WORKDIR}/tINDEX`" > ${WORKDIR}/tag
+
+# Create tarball
+tar -czf ${TARBALL} -C ${WORKDIR} tag tINDEX INDEX.gz files
+
+# Unmount and delete the snapshot disk
+umount /dev/md${SNAPMD}
+mdconfig -d -u ${SNAPMD}
+
+# Delete temporary directories
+rmdir ${WORKDIR}
+
+# Report sucess
+echo "`date`: Finished"
Added: user/cperciva/portsnap-build/s/alias-all.sh
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/portsnap-build/s/alias-all.sh Mon Mar 4 01:56:00 2013 (r247750)
@@ -0,0 +1,40 @@
+#!/bin/sh -e
+
+# No user-serviceable parts
+if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then
+ echo "Do not run $0 manually"
+ exit 1
+fi
+
+# usage: sh -e alias-all.sh SNAP OLDFILES ALIASFILE WORKDIR
+SNAP="$1"
+OLDFILES="$2"
+ALIASFILE="$3"
+WORKDIR="$4"
+
+# Report progress
+echo "`date`: Reverting files to aliased versions"
+
+# Revert index
+sort -k2 -t '|' ${SNAP}/INDEX |
+ join -1 2 -t '|' -o 1.1,1.2,2.2 - ${ALIASFILE} > ${WORKDIR}/INDEX.remap
+cut -f 1,2 -d '|' ${WORKDIR}/INDEX.remap |
+ sort > ${WORKDIR}/INDEX.remap.delete
+cut -f 1,3 -d '|' ${WORKDIR}/INDEX.remap |
+ sort > ${WORKDIR}/INDEX.remap.add
+sort ${SNAP}/INDEX |
+ comm -23 - ${WORKDIR}/INDEX.remap.delete |
+ sort -k 1,1 -t '|' - ${WORKDIR}/INDEX.remap.add > ${SNAP}/INDEX.new
+mv ${SNAP}/INDEX.new ${SNAP}/INDEX
+
+# Revert files
+cut -f 2,3 -d '|' ${WORKDIR}/INDEX.remap |
+ tr '|' ' ' | while read OLD NEW; do
+ rm ${SNAP}/${OLD}.gz
+ cp ${OLDFILES}/${NEW}.gz ${SNAP}/
+done
+
+# Clean up
+rm ${WORKDIR}/INDEX.remap
+rm ${WORKDIR}/INDEX.remap.add
+rm ${WORKDIR}/INDEX.remap.delete
Added: user/cperciva/portsnap-build/s/alias-index.sh
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/portsnap-build/s/alias-index.sh Mon Mar 4 01:56:00 2013 (r247750)
@@ -0,0 +1,33 @@
+#!/bin/sh -e
+
+# No user-serviceable parts
+if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then
+ echo "Do not run $0 manually"
+ exit 1
+fi
+
+# usage: sh -e alias-index.sh SNAP OLDFILES ALIASFILE WORKDIR
+SNAP="$1"
+OLDFILES="$2"
+ALIASFILE="$3"
+WORKDIR="$4"
+
+# Report progress
+echo "`date`: Reverting files to aliased versions"
+
+# Revert index
+sort -k2 -t '|' ${SNAP}/INDEX |
+ join -1 2 -t '|' -o 1.1,1.2,2.2 - ${ALIASFILE} > ${WORKDIR}/INDEX.remap
+cut -f 1,2 -d '|' ${WORKDIR}/INDEX.remap |
+ sort > ${WORKDIR}/INDEX.remap.delete
+cut -f 1,3 -d '|' ${WORKDIR}/INDEX.remap |
+ sort > ${WORKDIR}/INDEX.remap.add
+sort ${SNAP}/INDEX |
+ comm -23 - ${WORKDIR}/INDEX.remap.delete |
+ sort -k 1,1 -t '|' - ${WORKDIR}/INDEX.remap.add > ${SNAP}/INDEX.new
+mv ${SNAP}/INDEX.new ${SNAP}/INDEX
+
+# Clean up
+rm ${WORKDIR}/INDEX.remap
+rm ${WORKDIR}/INDEX.remap.add
+rm ${WORKDIR}/INDEX.remap.delete
Added: user/cperciva/portsnap-build/s/describes-err.sh
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/portsnap-build/s/describes-err.sh Mon Mar 4 01:56:00 2013 (r247750)
@@ -0,0 +1,19 @@
+#!/bin/sh -e
+
+# No user-serviceable parts
+if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then
+ echo "Do not run $0 manually"
+ exit 1
+fi
+
+# usage: sh -e describes-err.sh SNAP DESCRIBES
+SNAP="$1"
+DESCRIBES="$2"
+
+# Check that the required describes files exist
+for N in $DESCRIBES; do
+ if ! [ -f ${SNAP}/DESCRIBE.${N} ]; then
+ echo "DESCRIBE.${N} does not exist!"
+ exit 1
+ fi
+done
Added: user/cperciva/portsnap-build/s/describes-fallback.sh
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/portsnap-build/s/describes-fallback.sh Mon Mar 4 01:56:00 2013 (r247750)
@@ -0,0 +1,24 @@
+#!/bin/sh -e
+
+# No user-serviceable parts
+if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then
+ echo "Do not run $0 manually"
+ exit 1
+fi
+
+# usage: sh -e describes-fallback.sh SNAP DESCDIR DESCRIBES
+SNAP="$1"
+DESCDIR="$2"
+DESCRIBES="$3"
+
+# For each DESCRIBE file...
+for N in ${DESCRIBES}; do
+ # If a DESCRIBE failed...
+ if ! [ -f ${SNAP}/DESCRIBE.${N} ]; then
+ # ... use an old version ...
+ cp ${DESCDIR}/DESCRIBE.${N} ${SNAP}/DESCRIBE.${N}
+ else
+ # ... otherwise, store what we have for future reference.
+ cp ${SNAP}/DESCRIBE.${N} ${DESCDIR}/DESCRIBE.${N}
+ fi
+done
Added: user/cperciva/portsnap-build/s/describes-icbm.sh
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/portsnap-build/s/describes-icbm.sh Mon Mar 4 01:56:00 2013 (r247750)
@@ -0,0 +1,62 @@
+#!/bin/sh -e
+
+# No user-serviceable parts
+if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then
+ echo "Do not run $0 manually"
+ exit 1
+fi
+
+# usage: sh -e describes-icbm.sh GOODREV BADREV ERRFILE
+GOODREV="$1"
+BADREV="$2"
+ERRFILE="$3"
+
+# The first potentially faulty commit is GOODREV+1
+BADSTART=`expr "$GOODREV" + 1`
+
+# Standard From/To/Subject lines
+cat <<EOF
+From: ${INDEXMAIL_FROM}
+To: ${INDEXMAIL_TO}
+Subject: INDEX build breakage
+EOF
+
+# CC people who might have broken the INDEX
+jot - ${BADSTART} ${BADREV} |
+ while read REV; do
+ svn log -c ${REV} ${REPO} |
+ tail +2 |
+ head -1;
+done |
+ cut -f 2 -d '|' |
+ sort -u |
+ tr -d ' ' |
+ lam -s 'CC: ' - -s '@freebsd.org'
+
+# Blank line and build failure output
+echo
+cat ${ERRFILE}
+
+# List potentially at-fault committers (again) and SVN history
+echo
+echo "Committers on the hook (CCed):"
+jot - ${BADSTART} ${BADREV} |
+ while read REV; do
+ svn log -c ${REV} ${REPO} |
+ tail +2 |
+ head -1;
+done |
+ cut -f 2 -d '|' |
+ sort -u |
+ tr -d ' '
+echo
+echo "Latest SVN commits:"
+svn log -r ${BADSTART}:${BADREV} ${REPO}
+
+# Final message about when emails are sent
+cat <<EOF
+
+There may be different errors exposed by INDEX builds on other
+branches, but no further emails will be sent until after the
+INDEX next builds successfully on all branches.
+EOF
Added: user/cperciva/portsnap-build/s/describes-run.sh
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/portsnap-build/s/describes-run.sh Mon Mar 4 01:56:00 2013 (r247750)
@@ -0,0 +1,98 @@
+#!/bin/sh -e
+
+# No user-serviceable parts
+if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then
+ echo "Do not run $0 manually"
+ exit 1
+fi
+
+# usage: sh -e describes-run.sh PORTSDISK WORLDTAR JAILDIR OSVERSION DESCFILE
+PORTSDISK="$1"
+WORLDTAR="$2"
+JAILDIR="$3"
+OSVERSION="$4"
+DESCFILE="$5"
+
+# helper function
+findruleset () {
+ jot 0 |
+ while read N; do
+ if ! devfs rule -s ${N} show | grep -q .; then
+ echo ${N}
+ break
+ fi
+ done
+}
+
+# Create memory disks and format filesystems
+JAILMD=`mdconfig -a -t swap -s ${JAILMDSIZE} -n`
+TMPMD=`mdconfig -a -t swap -s ${TMPMDSIZE} -n`
+newfs -O 1 -n /dev/md${JAILMD} >/dev/null
+newfs -O 1 -n /dev/md${TMPMD} >/dev/null
+
+# Mount filesystems under jail mount point
+mount -o noatime,nosuid /dev/md${JAILMD} ${JAILDIR}
+mkdir ${JAILDIR}/tmp
+mount -o noatime,nosuid,noexec /dev/md${TMPMD} ${JAILDIR}/tmp
+chmod 1777 ${JAILDIR}/tmp
+mkdir ${JAILDIR}/usr ${JAILDIR}/usr/ports
+mount -o noatime,nosuid,noexec,ro ${PORTSDISK} ${JAILDIR}/usr/ports
+
+# Attach device filesystem with null, fd/*, and std*
+mkdir ${JAILDIR}/dev
+mount -t devfs devfs ${JAILDIR}/dev
+RULESET=`findruleset`
+devfs rule -s ${RULESET} add hide
+devfs rule -s ${RULESET} add path null unhide
+devfs rule -s ${RULESET} add path fd unhide
+devfs rule -s ${RULESET} add path 'fd/*' unhide
+devfs rule -s ${RULESET} add path 'std*' unhide
+devfs -m ${JAILDIR}/dev ruleset ${RULESET}
+devfs -m ${JAILDIR}/dev rule applyset
+
+# Extract world tarball
+tar -xf ${WORLDTAR} -C ${JAILDIR}
+
+# Protect against naughtiness
+mount -u -o noatime,nosuid,ro /dev/md${JAILMD}
+
+# Build the describes output
+if env - PATH=${PATH} jail -c path=${JAILDIR} host.hostname=localhost \
+ exec.jail_user=nobody exec.system_jail_user command=/bin/sh -e \
+ > ${DESCFILE} <<- EOF
+ export __MAKE_CONF=/nonexistant
+ export OSVERSION=${OSVERSION}
+ export PORTOBJFORMAT=elf
+ export INDEX_TMPDIR=/tmp
+ export WRKDIRPREFIX=/tmp
+ export BUILDING_INDEX=1
+ export LOCALBASE=/removeme/usr/local
+ export ECHO_MSG="echo >/dev/null"
+ cd /usr/ports && make describe -j ${JNUM} 1>&2
+ cd /tmp && cat INDEX* | sed -e 's/ */ /g' -e 's/| */|/g' \
+ -e 's/ *|/|/g' -e 's./removeme..g' | \
+ sed -E -e ':x' -e 's|/[^/]+/\.\.||' -e 'tx' | \
+ sort -k 1,1 -t '|' | tr -d '\r'
+EOF
+then
+ R=0
+else
+ R=1
+fi
+
+# Clean up
+umount ${JAILDIR}/dev
+umount ${JAILDIR}/tmp
+umount ${JAILDIR}/usr/ports
+umount ${JAILDIR}
+mdconfig -d -u ${JAILMD}
+mdconfig -d -u ${TMPMD}
+devfs rule -s ${RULESET} delset
+
+# Test if make_index works
+if [ ${R} = 0 ] && ! /usr/libexec/make_index ${DESCFILE} >/dev/null; then
+ R=1
+fi
+
+# Return success/failure
+exit ${R}
Added: user/cperciva/portsnap-build/s/describes-warn.sh
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/portsnap-build/s/describes-warn.sh Mon Mar 4 01:56:00 2013 (r247750)
@@ -0,0 +1,51 @@
+#!/bin/sh -e
+
+# No user-serviceable parts
+if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then
+ echo "Do not run $0 manually"
+ exit 1
+fi
+
+# usage: sh -e describes-warn.sh SNAP NEWREV DESCDIR DESCRIBES
+SNAP="$1"
+NEWREV="$2"
+DESCDIR="$3"
+DESCRIBES="$4"
+
+# Haven't found any broken describes yet
+BROKEN=0
+
+# Check if any describes broke
+for N in ${DESCRIBES}; do
+ # Did this one break?
+ if [ -f ${SNAP}/DESCRIBE.${N}.err ]; then
+ # Yep, remember this
+ BROKEN=1
+
+ # Launch an ICBM if this is new breakage
+ if [ -f ${DESCDIR}/indexok ]; then
+ sh -e s/describes-icbm.sh `cat ${DESCDIR}/indexok` \
+ ${NEWREV} ${SNAP}/DESCRIBE.${N}.err |
+ sendmail -t
+ rm ${DESCDIR}/indexok
+ fi
+
+ # Delete the error messages
+ rm ${SNAP}/DESCRIBE.${N}.err
+ fi
+done
+
+# If the tree is non-broken but was previously broken, send an email
+if [ ${BROKEN} = 0 ] && ! [ -f ${DESCDIR}/indexok ]; then
+ sendmail -t <<- EOF
+ From: ${INDEXMAIL_FROM}
+ To: ${INDEXMAIL_TO}
+ Subject: INDEX build fixed
+
+ EOF
+fi
+
+# If the tree is non-broken, record the new index-ok-at SVN #
+if [ ${BROKEN} = 0 ]; then
+ echo ${NEWREV} > ${DESCDIR}/indexok
+fi
Added: user/cperciva/portsnap-build/s/fileset-addfiles.sh
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/portsnap-build/s/fileset-addfiles.sh Mon Mar 4 01:56:00 2013 (r247750)
@@ -0,0 +1,20 @@
+#!/bin/sh -e
+
+# No user-serviceable parts
+if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then
+ echo "Do not run $0 manually"
+ exit 1
+fi
+
+# usage: sh -e fileset-addfiles.sh FSETDIR SNAP
+FSETDIR="$1"
+SNAP="$2"
+
+# Report progress
+echo "`date`: Copying files into /oldfiles/"
+
+# Copy data files -- we don't need metadata files since we
+# don't generate patches for those here
+cut -f 2 -d '|' ${FSETDIR}/filedb.news |
+ lam -s "${SNAP}/" - -s '.gz' |
+ xargs -J % cp % ${FSETDIR}/oldfiles/
Added: user/cperciva/portsnap-build/s/fileset-findnew-cleanup.sh
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/portsnap-build/s/fileset-findnew-cleanup.sh Mon Mar 4 01:56:00 2013 (r247750)
@@ -0,0 +1,13 @@
+#!/bin/sh -e
+
+# No user-serviceable parts
+if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then
+ echo "Do not run $0 manually"
+ exit 1
+fi
+
+# usage: sh -e fileset-findnew-cleanup.sh FSETDIR
+FSETDIR="$1"
+
+# Clean up new-files lists generated be fileset-findnew.sh
+rm ${FSETDIR}/filedb.news ${FSETDIR}/metadb.news
Added: user/cperciva/portsnap-build/s/fileset-findnew.sh
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/portsnap-build/s/fileset-findnew.sh Mon Mar 4 01:56:00 2013 (r247750)
@@ -0,0 +1,36 @@
+#!/bin/sh -e
+
+# No user-serviceable parts
+if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then
+ echo "Do not run $0 manually"
+ exit 1
+fi
+
+# usage: sh -e fileset-findnew.sh FSETDIR INDEX tINDEX TMP
+FSETDIR="$1"
+INDEX="$2"
+tINDEX="$3"
+TMP="$4"
+
+# Report progress
+echo "`date`: Identifying new files"
+
+# Find new port tarballs
+sort -k 3,3 -t '|' ${FSETDIR}/filedb > ${TMP}/filedb.sorted
+sort -k 2,2 -t '|' ${INDEX} |
+ join -1 3 -2 2 -t '|' -v 2 ${TMP}/filedb.sorted - |
+ sort -k 1,1 -t '|' > ${FSETDIR}/filedb.news
+
+# Report new files
+echo "New files:"
+cut -f 1 -d '|' < ${FSETDIR}/filedb.news |
+ lam -s ' ' -
+
+# Find new metadata files
+sort -k 3,3 -t '|' ${FSETDIR}/metadb > ${TMP}/metadb.sorted
+sort -k 2,2 -t '|' ${tINDEX} |
+ join -1 3 -2 2 -t '|' -v 2 ${TMP}/metadb.sorted - |
+ sort -k 1,1 -t '|' > ${FSETDIR}/metadb.news
+
+# Clean up temporary files
+rm ${TMP}/filedb.sorted ${TMP}/metadb.sorted
Added: user/cperciva/portsnap-build/s/fileset-mergedb.sh
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/portsnap-build/s/fileset-mergedb.sh Mon Mar 4 01:56:00 2013 (r247750)
@@ -0,0 +1,45 @@
+#!/bin/sh -e
+
+# No user-serviceable parts
+if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then
+ echo "Do not run $0 manually"
+ exit 1
+fi
+
+# usage: sh -e fileset-mergedb.sh FSETDIR SNAPDATE INDEX tINDEX TMP
+FSETDIR="$1"
+SNAPDATE="$2"
+INDEX="$3"
+tINDEX="$4"
+TMP="$5"
+
+# Report progress
+echo "`date`: Updating databases"
+
+# The filedb and metadb files contain lines of the form
+# FILENAME|TIMESTAMP|HASH
+# where TIMESTAMP is the *most recent* SNAPDATE for which the data
+# associated with FILENAME is HASH.gz (and has SHA256 hash HASH).
+# We use this to tell us which binary patches to generate, which
+# also tells us which old data files to keep around; the mirrors
+# use this to prune the data they publihsh and figure out which
+# metadata patches they should generate.
+
+# Update filedb using INDEX
+sed -e "s/|/|${SNAPDATE}|/" ${INDEX} |
+ sort -k 3,3 -t '|' > ${TMP}/new.dated
+sort -k 3,3 -t '|' ${FSETDIR}/filedb |
+ join -1 3 -2 3 -t '|' -v 1 - ${TMP}/new.dated |
+ sort -k 1,1 -t '|' - ${TMP}/new.dated > ${FSETDIR}/filedb.tmp
+mv ${FSETDIR}/filedb.tmp ${FSETDIR}/filedb
+
+# Update metadb using tINDEX
+sed -e "s/|/|${SNAPDATE}|/" ${tINDEX} |
+ sort -k 3,3 -t '|' > ${TMP}/new.dated
+sort -k 3,3 -t '|' ${FSETDIR}/metadb |
+ join -1 3 -2 3 -t '|' -v 1 - ${TMP}/new.dated |
+ sort -k 1,1 -t '|' - ${TMP}/new.dated > ${FSETDIR}/metadb.tmp
+mv ${FSETDIR}/metadb.tmp ${FSETDIR}/metadb
+
+# Clean up
+rm ${TMP}/new.dated
Added: user/cperciva/portsnap-build/s/fileset-mirrorlists.sh
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/portsnap-build/s/fileset-mirrorlists.sh Mon Mar 4 01:56:00 2013 (r247750)
@@ -0,0 +1,19 @@
+#!/bin/sh -e
+
+# No user-serviceable parts
+if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then
+ echo "Do not run $0 manually"
+ exit 1
+fi
+
+# usage: sh -e fileset-mirrorlists.sh FSETDIR STAGE
+FSETDIR="$1"
+STAGE="$2"
+
+# Publish file lists
+echo "`date`: Copying file lists into staging area"
+gzip -c < ${FSETDIR}/filedb > ${STAGE}/bl.gz
+gzip -c < ${FSETDIR}/metadb > ${STAGE}/tl.gz
+cut -f 2 -d '|' ${FSETDIR}/extradb |
+ sort -u |
+ gzip -c > ${STAGE}/el.gz
Added: user/cperciva/portsnap-build/s/fileset-mkpatches.sh
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/portsnap-build/s/fileset-mkpatches.sh Mon Mar 4 01:56:00 2013 (r247750)
@@ -0,0 +1,32 @@
+#!/bin/sh -e
+
+# No user-serviceable parts
+if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then
+ echo "Do not run $0 manually"
+ exit 1
+fi
+
+# usage: sh -e fileset-mkpatches.sh FSETDIR SNAP STAGE TMP
+FSETDIR="$1"
+SNAP="$2"
+STAGE="$3"
+TMP="$4"
+
+# Report progress
+echo "`date`: Building patches"
+
+# Figure out which "OLD NEW" pairs we need
+sort -k 1,1 -t '|' ${FSETDIR}/filedb |
+ join -t '|' -o 1.3,2.2 - ${FSETDIR}/filedb.news |
+ tr '|' ' ' > ${TMP}/patches
+
+# Build the patches
+while read X Y; do
+ gunzip -c ${FSETDIR}/oldfiles/${X}.gz > ${TMP}/${X}
+ gunzip -c ${SNAP}/${Y}.gz > ${TMP}/${Y}
+ bsdiff ${TMP}/${X} ${TMP}/${Y} ${STAGE}/bp/${X}-${Y}
+ rm ${TMP}/${X} ${TMP}/${Y}
+done < ${TMP}/patches
+
+# Clean up temporary file
+rm ${TMP}/patches
Added: user/cperciva/portsnap-build/s/fileset-prune.sh
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/portsnap-build/s/fileset-prune.sh Mon Mar 4 01:56:00 2013 (r247750)
@@ -0,0 +1,45 @@
+#!/bin/sh -e
+
+# No user-serviceable parts
+if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then
+ echo "Do not run $0 manually"
+ exit 1
+fi
+
+# usage: sh -e fileset-prune.sh SNAPDATE FSETDIR TMP
+SNAPDATE="$1"
+FSETDIR="$2"
+TMP="$3"
+
+# Report progress
+echo "`date`: Removing old files and database entries"
+
+# Sort list of files so we can use comm(1) on it
+sort ${FSETDIR}/filedb > ${TMP}/filedb.sorted
+
+# Find lines corresponding to files we don't want any more
+awk -F \| -v cutoff=`expr ${SNAPDATE} - ${MAXAGE_DATA}` \
+ '{ if ($2 < cutoff) { print } }' ${TMP}/filedb.sorted > ${TMP}/filedb.olds
+
+# Delete old files
+cut -f 3 -d '|' ${TMP}/filedb.olds |
+ grep -E '^[0-9a-f]{64}$' |
+ lam -s "${FSETDIR}/oldfiles/" - -s '.gz' |
+ xargs rm -f
+
+# Construct a new old-files list
+comm -23 ${TMP}/filedb.sorted ${TMP}/filedb.olds |
+ sort -k 1,1 -t '|' > ${FSETDIR}/filedb.tmp
+mv ${FSETDIR}/filedb.tmp ${FSETDIR}/filedb
+
+# Remove old lines from metadb and extradb
+echo "`date`: Removing old metadata and extra database entries"
+awk -F \| -v cutoff=`expr ${SNAPDATE} - ${MAXAGE_META}` \
+ '{ if ($2 >= cutoff) { print } }' ${FSETDIR}/metadb > ${FSETDIR}/metadb.tmp
+mv ${FSETDIR}/metadb.tmp ${FSETDIR}/metadb
+awk -F \| -v cutoff=`expr ${SNAPDATE} - ${MAXAGE_EXTRA}` \
+ '{ if ($1 >= cutoff) { print } }' ${FSETDIR}/extradb > ${FSETDIR}/extradb.tmp
+mv ${FSETDIR}/extradb.tmp ${FSETDIR}/extradb
+
+# Remove temporary files
+rm ${TMP}/filedb.sorted ${TMP}/filedb.olds
Added: user/cperciva/portsnap-build/s/fileset-publish-tindex.sh
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/portsnap-build/s/fileset-publish-tindex.sh Mon Mar 4 01:56:00 2013 (r247750)
@@ -0,0 +1,22 @@
+#!/bin/sh -e
+
+# No user-serviceable parts
+if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then
+ echo "Do not run $0 manually"
+ exit 1
+fi
+
+# usage: sh -e fileset-publish-tindex.sh FSETDIR SNAPDATE tINDEX STAGE
+FSETDIR="$1"
+SNAPDATE="$2"
+tINDEX="$3"
+STAGE="$4"
+
+# Figure out the tINDEX hash
+SNAPSHOTHASH=`sha256 -q ${tINDEX}`
+
+# Copy the tINDEX file into place in the staging directory
+cp ${tINDEX} ${STAGE}/t/${SNAPSHOTHASH}
+
+# Add to our database
+echo "${SNAPDATE}|t/${SNAPSHOTHASH}" >> ${FSETDIR}/extradb
Added: user/cperciva/portsnap-build/s/fileset-publish.sh
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/portsnap-build/s/fileset-publish.sh Mon Mar 4 01:56:00 2013 (r247750)
@@ -0,0 +1,20 @@
+#!/bin/sh -e
+
+# No user-serviceable parts
+if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then
+ echo "Do not run $0 manually"
+ exit 1
+fi
+
+# usage: sh -e fileset-publish.sh FSETDIR SNAP STAGE
+FSETDIR="$1"
+SNAP="$2"
+STAGE="$3"
+
+# Report progress
+echo "`date`: Copying files into staging area"
+
+# Copy new files
+cut -f 2 -d '|' ${FSETDIR}/filedb.news ${FSETDIR}/metadb.news |
+ lam -s "${SNAP}/" - -s '.gz' |
+ xargs -J % cp % ${STAGE}/f/
Added: user/cperciva/portsnap-build/s/fileset-snap.sh
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/portsnap-build/s/fileset-snap.sh Mon Mar 4 01:56:00 2013 (r247750)
@@ -0,0 +1,47 @@
+#!/bin/sh -e
+
+# No user-serviceable parts
+if [ -z "$PORTSNAP_BUILD_CONF_READ" ]; then
+ echo "Do not run $0 manually"
+ exit 1
+fi
+
+# usage: sh -e fileset-snap.sh SNAPDATE FSETDIR SNAP INDEX tINDEX STAGE WORKDIR
+SNAPDATE="$1"
+FSETDIR="$2"
+SNAP="$3"
+INDEX="$4"
+tINDEX="$5"
+STAGE="$6"
+WORKDIR="$7"
+
+# Create temporary directory
+TMPDIR=${WORKDIR}/tmp
+mkdir ${TMPDIR}
+
+# Prune old files from the file set
+sh -e s/fileset-prune.sh ${SNAPDATE} ${FSETDIR} ${TMP}
+
+# Figure out which files in this snapshot are new
+sh -e s/fileset-findnew.sh ${FSETDIR} ${INDEX} ${tINDEX} ${TMP}
+
+# Build binary patches and place them into the staging directory
+sh -e s/fileset-mkpatches.sh ${FSETDIR} ${SNAP} ${STAGE} ${TMP}
+
+# Make copies of old files so we can use them for future patch-building
+sh -e s/fileset-addfiles.sh ${FSETDIR} ${SNAP}
+
+# Copy new files into the staging directory
+sh -e s/fileset-publish.sh ${FSETDIR} ${SNAP} ${STAGE}
+
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-user
mailing list