svn commit: r246051 - in projects/portbuild: admin/scripts scripts
Mark Linimon
linimon at FreeBSD.org
Tue Jan 29 03:27:25 UTC 2013
Author: linimon (doc,ports committer)
Date: Tue Jan 29 03:27:24 2013
New Revision: 246051
URL: http://svnweb.freebsd.org/changeset/base/246051
Log:
Move scripts that need to be only owned by administrator to admin/scripts.
These must not be writeable by portbuild due to security concerns.
Added:
projects/portbuild/admin/scripts/updatesnap
- copied unchanged from r245975, projects/portbuild/scripts/updatesnap
projects/portbuild/admin/scripts/updatesnap.ports
- copied unchanged from r245975, projects/portbuild/scripts/updatesnap.ports
projects/portbuild/admin/scripts/zbackup
- copied unchanged from r245975, projects/portbuild/scripts/zbackup
projects/portbuild/admin/scripts/zexpire
- copied unchanged from r245975, projects/portbuild/scripts/zexpire
Deleted:
projects/portbuild/scripts/updatesnap
projects/portbuild/scripts/updatesnap.ports
projects/portbuild/scripts/zbackup
projects/portbuild/scripts/zexpire
Copied: projects/portbuild/admin/scripts/updatesnap (from r245975, projects/portbuild/scripts/updatesnap)
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ projects/portbuild/admin/scripts/updatesnap Tue Jan 29 03:27:24 2013 (r246051, copy of r245975, projects/portbuild/scripts/updatesnap)
@@ -0,0 +1,97 @@
+#!/bin/sh
+#
+# Update the master source trees that are used by package builds
+# and other consumers
+
+pbc=${PORTBUILD_CHECKOUT:-/var/portbuild}
+
+. ${pbc}/conf/server.conf
+
+base=${ZFS_MOUNTPOINT}/${SNAP_DIRECTORY}
+zbase=${ZFS_VOLUME}/${SNAP_DIRECTORY}
+base_prefix=${ZFS_MOUNTPOINT}/${SNAP_DIRECTORY}/${SRC_DIRECTORY_PREFIX}
+zbase_prefix=${ZFS_VOLUME}/${SNAP_DIRECTORY}/${SRC_DIRECTORY_PREFIX}
+
+VERBOSE=1
+
+stamp() {
+ fulldate=$1
+ date -j -f %+ "${fulldate}" +%Y%m%d%H%M%S
+}
+
+finish() {
+ err=$1
+
+ end=$(date +%s)
+ echo "Finished at $(date)"
+ len=$((end-begin))
+ echo "Duration = $(date -j -f %s +%H:%M:%S ${len})"
+ exit 1
+}
+
+begin=$(date +%s)
+echo "Started at $(date)"
+
+uid=${PORTBUILD_USER}
+if [ ! -z "${PORTBUILD_GROUP}" ]; then
+ gid=${PORTBUILD_GROUP}
+else
+ gid=${uid}
+fi
+
+# create /a/snap if not already there
+if [ ! -d ${base} ]; then
+ echo "creating new base directory using ${base}"
+ zfs create ${zbase} || finish 1
+ chown -R ${uid}:${gid} ${base}
+ chmod -R g+w ${base}
+fi
+
+for branch in $SRC_BRANCHES; do
+ # create /a/snap/src-<branch> if not already there
+ mountpoint=${base_prefix}${branch}
+ if [ ! -d ${mountpoint} ]; then
+ echo "creating new source branch using ${mountpoint}"
+ zfs create ${zbase_prefix}${branch} || finish 1
+ chown -R ${uid}:${gid} ${mountpoint}
+ chmod -R g+w ${mountpoint}
+ fi
+ # create /a/snap/src-<branch>/src if not already there
+ if [ ! -d ${mountpoint}/src ]; then
+ echo "creating new source branch subdirectory using ${mountpoint}/src"
+ zfs create ${zbase_prefix}${branch}/src || finish 1
+ chown -R ${uid}:${gid} ${mountpoint}/src
+ chmod -R g+w ${mountpoint}/src
+ fi
+ cd ${mountpoint}
+ fulldate=$(date)
+
+ # TODO: fix identation post-rework
+
+ # example destination directory: /a/snap/src-7/src/ (tricky!)
+ if [ ! -d ${mountpoint}/src ]; then
+ mkdir ${mountpoint}/src || finish 1
+ fi
+ if [ ! -d ${mountpoint}/src/${VCS_SUBDIR} ]; then
+ eval subdir=\$SRC_BRANCH_${branch}_SUBDIR
+ if [ $VERBOSE ]; then
+ echo "${VCS_CHECKOUT_COMMAND} ${VCS_SRC_REPOSITORY}/${subdir} ${mountpoint}/src"
+ fi
+ ${VCS_CHECKOUT_COMMAND} ${VCS_SRC_REPOSITORY}/${subdir} ${mountpoint}/src || finish 1
+ else
+ if [ $VERBOSE ]; then
+ echo "${VCS_OVERWRITE_COMMAND} ${mountpoint}/src"
+ fi
+ ${VCS_OVERWRITE_COMMAND} ${mountpoint}/src || finish 1
+ fi
+ # hack for zfs breakiness
+ find . -group wheel | xargs chgrp ${gid}
+ snapdate=$(stamp ${fulldate})
+ if [ $VERBOSE ]; then
+ echo "zfs snapshot ${zbase_prefix}${branch}/src@${snapdate}"
+ fi
+ zfs snapshot ${zbase_prefix}${branch}/src@${snapdate} || finish 1
+ echo ${fulldate} > src/.updated
+done
+
+finish 0
Copied: projects/portbuild/admin/scripts/updatesnap.ports (from r245975, projects/portbuild/scripts/updatesnap.ports)
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ projects/portbuild/admin/scripts/updatesnap.ports Tue Jan 29 03:27:24 2013 (r246051, copy of r245975, projects/portbuild/scripts/updatesnap.ports)
@@ -0,0 +1,88 @@
+#!/bin/sh
+#
+# Update the master ports tree that is used by package builds
+# and other consumers
+
+pbc=${PORTBUILD_CHECKOUT:-/var/portbuild}
+
+. ${pbc}/conf/server.conf
+
+base=${ZFS_MOUNTPOINT}/${SNAP_DIRECTORY}
+zbase=${ZFS_VOLUME}/${SNAP_DIRECTORY}
+head=${ZFS_MOUNTPOINT}/${SNAP_PORTS_DIRECTORY}
+zhead=${ZFS_VOLUME}/${SNAP_PORTS_DIRECTORY}
+
+VERBOSE=1
+
+stamp() {
+ fulldate=$1
+ date -j -f %+ "${fulldate}" +%Y%m%d%H%M%S
+}
+
+finish() {
+ err=$1
+
+ end=$(date +%s)
+ echo "Finished at $(date)"
+ len=$((end-begin))
+ echo "Duration = $(date -j -f %s +%H:%M:%S ${len})"
+ exit 1
+}
+
+begin=$(date +%s)
+echo "Started at $(date)"
+
+uid=${PORTBUILD_USER}
+if [ ! -z "${PORTBUILD_GROUP}" ]; then
+ gid=${PORTBUILD_GROUP}
+else
+ gid=${uid}
+fi
+
+# create /a/snap/ if not already there
+if [ ! -d ${base} ]; then
+ echo "creating new base directory using ${base}"
+ zfs create ${zbase} || finish 1
+ chown -R ${uid}:${gid} ${base}
+ chmod -R g+w ${base}
+fi
+
+# create /a/snap/ports-head/ if not already there
+if [ ! -d ${head} ]; then
+ echo "creating new ports-head directory using ${head}"
+ zfs create ${zhead} || finish 1
+ chown -R ${uid}:${gid} ${head}
+ chmod -R g+w ${head}
+fi
+
+# create /a/snap/ports-head/ports/ if not already there
+if [ ! -d ${head}/ports ]; then
+ echo "creating new ports-head/ports directory using ${head}/ports"
+ zfs create ${zhead}/ports || finish 1
+ chown -R ${uid}:${gid} ${head}/ports
+ chmod -R g+w ${head}/ports
+fi
+
+cd ${head}
+fulldate=$(date)
+if [ ! -d ${head}/ports/${VCS_SUBDIR} ]; then
+ if [ $VERBOSE ]; then
+ echo "${VCS_CHECKOUT_COMMAND} ${VCS_PORTS_REPOSITORY}/head ${head}/ports"
+ fi
+ ${VCS_CHECKOUT_COMMAND} ${VCS_PORTS_REPOSITORY}/head ${head}/ports || finish 1
+else
+ if [ $VERBOSE ]; then
+ echo "${VCS_OVERWRITE_COMMAND} ${head}/ports"
+ fi
+ ${VCS_OVERWRITE_COMMAND} ${head}/ports || finish 1
+fi
+# hack for zfs breakiness
+find . -group wheel | xargs chgrp ${gid}
+snapdate=$(stamp ${fulldate})
+if [ $VERBOSE ]; then
+ echo "zfs snapshot ${zhead}/ports@${snapdate}"
+fi
+zfs snapshot ${zhead}/ports@${snapdate} || finish 1
+echo ${fulldate} > ports/.updated
+
+finish 0
Copied: projects/portbuild/admin/scripts/zbackup (from r245975, projects/portbuild/scripts/zbackup)
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ projects/portbuild/admin/scripts/zbackup Tue Jan 29 03:27:24 2013 (r246051, copy of r245975, projects/portbuild/scripts/zbackup)
@@ -0,0 +1,236 @@
+#!/usr/bin/env python
+
+# Back up a list of ZFS filesystems, doing a full backup periodically
+# and using incremental diffs in between
+
+import commands, datetime, sys, os, bz2
+from signal import *
+
+sys.path.insert(0, '/var/portbuild/lib/python')
+
+import zfs
+
+# List of filesystems to backup
+# XXX MCL
+backuplist=["a", "a/nfs", "a/local", "a/portbuild",
+ "a/portbuild/amd64", "a/portbuild/arm", "a/portbuild/i386",
+ "a/portbuild/ia64", "a/portbuild/mips", "a/portbuild/powerpc",
+ "a/portbuild/sparc64"]
+
+# Directory to store backups
+# XXX MCL
+backupdir="/dumpster/pointyhat/backup"
+
+# How many days between full backups
+# XXX MCL
+fullinterval=9
+
+def validate():
+ fslist = zfs.getallfs()
+
+ missing = set(backuplist).difference(set(fslist))
+ if len(missing) > 0:
+ print "Backup list refers to filesystems that do not exist: %s" % missing
+ sys.exit(1)
+
+def mkdirp(path):
+
+ plist = path.split("/")
+
+ for i in xrange(2,len(plist)+1):
+ sofar = "/".join(plist[0:i])
+ if not os.path.isdir(sofar):
+ os.mkdir(sofar)
+
+class node(object):
+ child=None
+ parent=None
+ name=None
+ visited=0
+
+ def __init__(self, name):
+ self.name = name
+ self.child = []
+ self.parent = None
+ self.visited = 0
+
+print "zbackup: starting at " + datetime.datetime.now().ctime()
+
+for fs in backuplist:
+
+ print
+
+ dir = backupdir + "/" + fs
+ mkdirp(dir)
+
+ snaplist = None
+ try:
+ snaplist = [snap[0] for snap in zfs.getallsnaps(fs) if snap[0].isdigit()]
+ except zfs.NoSuchFS:
+ print "no such fs %s, skipping" % fs
+ continue
+
+ dofull = 0
+
+ # Mapping from backup date tag to node
+ backups={}
+
+ # list of old-new pairs seen
+ seen=[]
+
+ # Most recent snapshot date
+ latest = "0"
+ for j in os.listdir(dir):
+ (old, sep, new) = j.partition('-')
+ if not old.isdigit() or not new.isdigit():
+ continue
+
+ seen.append("%s-%s" % (old, new))
+
+ if int(old) >= int(new):
+ print "Warning: backup sequence not monotonic: %s >= %s" % (old, new)
+ continue
+
+ try:
+ oldnode = backups[old]
+ except KeyError:
+ oldnode = node(old)
+ backups[old] = oldnode
+
+ try:
+ newnode = backups[new]
+ except KeyError:
+ newnode = node(new)
+ backups[new] = newnode
+
+ if int(new) > int(latest):
+ latest = new
+
+ oldnode.child.append(newnode)
+ if newnode.parent:
+ # We are not a tree!
+ if not dofull:
+ print "Multiple backup sequences found, forcing full dump!"
+ dofull = 1
+ continue
+
+ newnode.parent = oldnode
+
+ if not "0" in backups and not dofull:
+ # No root!
+ print "No full backup found!"
+ dofull = 1
+
+ if not latest in snaplist and not dofull:
+ print "Latest dumped snapshot no longer exists: forcing full dump"
+ dofull = 1
+
+ now = datetime.datetime.now()
+ nowdate = now.strftime("%Y%m%d%H%M")
+
+ try:
+ prev = datetime.datetime.strptime(latest, "%Y%m%d%H%M")
+ except ValueError:
+ if not dofull:
+ print "Unable to parse latest snapshot as a date, forcing full dump!"
+ dofull = 1
+
+ print "Creating zfs snapshot %s@%s" % (fs, nowdate)
+ zfs.createsnap(fs, nowdate)
+
+ # Find path from latest back to root
+ try:
+ cur = backups[latest]
+ except KeyError:
+ cur = None
+
+ chain = []
+ firstname = "0"
+ # Skip if latest doesn't exist or chain is corrupt
+ while cur:
+ chain.append("%s-%s" % (cur.parent.name, cur.name))
+ par = cur.parent
+
+ # Remove from the backup tree so we can delete the leftovers
+ # below
+ par.child.remove(cur)
+ cur.parent=None
+
+ if par.name == "0":
+ firstname = cur.name
+ break
+ cur = par
+
+ chain.reverse()
+
+ print "chain is " + str( chain )
+
+ # Prune stale links not in the backup chain
+ for j in backups.iterkeys():
+ cur = backups[j]
+ for k in cur.child:
+ stale="%s-%s" % (cur.name, k.name)
+ print "Deleting stale backup %s" % stale
+ os.remove("%s/%s/%s" % (backupdir, fs, stale))
+
+ # Lookup date of full dump
+ try:
+ first = datetime.datetime.strptime(firstname, "%Y%m%d%H%M")
+ except ValueError:
+ if not dofull:
+ print "Unable to parse first snapshot as a date, forcing full dump!"
+ dofull = 1
+
+ if not dofull and (now - first) > datetime.timedelta(days=fullinterval):
+ print "Previous full backup too old, forcing full dump!"
+ dofull = 1
+
+ # In case we are interrupted don't leave behind a truncated file
+ # that will corrupt the backup chain
+
+ if dofull:
+ latest = "0"
+
+ outfile="%s/%s/.%s-%s" % (backupdir, fs, latest, nowdate)
+
+ # zfs send aborts on receiving a signal
+ signal(SIGTSTP, SIG_IGN)
+ if not dofull:
+ print "Doing incremental backup of %s: %s-%s" % (fs, latest, nowdate)
+ (err, out) = \
+ commands.getstatusoutput("zfs send -i %s %s@%s | bzip2 > %s" %
+ (latest, fs, nowdate, outfile))
+ else:
+ print "Doing full backup of %s" % fs
+ latest = "0"
+ (err, out) = \
+ commands.getstatusoutput("zfs send %s@%s | bzip2 > %s" %
+ (fs, nowdate, outfile))
+ signal(SIGTSTP, SIG_DFL)
+
+ if err:
+ print "Error from snapshot: (%s, %s)" % (err, out)
+ try:
+ os.remove(outfile)
+ print "Deleted file %s" % outfile
+ except OSError, err:
+ print "OSError: " + repr(err)
+ if err.errno != 2:
+ raise
+ finally:
+ sys.exit(1)
+
+ # We seem to be finished
+ try:
+ os.rename(outfile, "%s/%s/%s-%s" % (backupdir, fs, latest, nowdate))
+ except:
+ print "Error renaming dump file" + outfile + "!"
+ raise
+
+ if dofull:
+ for i in seen:
+ print "Removing stale snapshot %s/%s" % (dir, i)
+ os.remove("%s/%s" % (dir, i))
+
+print
+print "zbackup: ending at " + datetime.datetime.now().ctime()
Copied: projects/portbuild/admin/scripts/zexpire (from r245975, projects/portbuild/scripts/zexpire)
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ projects/portbuild/admin/scripts/zexpire Tue Jan 29 03:27:24 2013 (r246051, copy of r245975, projects/portbuild/scripts/zexpire)
@@ -0,0 +1,129 @@
+#!/usr/bin/env python
+#
+# Expire old snapshots
+
+import sys, commands, datetime, os, string
+
+pbc = os.getenv('PORTBUILD_CHECKOUT') \
+ if os.getenv('PORTBUILD_CHECKOUT') else "/a/portbuild"
+pbd = os.getenv('PORTBUILD_DATA') \
+ if os.getenv('PORTBUILD_DATA') else "/a/portbuild"
+
+sys.path.insert(0, '%s/lib/python' % pbc)
+
+from freebsd import *
+from freebsd_config import *
+import zfs
+
+CONFIG_SUBDIR="conf"
+CONFIG_FILENAME="server.conf"
+
+ENABLED = True
+VERBOSE= True
+
+expirelist={}
+
+now = datetime.datetime.now()
+print "zexpire: starting at " + now.ctime()
+
+config = getConfig( pbc, CONFIG_SUBDIR, CONFIG_FILENAME )
+ZFS_VOLUME = config.get( 'ZFS_VOLUME' )
+if not ZFS_VOLUME:
+ print "you must define ZFS_VOLUME"
+ sys.exit( 1 )
+ZFS_MOUNTPOINT = config.get( 'ZFS_MOUNTPOINT' )
+if not ZFS_MOUNTPOINT:
+ print "you must define ZFS_MOUNTPOINT"
+ sys.exit( 1 )
+PORTBUILD_DIRECTORY = config.get( 'PORTBUILD_DIRECTORY' )
+if not PORTBUILD_DIRECTORY:
+ print "you must define PORTBUILD_DIRECTORY"
+ sys.exit( 1 )
+portbuild_directory = os.path.join( ZFS_VOLUME, PORTBUILD_DIRECTORY )
+SNAP_DIRECTORY = config.get( 'SNAP_DIRECTORY' )
+if not SNAP_DIRECTORY:
+ print "you must define SNAP_DIRECTORY"
+ sys.exit( 1 )
+snap_directory = os.path.join( ZFS_VOLUME, SNAP_DIRECTORY )
+snap_mountpoint = os.path.join( ZFS_MOUNTPOINT, SNAP_DIRECTORY )
+SUPPORTED_ARCHS = config.get( 'SUPPORTED_ARCHS' )
+if not SUPPORTED_ARCHS:
+ print "you must define SUPPORTED_ARCHS"
+ sys.exit( 1 )
+supported_archs = string.split( SUPPORTED_ARCHS )
+ZFS_DEFAULT_EXPIRATION = config.get( 'ZFS_DEFAULT_EXPIRATION' )
+if not ZFS_DEFAULT_EXPIRATION:
+ print "you must define ZFS_DEFAULT_EXPIRATION"
+ sys.exit( 1 )
+ZFS_SNAPSHOT_EXPIRATION = config.get( 'ZFS_SNAPSHOT_EXPIRATION' )
+if not ZFS_SNAPSHOT_EXPIRATION:
+ print "you must define ZFS_SNAPSHOT_EXPIRATION"
+ sys.exit( 1 )
+
+expirelist[ ZFS_VOLUME ] = ZFS_DEFAULT_EXPIRATION
+expirelist[ portbuild_directory ] = ZFS_DEFAULT_EXPIRATION
+for arch in supported_archs:
+ expirelist[ os.path.join( portbuild_directory, arch ) ] = ZFS_DEFAULT_EXPIRATION
+expirelist[ snap_directory ] = ZFS_SNAPSHOT_EXPIRATION
+try:
+ snapdirs = os.listdir( snap_mountpoint )
+ for snapdir in snapdirs:
+ subdir = os.path.join( snap_mountpoint, snapdir )
+ tmp = os.path.join( snap_directory, snapdir )
+ expirelist[ tmp ] = ZFS_SNAPSHOT_EXPIRATION
+ try:
+ snapsubdirs = os.listdir( subdir )
+ for snapsubdir in snapsubdirs:
+ expirelist[ os.path.join( tmp, snapsubdir ) ] = ZFS_SNAPSHOT_EXPIRATION
+ except:
+ pass
+except:
+ pass
+
+keys = expirelist.keys()
+keys.sort()
+for key in keys:
+ fs = key
+ maxage = 2
+ try:
+ maxage = int( expirelist[ key ] )
+ except:
+ pass
+
+ print
+
+ if VERBOSE:
+ print "fs: " + str(fs)
+ try:
+ snapdata = zfs.getallsnaps(fs)
+ if VERBOSE:
+ print "snapdata: " + str(snapdata)
+ except zfs.NoSuchFS:
+ print "zexpire: no such fs %s, skipping" % fs
+ continue
+
+ snaps = (i[0] for i in snapdata)
+
+ for snap in snaps:
+ try:
+ snapdate = datetime.datetime.strptime(snap, "%Y%m%d%H%M")
+ except ValueError:
+ try:
+ snapdate = datetime.datetime.strptime(snap, "%Y%m%d%H%M%S")
+ except ValueError:
+ print "zexpire: don't know what to do with snap `" + snap + "'"
+ continue
+
+ if VERBOSE:
+ print "zexpire: examining snapshot %s@%s" % (fs, snap)
+ if (now - snapdate) > datetime.timedelta(days=maxage):
+ print "zexpire: snapshot %s@%s too old, attempting zfs destroy" % (fs, snap)
+ if ENABLED:
+ (err, out) = commands.getstatusoutput("zfs destroy %s@%s" % (fs,snap))
+
+ if err:
+ print "zexpire: error deleting snapshot", out
+
+then = datetime.datetime.now()
+print
+print "zexpire: ending at " + then.ctime()
More information about the svn-src-projects
mailing list