svn commit: r221159 - in projects/portbuild: qmanager scripts
Florent Thoumie
flz at FreeBSD.org
Thu Apr 28 11:06:15 UTC 2011
Author: flz
Date: Thu Apr 28 11:06:14 2011
New Revision: 221159
URL: http://svn.freebsd.org/changeset/base/221159
Log:
portbuild: remove hardcoded /var/portbuild.
- Don't hardcode /var/portbuild.
- Upload per-build scripts/ and sources/ directories.
- Scripts which aren't tied to a specific build are uploaded first.
This change makes it effectively possible to checkout portbuild
anywhere and test changes without affecting any other builds.
Modified:
projects/portbuild/qmanager/dumpdb.py
projects/portbuild/qmanager/packagebuild
projects/portbuild/qmanager/qmanager
projects/portbuild/qmanager/qmanager.py
projects/portbuild/scripts/allgohans
projects/portbuild/scripts/build
projects/portbuild/scripts/buildenv
projects/portbuild/scripts/buildfailure
projects/portbuild/scripts/buildproxy
projects/portbuild/scripts/buildproxy-client
projects/portbuild/scripts/buildsuccess
projects/portbuild/scripts/claim-chroot
projects/portbuild/scripts/clean-chroot
projects/portbuild/scripts/cleanup-chroots
projects/portbuild/scripts/client-metrics
projects/portbuild/scripts/cpdistfiles
projects/portbuild/scripts/cppackages
projects/portbuild/scripts/do-cleanup-chroots
projects/portbuild/scripts/docppackages
projects/portbuild/scripts/dodistfiles
projects/portbuild/scripts/dologs
projects/portbuild/scripts/dopackages
projects/portbuild/scripts/dopackages.wrapper
projects/portbuild/scripts/dopackagestats
projects/portbuild/scripts/dosetupnode
projects/portbuild/scripts/flushsquid
projects/portbuild/scripts/keeprestr
projects/portbuild/scripts/makeduds
projects/portbuild/scripts/makeindex
projects/portbuild/scripts/makerestr
projects/portbuild/scripts/makeworld
projects/portbuild/scripts/mkbindist
projects/portbuild/scripts/pdispatch
projects/portbuild/scripts/pollmachine
projects/portbuild/scripts/portbuild
projects/portbuild/scripts/processfail
projects/portbuild/scripts/processlogs
projects/portbuild/scripts/prunefailure
projects/portbuild/scripts/reportload
projects/portbuild/scripts/setupnode
projects/portbuild/scripts/stats
projects/portbuild/scripts/updatesnap
projects/portbuild/scripts/updatesnap.ports
Modified: projects/portbuild/qmanager/dumpdb.py
==============================================================================
--- projects/portbuild/qmanager/dumpdb.py Thu Apr 28 09:09:34 2011 (r221158)
+++ projects/portbuild/qmanager/dumpdb.py Thu Apr 28 11:06:14 2011 (r221159)
@@ -4,17 +4,21 @@
import sys, os, threading, socket, Queue
+pbc = os.getenv('PORTBUILD_CHECKOUT') \
+ if os.getenv('PORTBUILD_CHECKOUT') else "/var/portbuild"
+pbd = os.getenv('PORTBUILD_DATA') \
+ if os.getenv('PORTBUILD_DATA') else "/var/portbuild"
+
+sys.path.insert(0, '%s/lib/python' % pbc)
+
from signal import *
from sys import exc_info
from itertools import chain
-sys.path.insert(0, '/var/portbuild/lib/python')
-
from freebsd_config import *
from qmanagerobj import *
-CONFIG_DIR="/var/portbuild"
CONFIG_SUBDIR="conf"
CONFIG_FILENAME="server.conf"
@@ -127,7 +131,7 @@ def show_machines_for_arch( engine, arch
if __name__ == '__main__':
print "acquiring engine and session"
- config = getConfig( CONFIG_DIR, CONFIG_SUBDIR, CONFIG_FILENAME )
+ config = getConfig( pbc, CONFIG_SUBDIR, CONFIG_FILENAME )
QMANAGER_PATH = config.get( 'QMANAGER_PATH' )
QMANAGER_DATABASE_FILE = config.get( 'QMANAGER_DATABASE_FILE' )
(engine, session) = obj_startup( \
Modified: projects/portbuild/qmanager/packagebuild
==============================================================================
--- projects/portbuild/qmanager/packagebuild Thu Apr 28 09:09:34 2011 (r221158)
+++ projects/portbuild/qmanager/packagebuild Thu Apr 28 11:06:14 2011 (r221159)
@@ -32,31 +32,33 @@
# * check mtime for package staleness (cf make)
# * option to skip phase 2
+import os
import sys
-sys.path.insert(0, '/var/portbuild/lib/python')
+pbc = os.getenv('PORTBUILD_CHECKOUT') \
+ if os.getenv('PORTBUILD_CHECKOUT') else "/var/portbuild"
+pbd = os.getenv('PORTBUILD_DATA') \
+ if os.getenv('PORTBUILD_DATA') else "/var/portbuild"
-from qmanagerclient import *
+sys.path.insert(0, '%s/lib/python' % pbc)
+from qmanagerclient import *
from freebsd_config import *
-import os, string, threading, time, subprocess
-#import random
+import string, threading, time, subprocess
from itertools import chain
-#import gc
from stat import *
from Queue import Queue
from heapq import *
-CONFIG_DIR="/var/portbuild"
CONFIG_SUBDIR="conf"
CONFIG_FILENAME="server.conf"
-config = getConfig( CONFIG_DIR, CONFIG_SUBDIR, CONFIG_FILENAME )
+config = getConfig( pbc, CONFIG_SUBDIR, CONFIG_FILENAME )
QMANAGER_MAX_JOB_ATTEMPTS = int( \
config.get( 'QMANAGER_MAX_JOB_ATTEMPTS' ) )
-QMANAGER_PRIORITY_PACKAGES = string.split( \
+QMANAGER_PRIORITY_PACKAGES = string.split( \
config.get( 'QMANAGER_PRIORITY_PACKAGES' ) )
QMANAGER_RUNAWAY_PERCENTAGE = float( \
config.get( 'QMANAGER_RUNAWAY_PERCENTAGE' ) )
@@ -323,8 +325,8 @@ they still need to know about us as depe
def packagename(self, arch, branch, buildid):
""" Return the path where a package may be found"""
- return "/var/portbuild/%s/%s/builds/%s/packages/All/%s.tbz" \
- % (arch, branch, buildid, self.name)
+ return "%s/%s/%s/builds/%s/packages/All/%s.tbz" \
+ % (pbd, arch, branch, buildid, self.name)
def is_stale(self, arch, branch, buildid):
""" Does a package need to be (re)-built?
@@ -409,6 +411,7 @@ class worker(threading.Thread):
self.branch = branch
self.buildid = buildid
self.queue = queue
+ self.basedir = pbd+"/"+arch+"/"+branch+"/builds/"+buildid
self.setDaemon(True)
@@ -422,19 +425,19 @@ class worker(threading.Thread):
print
try:
runenv={'HOME':"/root",
- 'PATH':'/sbin:/bin:/usr/sbin:/usr/bin:/usr/games:/usr/local/sbin:/usr/local/bin:/var/portbuild/scripts',
+ 'PATH':'/sbin:/bin:/usr/sbin:/usr/bin:/usr/games:/usr/local/sbin:/usr/local/bin:%s/scripts' + pbc,
'FD':" ".join(["%s.tbz" % p.name for p in pkg.fdep]),
'ED':" ".join(["%s.tbz" % p.name for p in pkg.edep]),
'PD':" ".join(["%s.tbz" % p.name for p in pkg.pdep]),
'BD':" ".join(["%s.tbz" % p.name for p in pkg.bdep]),
'RD':" ".join(["%s.tbz" % p.name for p in pkg.rdep])}
- for var in ["NOCLEAN", "NO_RESTRICTED", "NOPLISTCHECK", "NO_DISTFILES", "FETCH_ORIGINAL", "TRYBROKEN" ]:
+ for var in ["NOCLEAN", "NO_RESTRICTED", "NOPLISTCHECK", "NO_DISTFILES", "FETCH_ORIGINAL", "TRYBROKEN", "PORTBUILD_CHECKOUT", "PORTBUILD_DATA" ]:
if var in os.environ:
runenv[var] = os.environ.get(var)
build = subprocess.Popen(
- ["/bin/sh", "/var/portbuild/scripts/pdispatch",
+ ["/bin/sh", "%s/scripts/pdispatch" % pbc,
self.arch, self.branch, self.buildid, self.machine,
- "/var/portbuild/scripts/portbuild", "%s.tbz" % pkg.name,
+ "%s/scripts/portbuild" % self.basedir, "%s.tbz" % pkg.name,
pkg.path],
env=runenv,
stderr=subprocess.STDOUT, stdout=subprocess.PIPE, bufsize=0)
@@ -506,7 +509,7 @@ class worker(threading.Thread):
def main(arch, branch, buildid, args):
global index
- basedir="/var/portbuild/"+arch+"/"+branch+"/builds/"+buildid
+ basedir=pbd+"/"+arch+"/"+branch+"/builds/"+buildid
portsdir=basedir+"/ports"
# get the major branch number.
Modified: projects/portbuild/qmanager/qmanager
==============================================================================
--- projects/portbuild/qmanager/qmanager Thu Apr 28 09:09:34 2011 (r221158)
+++ projects/portbuild/qmanager/qmanager Thu Apr 28 11:06:14 2011 (r221159)
@@ -49,11 +49,15 @@
# - OR, NOT job description entries
# - query jobs.machine properties
+import os
import sys
-sys.path.insert(0, '/var/portbuild/lib/python')
+pbc = os.getenv('PORTBUILD_CHECKOUT') \
+ if os.getenv('PORTBUILD_CHECKOUT') else "/var/portbuild"
+pbd = os.getenv('PORTBUILD_DATA') \
+ if os.getenv('PORTBUILD_DATA') else "/var/portbuild"
-from freebsd_config import *
+sys.path.insert(0, '%s/lib/python' % pbc)
import os, threading, socket, Queue
@@ -61,12 +65,12 @@ from signal import *
from itertools import chain
from qmanagerobj import *
+from freebsd_config import *
-CONFIG_DIR="/var/portbuild"
CONFIG_SUBDIR="conf"
CONFIG_FILENAME="server.conf"
-config = getConfig( CONFIG_DIR, CONFIG_SUBDIR, CONFIG_FILENAME )
+config = getConfig( pbc, CONFIG_SUBDIR, CONFIG_FILENAME )
QMANAGER_SOCKET_FILE = config.get( 'QMANAGER_SOCKET_FILE' )
class Worker(object):
Modified: projects/portbuild/qmanager/qmanager.py
==============================================================================
--- projects/portbuild/qmanager/qmanager.py Thu Apr 28 09:09:34 2011 (r221158)
+++ projects/portbuild/qmanager/qmanager.py Thu Apr 28 11:06:14 2011 (r221159)
@@ -49,24 +49,28 @@
# - OR, NOT job description entries
# - query jobs.machine properties
+import os
import sys
-sys.path.insert(0, '/var/portbuild/lib/python')
+pbc = os.getenv('PORTBUILD_CHECKOUT') \
+ if os.getenv('PORTBUILD_CHECKOUT') else "/var/portbuild"
+pbd = os.getenv('PORTBUILD_DATA') \
+ if os.getenv('PORTBUILD_DATA') else "/var/portbuild"
-from freebsd_config import *
+sys.path.insert(0, '%s/lib/python' % pbc)
-import os, socket, threading, time, Queue
+import socket, threading, time, Queue
from signal import *
from itertools import chain
from qmanagerobj import *
+from freebsd_config import *
-CONFIG_DIR="/var/portbuild"
CONFIG_SUBDIR="conf"
CONFIG_FILENAME="server.conf"
-config = getConfig( CONFIG_DIR, CONFIG_SUBDIR, CONFIG_FILENAME )
+config = getConfig( pbc, CONFIG_SUBDIR, CONFIG_FILENAME )
QMANAGER_SOCKET_FILE = config.get( 'QMANAGER_SOCKET_FILE' )
DEBUG = True
Modified: projects/portbuild/scripts/allgohans
==============================================================================
--- projects/portbuild/scripts/allgohans Thu Apr 28 09:09:34 2011 (r221158)
+++ projects/portbuild/scripts/allgohans Thu Apr 28 11:06:14 2011 (r221159)
@@ -1,30 +1,46 @@
#!/bin/sh
+pbc=${PORTBUILD_CHECKOUT:-/var/portbuild}
+pbd=${PORTBUILD_DATA:-/var/portbuild}
+
+. ${pbc}/conf/server.conf
+
doarch() {
arch=$1
shift
- if [ -f "${pb}/${arch}/portbuild.conf" ]; then
- . ${pb}/${arch}/portbuild.conf
+ if [ -f "${pbd}/${arch}/portbuild.conf" ]; then
+ . ${pbd}/${arch}/portbuild.conf
else
echo "Invalid arch"
exit 1
fi
- machines=$(awk '{print $1}' ${pb}/${arch}/mlist)
+ machines=$(awk '{print $1}' ${pbd}/${arch}/mlist)
for i in ${machines}; do
- . ${pb}/${arch}/portbuild.conf
- if [ -f "${pb}/${arch}/portbuild.${i}" ]; then
- . ${pb}/${arch}/portbuild.${i}
+ . ${pbd}/${arch}/portbuild.conf
+ if [ -f "${pbd}/${arch}/portbuild.${i}" ]; then
+ . ${pbd}/${arch}/portbuild.${i}
fi
if [ ${quiet} -eq 0 ]; then
echo "[$i]"
fi
- lockf -t 60 ${pb}/${arch}/lockfiles/lock.$i su ports-${arch} -c "ssh ${client_user}@$i ${sudo_cmd} $@"
+ cmdpath=${cmd}
+ if [ ${scpfirst} -ne 0 ]; then
+ # If we need to scp first, the command given in the parameters is the local
+ # path, which we need to upload first. The command will return the remote
+ # temporary file, which we can subsequently execute. So this isn't really scp.
+ cmdpath=$(su ports-${arch} -c "cat ${cmd} | ssh ${client_user}@$i 't=\$(mktemp -t ${cmd##*/}); cat >\$t; echo \$t; chmod 755 \$t'")
+ case ${cmdpath} in /tmp/*) ;; *) echo "Failed to scp ${cmd} to $i."; return 1;; esac
+ fi
+ lockf -t 60 ${pbd}/${arch}/lockfiles/lock.$i su ports-${arch} -c "ssh ${client_user}@$i ${sudo_cmd} ${cmdpath} $@"
result=$?
if [ $result -ne 0 ]; then
- echo "could not execute command $@ on $i: $result"
+ echo "could not execute command ${cmdpath} $@ on $i: $result"
+ fi
+ if [ ${scpfirst} -ne 0 ]; then
+ su ports-${arch} -c "ssh ${client_user}@$i 'rm -f ${cmdpath}'"
fi
done
}
@@ -39,6 +55,7 @@ else
root=0
fi
+# XXX - Have to convert this to getopt().
if [ "$1" = "-q" ]; then
quiet=1
shift
@@ -46,11 +63,18 @@ else
quiet=0
fi
-pb=/var/portbuild
-. ${pb}/conf/server.conf
+if [ "$1" = "-scp" ]; then
+ scpfirst=1
+ shift
+else
+ scpfirst=0
+fi
+
+cmd=$1
+shift
if [ "${arch}" = "all" ]; then
- arches=$(find ${pb}/*/portbuild.conf)
+ arches=$(find ${pbd}/*/portbuild.conf)
for i in ${arches}; do
arch=$(basename $(dirname $i))
doarch $arch "$@"
Modified: projects/portbuild/scripts/build
==============================================================================
--- projects/portbuild/scripts/build Thu Apr 28 09:09:34 2011 (r221158)
+++ projects/portbuild/scripts/build Thu Apr 28 11:06:14 2011 (r221159)
@@ -4,7 +4,8 @@
# server-side script to handle various commands common to builds
# configurable variables
-pb=/var/portbuild
+pbc=${PORTBUILD_CHECKOUT:-/var/portbuild}
+pbd=${PORTBUILD_DATA:-/var/portbuild}
# subdirectories to process. yes, this is a hack, but it saves code
# duplication.
@@ -25,7 +26,7 @@ do_list() {
arch=$1
branch=$2
- buildpar=/var/portbuild/${arch}/${branch}/builds
+ buildpar=${pbd}/${arch}/${branch}/builds
if [ -d ${buildpar} ]; then
snaps=$(cd ${buildpar}; ls -1d 2* 2> /dev/null)
@@ -51,7 +52,7 @@ do_create() {
buildid=$3
shift 3
- archivedir=${pb}/${arch}/archive
+ archivedir=${pbd}/${arch}/archive
# create directory for all build logs
buildlogsdir=${archivedir}/buildlogs
@@ -93,8 +94,7 @@ do_create() {
ln -sf ${builddir} ${pbab}/builds/latest
# create the links for the webserver
- # XXX MCL hardcoding
- errorlogs=/var/portbuild/errorlogs
+ errorlogs=${pbd}/errorlogs
ln -s ${buildsdir}/latest/bak/errors ${errorlogs}/${arch}-${branch}-previous
ln -s ${buildsdir}/latest/bak/logs ${errorlogs}/${arch}-${branch}-previous-logs
@@ -148,7 +148,7 @@ do_clone() {
if [ -d ${newbuilddir} ]; then
if [ ! -f ${pbab}/builds/previous/.keep ]; then
- /var/portbuild/scripts/build destroy ${arch} ${branch} previous
+ ${pbc}/scripts/build destroy ${arch} ${branch} previous
fi
rm -f ${pbab}/builds/previous
mv ${pbab}/builds/latest ${pbab}/builds/previous
@@ -251,10 +251,10 @@ cleanup_client() {
echo "Started cleaning up ${arch}/${branch} build ID ${buildid} on ${mach}"
- test -f ${pb}/${arch}/portbuild.${mach} && . ${pb}/${arch}/portbuild.${mach}
+ test -f ${pbd}/${arch}/portbuild.${mach} && . ${pbd}/${arch}/portbuild.${mach}
# Kill off builds and clean up chroot
- ${pb}/scripts/dosetupnode ${arch} ${branch} ${buildid} ${mach} -nocopy -queue -full
+ ${pbc}/scripts/dosetupnode ${arch} ${branch} ${buildid} ${mach} -nocopy -queue -full
echo "Finished cleaning up ${arch}/${branch} build ID ${buildid} on ${mach}"
}
@@ -267,7 +267,7 @@ do_cleanup() {
arg=$5
shift 5
- for i in `cat ${pb}/${arch}/mlist`; do
+ for i in `cat ${pbd}/${arch}/mlist`; do
cleanup_client ${arch} ${branch} ${buildid} ${i} ${arg} &
done
wait
@@ -394,7 +394,7 @@ do_destroy() {
shift 4
echo
- buildid=$(resolve ${pb} ${arch} ${branch} ${buildid})
+ buildid=$(resolve ${pbd} ${arch} ${branch} ${buildid})
if [ -z "${buildid}" ]; then
echo "Invalid build ID ${buildid}"
exit 1
@@ -435,7 +435,7 @@ proxy_root() {
id=$(id -u)
if [ ${id} != "0" ]; then
- /var/portbuild/scripts/buildproxy-client "build ${cmd} ${arch} ${branch} ${buildid} ${args}"
+ ${pbc}/scripts/buildproxy-client "build ${cmd} ${arch} ${branch} ${buildid} ${args}"
error=$?
if [ ${error} -eq 254 ]; then
echo "Proxy error"
@@ -463,7 +463,7 @@ proxy_user() {
eval "do_${cmd} ${arch} ${branch} ${buildid} \"${builddir}\" ${args}"
error=$?
else
- su ports-${arch} -c "/var/portbuild/scripts/build ${cmd} ${arch} ${branch} ${buildid} \"${builddir}\" ${args}"
+ su ports-${arch} -c "${pbc}/scripts/build ${cmd} ${arch} ${branch} ${buildid} \"${builddir}\" ${args}"
error=$?
fi
@@ -486,11 +486,11 @@ arch=$2
branch=$3
shift 3
-. ${pb}/conf/server.conf
-. ${pb}/${arch}/portbuild.conf
-. ${pb}/scripts/buildenv
+. ${pbc}/conf/server.conf
+. ${pbd}/${arch}/portbuild.conf
+. ${pbc}/scripts/buildenv
-pbab=${pb}/${arch}/${branch}
+pbab=${pbd}/${arch}/${branch}
validate_env ${arch} ${branch} || exit 1
@@ -505,7 +505,7 @@ if [ $# -ge 1 ]; then
case "$cmd" in
cleanup)
# Resolve symlinks but don't bail if the build doesn't exist.
- newbuildid=$(resolve ${pb} ${arch} ${branch} ${buildid})
+ newbuildid=$(resolve ${pbd} ${arch} ${branch} ${buildid})
if [ ! -z "${newbuildid}" -a "${newbuildid}" != "${buildid}" ]; then
echo "Resolved ${buildid} to ${newbuildid}"
buildid=${newbuildid}
@@ -527,7 +527,7 @@ if [ $# -ge 1 ]; then
# We can't rely on buildenv for this code path
;;
*)
- newbuildid=$(resolve ${pb} ${arch} ${branch} ${buildid})
+ newbuildid=$(resolve ${pbd} ${arch} ${branch} ${buildid})
if [ -z "${newbuildid}" ]; then
echo "Build ID ${buildid} does not exist"
exit 1
@@ -539,7 +539,7 @@ if [ $# -ge 1 ]; then
builddir=$(realpath ${pbab}/builds/${buildid}/)
- buildenv ${pb} ${arch} ${branch} ${builddir}
+ buildenv ${pbd} ${arch} ${branch} ${builddir}
;;
esac
fi
Modified: projects/portbuild/scripts/buildenv
==============================================================================
--- projects/portbuild/scripts/buildenv Thu Apr 28 09:09:34 2011 (r221158)
+++ projects/portbuild/scripts/buildenv Thu Apr 28 11:06:14 2011 (r221159)
@@ -40,7 +40,7 @@ validate_env() {
# only used on server side
resolve() {
- pb=$1
+ pbd=$1
arch=$2
branch=$3
buildid=$4
@@ -48,7 +48,7 @@ resolve() {
# Resolve a possibly symlinked buildid (e.g. "latest") to the
# underlying physical directory
- pbab=${pb}/${arch}/${branch}
+ pbab=${pbd}/${arch}/${branch}
builddir=${pbab}/builds/${buildid}/
if [ ! -d ${builddir} ]; then
@@ -83,7 +83,7 @@ export_src_metadata() {
# establish commonly-used environment variables (server-side)
#
buildenv () {
- pb=$1
+ pbd=$1
arch=$2
branch=$3
builddir=$4
@@ -139,7 +139,7 @@ buildenv () {
fi
# probably only used in mkbindist
- export __MAKE_CONF=${pb}/${arch}/make.conf
+ export __MAKE_CONF=${pbd}/${arch}/make.conf
}
#
Modified: projects/portbuild/scripts/buildfailure
==============================================================================
--- projects/portbuild/scripts/buildfailure Thu Apr 28 09:09:34 2011 (r221158)
+++ projects/portbuild/scripts/buildfailure Thu Apr 28 11:06:14 2011 (r221159)
@@ -9,7 +9,8 @@ cleanup() {
}
# configurable variables
-pb=/var/portbuild
+pbc=${PORTBUILD_CHECKOUT:-/var/portbuild}
+pbd=${PORTBUILD_DATA:-/var/portbuild}
usage () {
echo "usage: buildfailure arch branch buildid pkgname"
@@ -26,14 +27,14 @@ buildid=$3
pkgname=$4
shift 4
-builddir=${pb}/${arch}/${branch}/builds/${buildid}
+builddir=${pbd}/${arch}/${branch}/builds/${buildid}
-. ${pb}/conf/server.conf
-. ${pb}/conf/common.conf
-. ${pb}/${arch}/portbuild.conf
-. ${pb}/scripts/buildenv
+. ${pbc}/conf/server.conf
+. ${pbc}/conf/common.conf
+. ${pbd}/${arch}/portbuild.conf
+. ${pbc}/scripts/buildenv
-buildenv ${pb} ${arch} ${branch} ${builddir}
+buildenv ${pbd} ${arch} ${branch} ${builddir}
# Don't pick up installed packages from the host
export LOCALBASE=/nonexistentlocal
@@ -43,7 +44,7 @@ index=${PORTSDIR}/${INDEXFILE}
portloc=$(grep "^$pkgname|" ${index} | cut -f 2 -d \| | sed s,/usr/ports/,,)
pkgbase=$(cd ${PORTSDIR}/${portloc}/ && make -V PKGBASE)
-cd ${pb}/${arch}/${branch}
+cd ${pbd}/${arch}/${branch}
entry=$(grep "^${portloc}|" failure)
date=$(date +%s)
IFS='|'
@@ -58,7 +59,7 @@ else
(echo "${portloc}|${pkgbase}|${pkgname}|${date}|${date}|1" >> failure) || cleanup
fi
-link=${pb}/${arch}/${branch}/latest/${portloc}
+link=${pbd}/${arch}/${branch}/latest/${portloc}
mkdir -p $(dirname ${link})
errorloc=$(realpath ${builddir}/errors/${pkgname}.log)
ln -sf ${errorloc} ${link}
Modified: projects/portbuild/scripts/buildproxy
==============================================================================
--- projects/portbuild/scripts/buildproxy Thu Apr 28 09:09:34 2011 (r221158)
+++ projects/portbuild/scripts/buildproxy Thu Apr 28 11:06:14 2011 (r221159)
@@ -6,12 +6,16 @@
import sys, socket, os, commands
-sys.path.insert(0, '/var/portbuild/lib/python')
+pbc = os.getenv('PORTBUILD_CHECKOUT') \
+ if os.getenv('PORTBUILD_CHECKOUT') else "/var/portbuild"
+pbd = os.getenv('PORTBUILD_DATA') \
+ if os.getenv('PORTBUILD_DATA') else "/var/portbuild"
+
+sys.path.insert(0, '%s/lib/python' % pbc)
from freebsd import *
from freebsd_config import *
-CONFIG_DIR="/var/portbuild"
CONFIG_SUBDIR="conf"
CONFIG_FILENAME="server.conf"
@@ -48,11 +52,11 @@ def process(cmd, sockfile):
if not j.isalnum() and not j in "-_.":
return (254, "Illegal characters in input")
- (status, out) = commands.getstatusoutput("/var/portbuild/scripts/build %s" % " ".join(cmd[1:]))
+ (status, out) = commands.getstatusoutput("%s/scripts/build %s" % (pbc, " ".join(cmd[1:])))
return (status, out)
-config = getConfig( CONFIG_DIR, CONFIG_SUBDIR, CONFIG_FILENAME )
+config = getConfig( pbc, CONFIG_SUBDIR, CONFIG_FILENAME )
BUILDPROXY_SOCKET_FILE = config.get( 'BUILDPROXY_SOCKET_FILE' )
if os.path.exists(BUILDPROXY_SOCKET_FILE):
Modified: projects/portbuild/scripts/buildproxy-client
==============================================================================
--- projects/portbuild/scripts/buildproxy-client Thu Apr 28 09:09:34 2011 (r221158)
+++ projects/portbuild/scripts/buildproxy-client Thu Apr 28 11:06:14 2011 (r221159)
@@ -5,16 +5,20 @@
import sys, socket, os, commands
-sys.path.insert(0, '/var/portbuild/lib/python')
+pbc = os.getenv('PORTBUILD_CHECKOUT') \
+ if os.getenv('PORTBUILD_CHECKOUT') else "/var/portbuild"
+pbd = os.getenv('PORTBUILD_DATA') \
+ if os.getenv('PORTBUILD_DATA') else "/var/portbuild"
+
+sys.path.insert(0, '%s/lib/python' % pbc)
from freebsd import *
from freebsd_config import *
-CONFIG_DIR="/var/portbuild"
CONFIG_SUBDIR="conf"
CONFIG_FILENAME="server.conf"
-config = getConfig( CONFIG_DIR, CONFIG_SUBDIR, CONFIG_FILENAME )
+config = getConfig( pbc, CONFIG_SUBDIR, CONFIG_FILENAME )
BUILDPROXY_SOCKET_FILE = config.get( 'BUILDPROXY_SOCKET_FILE' )
try:
Modified: projects/portbuild/scripts/buildsuccess
==============================================================================
--- projects/portbuild/scripts/buildsuccess Thu Apr 28 09:09:34 2011 (r221158)
+++ projects/portbuild/scripts/buildsuccess Thu Apr 28 11:06:14 2011 (r221159)
@@ -3,7 +3,8 @@
# buildsuccess <arch> <branch> <buildid> <pkgname>
# configurable variables
-pb=/var/portbuild
+pbc=${PORTBUILD_CHECKOUT:-/var/portbuild}
+pbd=${PORTBUILD_DATA:-/var/portbuild}
usage () {
echo "usage: buildsuccess arch branch buildid pkgname"
@@ -20,14 +21,14 @@ buildid=$3
pkgname=$4
shift 4
-builddir=${pb}/${arch}/${branch}/builds/${buildid}
+builddir=${pbd}/${arch}/${branch}/builds/${buildid}
-. ${pb}/conf/server.conf
-. ${pb}/conf/common.conf
-. ${pb}/${arch}/portbuild.conf
-. ${pb}/scripts/buildenv
+. ${pbc}/conf/server.conf
+. ${pbc}/conf/common.conf
+. ${pbd}/${arch}/portbuild.conf
+. ${pbc}/scripts/buildenv
-buildenv ${pb} ${arch} ${branch} ${builddir}
+buildenv ${pbd} ${arch} ${branch} ${builddir}
# Don't pick up installed packages from the host
export LOCALBASE=/nonexistentlocal
@@ -36,13 +37,13 @@ index=${PORTSDIR}/${INDEXFILE}
portloc=$(grep "^$pkgname|" ${index} | cut -f 2 -d \| | sed s,/usr/ports/,,)
-cd ${pb}/${arch}/${branch}
+cd ${pbd}/${arch}/${branch}
if grep -q "^${portloc}|" failure; then
echo | mail -s "${pkgname} now builds on ${arch} ${branch}" ${mailto}
grep -v "^${portloc}|" failure > failure.new
mv failure.new failure
fi
-if [ -L ${pb}/${arch}/${branch}/latest/${portloc} ]; then
- rm -f ${pb}/${arch}/${branch}/latest/${portloc}
+if [ -L ${pbd}/${arch}/${branch}/latest/${portloc} ]; then
+ rm -f ${pbd}/${arch}/${branch}/latest/${portloc}
fi
Modified: projects/portbuild/scripts/claim-chroot
==============================================================================
--- projects/portbuild/scripts/claim-chroot Thu Apr 28 09:09:34 2011 (r221158)
+++ projects/portbuild/scripts/claim-chroot Thu Apr 28 11:06:14 2011 (r221159)
@@ -18,7 +18,7 @@
# could use a lockf lock, and be able to tell if the setup process was
# still running or died prematurely
-pb=/var/portbuild
+pbd=${PORTBUILD_DATA:-/var/portbuild}
usage () {
echo "usage: claim-chroot arch branch buildid"
@@ -48,7 +48,7 @@ fi
# provide mutual exclusion against other callers, since we only want
# one of them to perform setup
-builddir=${pb}/${arch}/${branch}/builds/${buildid}
+builddir=${pbd}/${arch}/${branch}/builds/${buildid}
# Is the build environment populated? Again we only want a single
# instance to gain setup rights if not.
@@ -64,10 +64,10 @@ if [ ! -f ${builddir}/.ready ]; then
exit 1
fi
-. ${pb}/${arch}/client.conf
-. ${pb}/${arch}/common.conf
-. ${pb}/${arch}/portbuild.conf
-. ${pb}/${arch}/portbuild.$(hostname)
+. ${pbd}/${arch}/client.conf
+. ${pbd}/${arch}/common.conf
+. ${pbd}/${arch}/portbuild.conf
+. ${pbd}/${arch}/portbuild.$(hostname)
buildroot=${scratchdir}
pkgname=${pkgname%.${PKGSUFFIX}}
@@ -109,7 +109,7 @@ for dir in ${chrootdir}/*; do
mkdir ${dir}/used 2>/dev/null || continue
touch ${dir}/used/${pkgname}
if [ -f ${dir}/.dirty ]; then
- ${pb}/scripts/clean-chroot ${arch} ${branch} ${buildid} ${dir} 2 >/dev/null 2>/dev/null &
+ ${builddir}/scripts/clean-chroot ${arch} ${branch} ${buildid} ${dir} 2 >/dev/null 2>/dev/null &
continue
fi
found=1
Modified: projects/portbuild/scripts/clean-chroot
==============================================================================
--- projects/portbuild/scripts/clean-chroot Thu Apr 28 09:09:34 2011 (r221158)
+++ projects/portbuild/scripts/clean-chroot Thu Apr 28 11:06:14 2011 (r221159)
@@ -42,12 +42,12 @@ buildid=$3
chroot=$4
clean=$5
-pb=/var/portbuild
+pbd=${PORTBUILD_DATA:-/var/portbuild}
-. ${pb}/${arch}/client.conf
-. ${pb}/${arch}/common.conf
-. ${pb}/${arch}/portbuild.conf
-. ${pb}/${arch}/portbuild.$(hostname)
+. ${pbd}/${arch}/client.conf
+. ${pbd}/${arch}/common.conf
+. ${pbd}/${arch}/portbuild.conf
+. ${pbd}/${arch}/portbuild.$(hostname)
# directories to clean
cleandirs="${LOCALBASE} /compat /var/db/pkg"
Modified: projects/portbuild/scripts/cleanup-chroots
==============================================================================
--- projects/portbuild/scripts/cleanup-chroots Thu Apr 28 09:09:34 2011 (r221158)
+++ projects/portbuild/scripts/cleanup-chroots Thu Apr 28 11:06:14 2011 (r221159)
@@ -7,7 +7,7 @@
# port builds that have timed out or shut down uncleanly) and prunes
# them to reclaim space.
-pb=/var/portbuild
+pbd=${PORTBUILD_DATA:-/var/portbuild}
kill_procs()
{
@@ -44,9 +44,9 @@ cleanup_mount() {
# note: uname is not being overridden (should not need client.conf here)
arch=$(uname -m)
-. ${pb}/${arch}/client.conf
-. ${pb}/${arch}/portbuild.conf
-. ${pb}/${arch}/portbuild.$(hostname)
+. ${pbd}/${arch}/client.conf
+. ${pbd}/${arch}/portbuild.conf
+. ${pbd}/${arch}/portbuild.$(hostname)
if [ "${use_zfs}" = "1" ]; then
old=$(find ${scratchdir}/*/*/* -prune -mmin +60 2> /dev/null)
Modified: projects/portbuild/scripts/client-metrics
==============================================================================
--- projects/portbuild/scripts/client-metrics Thu Apr 28 09:09:34 2011 (r221158)
+++ projects/portbuild/scripts/client-metrics Thu Apr 28 11:06:14 2011 (r221159)
@@ -6,7 +6,7 @@
# - max vnodes
# - number of packages built in the past hour
-pb=/var/portbuild
+pbd=${PORTBUILD_DATA:-/var/portbuild}
arch=$(uname -m)
me=$(hostname)
@@ -16,10 +16,10 @@ export PATH=/sbin:/bin:/usr/sbin:/usr/bi
gmetric --name="maxvnodes" --value=`sysctl -n kern.maxvnodes` --tmax=120 --dmax=0 --type=uint32 --units="# vnodes"
gmetric --name="vnodes" --value=`sysctl -n vfs.numvnodes` --tmax=120 --dmax=0 --type=uint32 --units="# vnodes"
-if [ -f ${pb}/${arch}/portbuild.conf -a -f ${pb}/${arch}/portbuild.${me} ]; then
- . ${pb}/${arch}/client.conf
- . ${pb}/${arch}/portbuild.conf
- . ${pb}/${arch}/portbuild.${me}
+if [ -f ${pbd}/${arch}/portbuild.conf -a -f ${pbd}/${arch}/portbuild.${me} ]; then
+ . ${pbd}/${arch}/client.conf
+ . ${pbd}/${arch}/portbuild.conf
+ . ${pbd}/${arch}/portbuild.${me}
else
exit 1
fi
Modified: projects/portbuild/scripts/cpdistfiles
==============================================================================
--- projects/portbuild/scripts/cpdistfiles Thu Apr 28 09:09:34 2011 (r221158)
+++ projects/portbuild/scripts/cpdistfiles Thu Apr 28 11:06:14 2011 (r221159)
@@ -6,14 +6,16 @@ if [ $# -lt 3 ]; then
fi
# configurable variables
-pb=/var/portbuild
-. ${pb}/conf/server.conf
+pbc=${PORTBUILD_CHECKOUT:-/var/portbuild}
+pbd=${PORTBUILD_DATA:-/var/portbuild}
+
+. ${pbc}/conf/server.conf
arch=$1
branch=$2
buildid=$3
-builddir=${pb}/${arch}/${branch}/builds/${buildid}
+builddir=${pbd}/${arch}/${branch}/builds/${buildid}
yesreally=0
dryrun=-n
Modified: projects/portbuild/scripts/cppackages
==============================================================================
--- projects/portbuild/scripts/cppackages Thu Apr 28 09:09:34 2011 (r221158)
+++ projects/portbuild/scripts/cppackages Thu Apr 28 11:06:14 2011 (r221159)
@@ -1,11 +1,11 @@
#!/bin/sh
# configurable variables
-pb=/var/portbuild
+pbd=${PORTBUILD_DATA:-/var/portbuild}
-. ${pb}/portbuild.conf
+. ${pbd}/portbuild.conf
-lock=${pb}/cppackages.lock
+lock=${pbd}/cppackages.lock
unset DISPLAY
Modified: projects/portbuild/scripts/do-cleanup-chroots
==============================================================================
--- projects/portbuild/scripts/do-cleanup-chroots Thu Apr 28 09:09:34 2011 (r221158)
+++ projects/portbuild/scripts/do-cleanup-chroots Thu Apr 28 11:06:14 2011 (r221159)
@@ -2,4 +2,7 @@
#
# clean up stale chroots on all client machines. MCL 20081216.
#
-/var/portbuild/scripts/allgohans all -q /var/portbuild/scripts/cleanup-chroots
+
+pbc=${PORTBUILD_CHECKOUT:-/var/portbuild}
+
+${pbc}/scripts/allgohans all -q -scp ${pbc}/scripts/cleanup-chroots
Modified: projects/portbuild/scripts/docppackages
==============================================================================
--- projects/portbuild/scripts/docppackages Thu Apr 28 09:09:34 2011 (r221158)
+++ projects/portbuild/scripts/docppackages Thu Apr 28 11:06:14 2011 (r221159)
@@ -1,7 +1,11 @@
#!/bin/sh
+
if [ $# != 1 ]; then
echo "usage: $0 branch"
exit 1
fi
-echo /var/portbuild/scripts/cppackages $1 | at + 1 minute
+
+pbc=${PORTBUILD_CHECKOUT:-/var/portbuild}
+
+echo ${pbc}/scripts/cppackages $1 | at + 1 minute
atq
Modified: projects/portbuild/scripts/dodistfiles
==============================================================================
--- projects/portbuild/scripts/dodistfiles Thu Apr 28 09:09:34 2011 (r221158)
+++ projects/portbuild/scripts/dodistfiles Thu Apr 28 11:06:14 2011 (r221159)
@@ -30,8 +30,9 @@ arch=$1
branch=$2
buildid=$3
-pb=/var/portbuild
-builddir=${pb}/${arch}/${branch}/builds/${buildid}
+pbd=${PORTBUILD_DATA:-/var/portbuild}
+
+builddir=${pbd}/${arch}/${branch}/builds/${buildid}
distdir=${builddir}/distfiles
Modified: projects/portbuild/scripts/dologs
==============================================================================
--- projects/portbuild/scripts/dologs Thu Apr 28 09:09:34 2011 (r221158)
+++ projects/portbuild/scripts/dologs Thu Apr 28 11:06:14 2011 (r221159)
@@ -4,21 +4,22 @@
arch=$1
versions="latest full"
-pb=/var/portbuild
+pbc=${PORTBUILD_CHECKOUT:-/var/portbuild}
+pbd=${PORTBUILD_DATA:-/var/portbuild}
-. ${pb}/conf/server.conf
-. ${pb}/${arch}/portbuild.conf
+. ${pbc}/conf/server.conf
+. ${pbd}/${arch}/portbuild.conf
-home=${pb}/errorlogs
-scripts=${pb}/scripts
+home=${pbd}/errorlogs
+scripts=${pbc}/scripts
-branches=`cd ${pb}/${arch} && ls -d [1-9]* 2> /dev/null`
+branches=`cd ${pbd}/${arch} && ls -d [1-9]* 2> /dev/null`
for version in ${versions}; do
for branch in ${branches}; do
dir=$home/$arch-$branch-$version
test -d $dir && cd $dir && ${scripts}/processlogs ${arch}
- dir=${pb}/${arch}/${branch}
+ dir=${pbd}/${arch}/${branch}
test -d $dir && cd $dir && ${scripts}/processfail ${arch} ${branch}
done
for branch in ${branches}; do
Modified: projects/portbuild/scripts/dopackages
==============================================================================
--- projects/portbuild/scripts/dopackages Thu Apr 28 09:09:34 2011 (r221158)
+++ projects/portbuild/scripts/dopackages Thu Apr 28 11:06:14 2011 (r221159)
@@ -4,9 +4,10 @@
# main server-side script to run a package build
# configurable variables
-pb=/var/portbuild
+pbc=${PORTBUILD_CHECKOUT:-/var/portbuild}
+pbd=${PORTBUILD_DATA:-/var/portbuild}
-PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:${pb}/scripts
+PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:${pbc}/scripts
# writable by portmgr
umask 002
@@ -52,26 +53,26 @@ buildid=$3
datestamp=$4
shift 4
-. ${pb}/conf/server.conf
-. ${pb}/conf/common.conf
-. ${pb}/scripts/buildenv
+. ${pbc}/conf/server.conf
+. ${pbc}/conf/common.conf
+. ${pbc}/scripts/buildenv
validate_env ${arch} ${branch} || usage
# XXX MCL too early to do this here.
-buildid=$(resolve ${pb} ${arch} ${branch} ${buildid})
+buildid=$(resolve ${pbd} ${arch} ${branch} ${buildid})
if [ -z "${buildid}" ]; then
echo "Invalid build ID ${buildid}"
exit 1
fi
-if [ -f ${pb}/${arch}/portbuild.conf ]; then
- . ${pb}/conf/server.conf
- . ${pb}/${arch}/portbuild.conf
+if [ -f ${pbd}/${arch}/portbuild.conf ]; then
+ . ${pbc}/conf/server.conf
+ . ${pbd}/${arch}/portbuild.conf
else
usage
fi
-pbab=${pb}/${arch}/${branch}
+pbab=${pbd}/${arch}/${branch}
trap "exit 1" 1 2 3 9 10 11 15
@@ -97,7 +98,7 @@ portstar() {
# usage: makeindex pb arch branch builddir [target]
# note: can take ~24 minutes!
makeindex () {
- pb=$1
+ pbc=$1
arch=$2
branch=$3
buildid=$4
@@ -109,7 +110,7 @@ makeindex () {
echo "generating index"
echo "================================================"
echo "index generation started at $(date)"
- ${pb}/scripts/makeindex ${arch} ${branch} ${buildid} ${target} || return 1
+ ${pbc}/scripts/makeindex ${arch} ${branch} ${buildid} ${target} || return 1
echo "index generation ended at $(date)"
echo $(wc -l ${INDEXFILE} | awk '{print $1}') "lines in INDEX"
@@ -142,7 +143,7 @@ checkindex () {
# usage: makeduds pb arch branch builddir [target]
# note: can take ~21 minutes!
makeduds () {
- pb=$1
+ pbc=$1
arch=$2
branch=$3
buildid=$4
@@ -157,7 +158,7 @@ makeduds () {
if [ -e ${builddir}/duds ]; then
cp -p ${builddir}/duds ${builddir}/duds.old
fi
- if ! ${pb}/scripts/makeduds ${arch} ${branch} ${buildid} ${target}; then
+ if ! ${pbc}/scripts/makeduds ${arch} ${branch} ${buildid} ${target}; then
echo "error(s) detected, exiting script at $(date). Failed duds list was:"
cat ${builddir}/duds
mailexit 1
@@ -176,7 +177,7 @@ makeduds () {
# usage: restrictedlist pb arch branch builddir [target]
# note: can take ~25 minutes!
restrictedlist () {
- pb=$1
+ pbc=$1
arch=$2
branch=$3
buildid=$4
@@ -188,7 +189,7 @@ restrictedlist () {
echo "creating restricted list"
echo "================================================"
echo "restricted list generation started at $(date)"
- ${pb}/scripts/makerestr ${arch} ${branch} ${buildid} ${target} || return 1
+ ${pbc}/scripts/makerestr ${arch} ${branch} ${buildid} ${target} || return 1
echo "restricted list generation ended at $(date)"
echo $(grep -c '^#' ${builddir}/restricted.sh) "ports in ${builddir}/restricted.sh"
}
@@ -196,7 +197,7 @@ restrictedlist () {
# usage: cdromlist pb arch branch builddir
# note: can take ~48 minutes!
cdromlist () {
- pb=$1
+ pbc=$1
arch=$2
branch=$3
builddir=$4
@@ -218,7 +219,7 @@ cdromlist () {
# XXX Should generate these as the packages are copied in, instead of all at once at the end
# usage: generatemd5 pb arch branch builddir
generatemd5 () {
- pb=$1
+ pbc=$1
arch=$2
branch=$3
builddir=$4
@@ -231,7 +232,7 @@ generatemd5 () {
dobuild() {
- pb=$1
+ pbc=$1
arch=$2
branch=$3
builddir=$4
@@ -241,7 +242,7 @@ dobuild() {
echo "================================================"
echo "started at $(date)"
start=$(date +%s)
- ${QMANAGER_PATH}/packagebuild ${arch} ${branch} ${buildid} > ${builddir}/${journalname} 2>&1 < /dev/null
+ ${pbc}/qmanager/packagebuild ${arch} ${branch} ${buildid} > ${builddir}/${journalname} 2>&1 < /dev/null
result=$?
if [ $result -ne 0 ]; then
echo "ERROR: packagebuild ${arch} ${branch} ${buildid} failed: see ${builddir}/${journalname} for details"
@@ -422,7 +423,7 @@ if [ ! -d ${builddir} ]; then
fi
# Set up our environment variables
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-projects
mailing list