svn commit: r253534 - user/nwhitehorn/condorports

[Nathan Whitehorn]

Author: nwhitehorn
Date: Sun Jul 21 22:20:44 2013
New Revision: 253534
URL: http://svnweb.freebsd.org/changeset/base/253534

Log:
  Fancify tarball preparation, connect entire ports tree. I suspect the
  non-shared-FS thing may have to be abandoned: 2500 ports have a
  .include directive pointing to another directory and there does not seem
  at present to be any way to programmatically detect such dependencies.

Modified:
  user/nwhitehorn/condorports/buildport.sh
  user/nwhitehorn/condorports/buildportsdag.sh
  user/nwhitehorn/condorports/stagebuildfiles.sh

Modified: user/nwhitehorn/condorports/buildport.sh
==============================================================================
--- user/nwhitehorn/condorports/buildport.sh	Sun Jul 21 21:10:53 2013	(r253533)
+++ user/nwhitehorn/condorports/buildport.sh	Sun Jul 21 22:20:44 2013	(r253534)
@@ -14,8 +14,8 @@ scratchdir=$(pwd)
 
 set -e
 sudo /pkgscripts/prepbuildjail $_CONDOR_SLOT $release
-cat $tarballname | sudo jexec $_CONDOR_SLOT tar xvzf -
-sudo jexec $_CONDOR_SLOT make -C $port install USE_PACKAGE_DEPENDS=true BATCH=true DISABLE_MAKE_JOBS=true
+cat $tarballname | sudo jexec $_CONDOR_SLOT tar xvf -
+sudo jexec $_CONDOR_SLOT make -C $port install USE_PACKAGE_DEPENDS=true BATCH=true DISABLE_MAKE_JOBS=true PKGREPOSITORY=/packages PACKAGES=/packages DISTDIR=/distfiles
 sudo jexec $_CONDOR_SLOT make -C $port package PKGREPOSITORY=/packages BATCH=true NO_LATEST_LINK=true
 ls -lh /scratch/$_CONDOR_SLOT/packages
 cp /scratch/$_CONDOR_SLOT/packages/$pkg $scratchdir/$pkg

Modified: user/nwhitehorn/condorports/buildportsdag.sh
==============================================================================
--- user/nwhitehorn/condorports/buildportsdag.sh	Sun Jul 21 21:10:53 2013	(r253533)
+++ user/nwhitehorn/condorports/buildportsdag.sh	Sun Jul 21 22:20:44 2013	(r253534)
@@ -1,13 +1,12 @@
 #!/bin/sh
 : ${PORTSDIR=/usr/ports}
-: ${PKGSDIR=/usr/ports/packages}
+: ${PKGSDIR=$PORTSDIR/packages}
+: ${DISTHORDE=$PORTSDIR/distfiles}
 : ${ARCH=`uname -p`}
 : ${RELEASE=$ARCH/`uname -r`}
 : ${STAGEDIR=$(pwd)/stage-$ARCH/$PKGSDIR}
 
-#(cd $PORTSDIR && make index)
-ports="$(cut -f 2 -d '|' $PORTSDIR/INDEX-`uname -r | cut -f 1 -d .`)"
-ports="/usr/ports/games/sl /usr/ports/ports-mgmt/pkg /usr/ports/japanese/nkf"
+#ports="/usr/ports/games/sl /usr/ports/ports-mgmt/pkg /usr/ports/japanese/nkf"
 
 mkdir -p $STAGEDIR
 
@@ -15,7 +14,9 @@ dagjobs=$(pwd)/ports.dagjobs
 dagdeps=$(pwd)/ports.dagdeps
 dag=$(pwd)/ports.dag
 rm -f $dagjobs $dagdeps
-for job in $ports; do
+for category in $(make -C $PORTSDIR -V SUBDIR); do for port in $(make -C $PORTSDIR/$category -V SUBDIR); do
+	echo $category/$port
+	job=$PORTSDIR/$category/$port
 	cd $job
 	echo JOB $job package.sub >> $dagjobs
 	# One fetch for all archs with dummy fetch job?
@@ -23,7 +24,7 @@ for job in $ports; do
 	export UNAME_p=$ARCH
 	PKGNAME=$(make package-name)
 	TARBALL=$STAGEDIR/$(make -V UNIQUENAME).tgz
-	echo SCRIPT PRE $job stagebuildfiles.sh $job $PKGSDIR $ARCH $TARBALL >> $dagjobs
+	echo SCRIPT PRE $job stagebuildfiles.sh $job $PKGSDIR $ARCH $TARBALL $DISTHORDE >> $dagjobs
 	echo SCRIPT POST $job postbuild.sh $TARBALL \$RETURN >> $dagjobs
 	echo VARS $job BuildArch=\"$ARCH\" >> $dagjobs
 	echo VARS $job port=\"$job\" >> $dagjobs
@@ -31,11 +32,11 @@ for job in $ports; do
 	echo VARS $job pkgdir=\"$PKGSDIR/All\" >> $dagjobs
 	echo VARS $job stagetarball=\"$TARBALL\" >> $dagjobs
 	echo VARS $job release=\"$RELEASE\" >> $dagjobs
-	deps=$(make all-depends-list)
+	deps=$(make -V _DEPEND_DIRS)
 	if [ ! -z "$deps" ]; then
 		echo PARENT $deps CHILD $job >> $dagdeps
 	fi
-done
+done; done
 
 cat $dagjobs $dagdeps > $dag
 rm -f $dagjobs $dagdeps

Modified: user/nwhitehorn/condorports/stagebuildfiles.sh
==============================================================================
--- user/nwhitehorn/condorports/stagebuildfiles.sh	Sun Jul 21 21:10:53 2013	(r253533)
+++ user/nwhitehorn/condorports/stagebuildfiles.sh	Sun Jul 21 22:20:44 2013	(r253534)
@@ -5,10 +5,11 @@
 
 set -e
 export UNAME_p=$3
+DISTHORDE=$5
 
 # Fetch distfiles if unfetched
 cd $1
-make fetch
+make checksum DISTDIR=$DISTHORDE
 make package-links PACKAGES=$2
 mkdir -p $2/All
 
@@ -18,9 +19,10 @@ mkdir -p $2/All
 # Need POST script to clean up
 
 TARBALL=$4
+mkdir -p $(dirname $TARBALL)
 
 # Find distfiles
-DIST_SUBDIR=$(make -V DISTDIR)/$(make -V DIST_SUBDIR)
+DIST_SUBDIR=$DISTHORDE/$(make -V DIST_SUBDIR)
 ALLFILES=$(for f in `make -V ALLFILES`; do echo $DIST_SUBDIR/$f; done)
 
 # Find packages
@@ -30,8 +32,19 @@ PKGLIST=$(for x in $DEPENDS; do make -C 
 # Take an interesting subset of the ports tree
 # TODO: how on Earth to do this robustly?
 PORTSDIR=$(make -V PORTSDIR)
-PORTS_SUBSET="$PORTSDIR/Mk $PORTSDIR/Templates $PORTSDIR/Tools $PORTSDIR/Keywords $PORTSDIR/UIDs $PORTSDIR/GIDs $PORTSDIR/*/Makefile.inc $DEPENDS" # Other important bits?
+PORTS_SUBSET="$PORTSDIR/Mk $PORTSDIR/Templates $PORTSDIR/Tools $PORTSDIR/Keywords $PORTSDIR/UIDs $PORTSDIR/GIDs $PORTSDIR/*/Makefile.inc $DEPENDS $(make -V DIR_DEPENDS) $(make -V MASTERDIR)" # Other important bits?
 
-tar cvzf $TARBALL $ALLFILES $PKGLIST $PORTS_SUBSET $1
+tar cvf $TARBALL.tmp $ALLFILES $PKGLIST $PORTS_SUBSET $1
+rm -rf $TARBALL-dir
+mkdir -p $TARBALL-dir/packages $TARBALL-dir/distfiles
+if [ -n "$ALLFILES" ]; then
+	ln -s $ALLFILES $TARBALL-dir/distfiles
+fi
+if [ -n "$PKGLIST" ]; then
+	ln -s $PKGLIST $TARBALL-dir/packages
+fi
+tar -r -C $TARBALL-dir -v -L -f $TARBALL.tmp distfiles packages
+tar cvzf $TARBALL @$TARBALL.tmp
+rm -rf $TARBALL.tmp $TARBALL-dir
 set +e