svn commit: r261953 - in user/cperciva/EC2-build: . ec2-bits ec2-bits/boot ec2-bits/etc ec2-bits/etc.marketplace keys
Colin Percival
cperciva at FreeBSD.org
Sun Feb 16 02:09:48 UTC 2014
Author: cperciva
Date: Sun Feb 16 02:09:42 2014
New Revision: 261953
URL: http://svnweb.freebsd.org/changeset/base/261953
Log:
Add FreeBSD/EC2 image building code; this is what I used for 10.0-RELEASE.
To use this, you need to:
1. Have the devel/awscli port installed.
2. Have an AWS account.
3. Put your AWS keys into aws-keys.conf.
4. Edit the parameters at the top of instructions.txt.
5. Run the commands in instructions.txt.
At some point the task of building EC2 images will be handed off to the
release engineering team and this code will be integrated into head, but
there are some (non-technical) questions to resolve before that will
happen.
Added:
user/cperciva/EC2-build/
user/cperciva/EC2-build/aws-keys.conf (contents, props changed)
user/cperciva/EC2-build/block-mappings-ami.conf (contents, props changed)
user/cperciva/EC2-build/block-mappings-test.conf (contents, props changed)
user/cperciva/EC2-build/block-mappings-windows.conf (contents, props changed)
user/cperciva/EC2-build/block-mappings.conf (contents, props changed)
user/cperciva/EC2-build/cpami.sh (contents, props changed)
user/cperciva/EC2-build/dobuild.sh (contents, props changed)
user/cperciva/EC2-build/doinit.sh (contents, props changed)
user/cperciva/EC2-build/domarketize.sh (contents, props changed)
user/cperciva/EC2-build/ec2-bits/
user/cperciva/EC2-build/ec2-bits/README
user/cperciva/EC2-build/ec2-bits/boot/
user/cperciva/EC2-build/ec2-bits/boot/loader.conf (contents, props changed)
user/cperciva/EC2-build/ec2-bits/ec2-config.sh (contents, props changed)
user/cperciva/EC2-build/ec2-bits/etc/
user/cperciva/EC2-build/ec2-bits/etc.marketplace/
user/cperciva/EC2-build/ec2-bits/etc.marketplace/rc.conf (contents, props changed)
user/cperciva/EC2-build/ec2-bits/etc/fstab
user/cperciva/EC2-build/ec2-bits/etc/rc.conf (contents, props changed)
user/cperciva/EC2-build/ec2-bits/etc/sysctl.conf (contents, props changed)
user/cperciva/EC2-build/ec2-knownhost (contents, props changed)
user/cperciva/EC2-build/instructions.txt (contents, props changed)
user/cperciva/EC2-build/keys/
user/cperciva/EC2-build/s-ami-test-hvm
user/cperciva/EC2-build/s-build-disks
user/cperciva/EC2-build/s-build-images
user/cperciva/EC2-build/s-config-US-W2
user/cperciva/EC2-build/s-copy-images
user/cperciva/EC2-build/s-publish-images
user/cperciva/EC2-build/s-test-images
user/cperciva/EC2-build/ssh-knownhost (contents, props changed)
Added: user/cperciva/EC2-build/aws-keys.conf
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/EC2-build/aws-keys.conf Sun Feb 16 02:09:42 2014 (r261953)
@@ -0,0 +1,4 @@
+[default]
+aws_access_key_id = XXX_KEY_ID_GOES_HERE_XXX
+aws_secret_access_key = XXX_SECRET_KEY_GOES_HERE_XXX
+region = us-east-1
Added: user/cperciva/EC2-build/block-mappings-ami.conf
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/EC2-build/block-mappings-ami.conf Sun Feb 16 02:09:42 2014 (r261953)
@@ -0,0 +1,25 @@
+[
+ {
+ "DeviceName": "/dev/sda1",
+ "Ebs": {
+ "SnapshotId": "@@SNAPID@@",
+ "DeleteOnTermination": false
+ }
+ },
+ {
+ "DeviceName": "/dev/sdb",
+ "VirtualName": "ephemeral0"
+ },
+ {
+ "DeviceName": "/dev/sdc",
+ "VirtualName": "ephemeral1"
+ },
+ {
+ "DeviceName": "/dev/sdd",
+ "VirtualName": "ephemeral2"
+ },
+ {
+ "DeviceName": "/dev/sde",
+ "VirtualName": "ephemeral3"
+ }
+]
\ No newline at end of file
Added: user/cperciva/EC2-build/block-mappings-test.conf
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/EC2-build/block-mappings-test.conf Sun Feb 16 02:09:42 2014 (r261953)
@@ -0,0 +1,9 @@
+[
+ {
+ "DeviceName": "/dev/sda1",
+ "Ebs": {
+ "DeleteOnTermination": true,
+ "VolumeType": "standard"
+ }
+ }
+]
\ No newline at end of file
Added: user/cperciva/EC2-build/block-mappings-windows.conf
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/EC2-build/block-mappings-windows.conf Sun Feb 16 02:09:42 2014 (r261953)
@@ -0,0 +1,18 @@
+[
+ {
+ "DeviceName": "/dev/sdb",
+ "VirtualName": "ephemeral0"
+ },
+ {
+ "DeviceName": "/dev/sdc",
+ "VirtualName": "ephemeral1"
+ },
+ {
+ "DeviceName": "/dev/sdd",
+ "VirtualName": "ephemeral2"
+ },
+ {
+ "DeviceName": "/dev/sde",
+ "VirtualName": "ephemeral3"
+ }
+]
\ No newline at end of file
Added: user/cperciva/EC2-build/block-mappings.conf
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/EC2-build/block-mappings.conf Sun Feb 16 02:09:42 2014 (r261953)
@@ -0,0 +1,33 @@
+[
+ {
+ "DeviceName": "/dev/sda1",
+ "Ebs": {
+ "DeleteOnTermination": true,
+ "VolumeType": "standard"
+ }
+ },
+ {
+ "DeviceName": "/dev/sdf",
+ "Ebs": {
+ "VolumeSize": 10,
+ "DeleteOnTermination": false,
+ "VolumeType": "standard"
+ }
+ },
+ {
+ "DeviceName": "/dev/sdg",
+ "Ebs": {
+ "VolumeSize": 10,
+ "DeleteOnTermination": false,
+ "VolumeType": "standard"
+ }
+ },
+ {
+ "DeviceName": "/dev/sdh",
+ "Ebs": {
+ "VolumeSize": 10,
+ "DeleteOnTermination": false,
+ "VolumeType": "standard"
+ }
+ }
+]
\ No newline at end of file
Added: user/cperciva/EC2-build/cpami.sh
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/EC2-build/cpami.sh Sun Feb 16 02:09:42 2014 (r261953)
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+REGION_TO=$1
+while read REGION_FROM SZ AMI; do
+ if [ ${REGION_FROM} = ${REGION_TO} ]; then
+ continue;
+ fi
+ echo "Copying ${AMI} from ${REGION_FROM} to ${REGION_TO}..."
+
+ AMI_FREEBSD=`aws ec2 copy-image --region ${REGION_TO} \
+ --source-region ${REGION_FROM} --source-image-id ${AMI} \
+ --output text`
+ while aws ec2 describe-images --region ${REGION_TO} \
+ --image-ids ${AMI_FREEBSD} | grep -q pending; do
+ sleep 15;
+ done
+ echo "${REGION_TO} ${SZ} ${AMI_FREEBSD}" >> ami.log
+done < ami-built.log
Added: user/cperciva/EC2-build/dobuild.sh
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/EC2-build/dobuild.sh Sun Feb 16 02:09:42 2014 (r261953)
@@ -0,0 +1,60 @@
+#!/bin/sh
+
+REL=$1
+REPO=$2
+ARCH=$3
+DISK=$4
+HASH=$5
+
+# Download release
+ISO=FreeBSD-${REL}-${ARCH}-disc1.iso
+fetch ${REPO}/${ISO}
+if ! [ `sha256 -q ${ISO}` = "${HASH}" ]; then
+ echo "SHA256 does not match!"
+ exit 1
+fi
+
+# Extract release distribution bits
+rm -rf /usr/dist/$ARCH
+mkdir /usr/dist/$ARCH
+tar -xf $ISO -C /usr/dist/$ARCH usr/freebsd-dist || true
+
+# Partition EBS disk, create a filesystem, and mount it.
+bsdlabel -w /dev/${DISK}
+bsdlabel -B /dev/${DISK}
+newfs -U /dev/${DISK}a
+mkdir -p /mnt/image
+mount /dev/${DISK}a /mnt/image
+
+# Extract release from ISO image. Skip the ports tree (it's probably going
+# to be obsolete before the AMI is used).
+for DIST in base doc games kernel lib32 src; do
+ DISTFILE=/usr/dist/$ARCH/usr/freebsd-dist/$DIST.txz
+ if [ -f $DISTFILE ]; then
+ tar -xf $DISTFILE -C /mnt/image
+ fi
+done
+
+# Install FreeBSD and EC2 configuration.
+tar -cf- -C /home/ec2-user/ec2-bits boot etc | tar -xpof- -C /mnt/image
+sh /home/ec2-user/ec2-bits/ec2-config.sh /mnt/image/etc
+touch /mnt/image/firstboot
+
+# Install FreeBSD EC2 rc.d scripts. Note that these ports and their
+# dependencies do not contain any architecture-specific binaries, so
+# we don't need to worry about cross-building issues.
+mount_unionfs /mnt/image/usr/local /usr/local
+mount_nullfs /mnt/image/var/db/pkg /var/db/pkg
+cd /usr/ports/sysutils/panicmail && make install clean BATCH=YES
+cd /usr/ports/sysutils/ec2-scripts && make install clean BATCH=YES
+cd /usr/ports/sysutils/firstboot-freebsd-update && make NO_IGNORE=YES install clean BATCH=YES
+cd /usr/ports/sysutils/firstboot-pkgs && make NO_IGNORE=YES install clean BATCH=YES
+umount /var/db/pkg
+umount /usr/local
+
+# Add EC2 bits
+tar -cf- -C /home/ec2-user ec2-bits | tar -xpof- -C /mnt/image/root/
+
+# Unmount filesystem
+sync
+umount /mnt/image
Added: user/cperciva/EC2-build/doinit.sh
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/EC2-build/doinit.sh Sun Feb 16 02:09:42 2014 (r261953)
@@ -0,0 +1,24 @@
+#!/bin/sh
+
+# Create a memory disk so we don't waste time writing to EBS
+mdconfig -a -t swap -s 25G -u 0
+newfs /dev/md0
+mkdir -p /mnt/md
+mount /dev/md0 /mnt/md
+for D in ports dist; do
+ rm -rf /usr/$D
+ mkdir /mnt/md/$D
+ ln -s /mnt/md/$D /usr/$D
+done
+rm -rf /var/db/portsnap
+mkdir /mnt/md/portsnapdb
+ln -s /mnt/md/portsnapdb /var/db/portsnap
+
+# Fetch and extract a ports tree
+/usr/bin/time -h portsnap fetch extract
+
+# If we don't have pkg installed yet, install it now.
+if ! [ -f /usr/local/sbin/pkg ]; then
+ cd /usr/ports/ports-mgmt/pkg && make install clean BATCH=YES
+ echo 'WITH_PKGNG=yes' >> /etc/make.conf
+fi
Added: user/cperciva/EC2-build/domarketize.sh
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/EC2-build/domarketize.sh Sun Feb 16 02:09:42 2014 (r261953)
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+SRC=$1
+DEST=$2
+
+# Copy filesystem
+dd if=/dev/${SRC} of=/dev/${DEST} conv=sparse bs=1M
+
+# Mount filesystem and install Marketplace customizations
+mkdir -p /mnt/image
+mount /dev/xbd7a /mnt/image
+tar -cf- -C /home/ec2-user/ec2-bits/etc.marketplace . | tar -xpof- -C /mnt/image/etc
+umount /dev/xbd7a
Added: user/cperciva/EC2-build/ec2-bits/README
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/EC2-build/ec2-bits/README Sun Feb 16 02:09:42 2014 (r261953)
@@ -0,0 +1,7 @@
+Differences between FreeBSD/EC2 and FreeBSD:
+============================================
+
+boot/*, etc/*: FreeBSD config files suitable for the EC2 environment
+etc.marketplace: Configuration changes for AWS Marketplace
+ec2-config.sh: Script to make edits to FreeBSD config files to make
+ them suitable for the EC2 environment
Added: user/cperciva/EC2-build/ec2-bits/boot/loader.conf
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/EC2-build/ec2-bits/boot/loader.conf Sun Feb 16 02:09:42 2014 (r261953)
@@ -0,0 +1,7 @@
+# Make booting fast
+autoboot_delay="-1"
+beastie_disable="YES"
+
+# Make the EC2 console work
+console="comconsole"
+hw.broken_txfifo=1
Added: user/cperciva/EC2-build/ec2-bits/ec2-config.sh
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/EC2-build/ec2-bits/ec2-config.sh Sun Feb 16 02:09:42 2014 (r261953)
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+if [ $# -ne 1 ]; then
+ echo "usage: ec2-config.sh /path/to/etc"
+ exit 1
+fi
+
+ETCDIR=$1
+
+# Disable ttys since they don't exist on EC2
+sed -E -i '' 's/^([^#].*[[:space:]])on/\1off/' $ETCDIR/ttys
Added: user/cperciva/EC2-build/ec2-bits/etc.marketplace/rc.conf
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/EC2-build/ec2-bits/etc.marketplace/rc.conf Sun Feb 16 02:09:42 2014 (r261953)
@@ -0,0 +1,41 @@
+### EC2-specific configuration
+# On first instance boot, download and process user-data.
+ec2_configinit_enable="YES"
+
+# On first instance boot, fetch the EC2 SSH keypair so that the user
+# can log in. Note that if ec2_fetchkey_user is root, sshd_config
+# must contain a PermitRootLogin option in order for this to be useful.
+ec2_fetchkey_enable="YES"
+
+# Use part of any attached EC2 ephemeral disks for swap.
+ec2_ephemeralswap_enable="YES"
+
+# Log the SSH host keys to the EC2 console.
+ec2_loghostkey_enable="YES"
+
+# EC2 uses DHCP; the network interface appears as xn0.
+ifconfig_xn0="DHCP"
+
+### Configuration for some pre-installed packages
+# Fetch security updates the first time the image boots.
+firstboot_freebsd_update_enable="YES"
+
+# Install packages the first time the image boots. Install the awscli port by
+# default, since it's a very useful tool to have when doing anything with AWS.
+firstboot_pkgs_enable="YES"
+firstboot_pkgs_list="awscli"
+
+# Automatically submit panic reports. Depending on your level of paranoia,
+# you may wish to unset panicmail_autosubmit; in that case panic reports will
+# be emailed to root with instructions to forward them (but make sure that you
+# read or forward root's email).
+dumpdev="AUTO"
+panicmail_enable="YES"
+panicmail_autosubmit="YES"
+
+# The freebsd.org mail servers don't accept email directly from EC2, so send
+# panic reports in via this address (which simply forwards them).
+panicmail_sendto="FreeBSD Panic Reporting <cperciva-panicmail at daemonology.net>"
+
+### Standard FreeBSD configuration from here onwards.
+sshd_enable="YES"
Added: user/cperciva/EC2-build/ec2-bits/etc/fstab
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/EC2-build/ec2-bits/etc/fstab Sun Feb 16 02:09:42 2014 (r261953)
@@ -0,0 +1,2 @@
+# Root device
+/dev/ada0a / ufs rw 1 1
Added: user/cperciva/EC2-build/ec2-bits/etc/rc.conf
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/EC2-build/ec2-bits/etc/rc.conf Sun Feb 16 02:09:42 2014 (r261953)
@@ -0,0 +1,48 @@
+### EC2-specific configuration
+# On first instance boot, download and process user-data.
+ec2_configinit_enable="YES"
+
+# On first instance boot, fetch the EC2 SSH keypair so that the user
+# can log in. Note that if ec2_fetchkey_user is root, sshd_config
+# must contain a PermitRootLogin option in order for this to be useful.
+ec2_fetchkey_enable="YES"
+
+# On first instance boot, send the AMI author an email. This (together
+# with panicmail) allows him to get some idea of how stable the platform
+# is; and it also gives him a Warm Fuzzy Feeling when he sees that his
+# work is being used.
+ec2_bootmail_enable="YES"
+ec2_bootmail_addr="cperciva-ec2launch at daemonology.net"
+
+# Use part of any attached EC2 ephemeral disks for swap.
+ec2_ephemeralswap_enable="YES"
+
+# Log the SSH host keys to the EC2 console.
+ec2_loghostkey_enable="YES"
+
+# EC2 uses DHCP; the network interface appears as xn0.
+ifconfig_xn0="DHCP"
+
+### Configuration for some pre-installed packages
+# Fetch security updates the first time the image boots.
+firstboot_freebsd_update_enable="YES"
+
+# Install packages the first time the image boots. Install the awscli port by
+# default, since it's a very useful tool to have when doing anything with AWS.
+firstboot_pkgs_enable="YES"
+firstboot_pkgs_list="awscli"
+
+# Automatically submit panic reports. Depending on your level of paranoia,
+# you may wish to unset panicmail_autosubmit; in that case panic reports will
+# be emailed to root with instructions to forward them (but make sure that you
+# read or forward root's email).
+dumpdev="AUTO"
+panicmail_enable="YES"
+panicmail_autosubmit="YES"
+
+# The freebsd.org mail servers don't accept email directly from EC2, so send
+# panic reports in via this address (which simply forwards them).
+panicmail_sendto="FreeBSD Panic Reporting <cperciva-panicmail at daemonology.net>"
+
+### Standard FreeBSD configuration from here onwards.
+sshd_enable="YES"
Added: user/cperciva/EC2-build/ec2-bits/etc/sysctl.conf
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/EC2-build/ec2-bits/etc/sysctl.conf Sun Feb 16 02:09:42 2014 (r261953)
@@ -0,0 +1,13 @@
+# $FreeBSD$
+#
+# This file is read when going to multi-user and its contents piped thru
+# ``sysctl'' to adjust kernel values. ``man 5 sysctl.conf'' for details.
+#
+
+# Uncomment this to prevent users from seeing information about processes that
+# are being run under another UID.
+#security.bsd.see_other_uids=0
+
+# The EC2 console is one-way; backtraces are useful, the debugger isn't.
+debug.trace_on_panic=1
+debug.debugger_on_panic=0
Added: user/cperciva/EC2-build/ec2-knownhost
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/EC2-build/ec2-knownhost Sun Feb 16 02:09:42 2014 (r261953)
@@ -0,0 +1,17 @@
+#!/bin/sh -e
+
+# I hereby place this script in the public domain -- Colin Percival
+
+# Usage
+if [ $# -lt 1 ]; then
+ echo "usage: ec2-get-console-output INSTANCE | $0 hostname" >/dev/stderr
+ exit 1;
+fi
+
+# Extract fingerprints and pass them to ssh-knownhost
+tr -d '\r' |
+ grep ^ec2: |
+ awk '/BEGIN SSH HOST KEY FINGERPRINTS/, /END SSH HOST KEY FINGERPRINTS/' |
+ grep -v 'HOST KEY FINGERPRINTS' |
+ cut -f 3 -d ' ' |
+ xargs ssh-knownhost $1
Added: user/cperciva/EC2-build/instructions.txt
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/EC2-build/instructions.txt Sun Feb 16 02:09:42 2014 (r261953)
@@ -0,0 +1,25 @@
+# Keys to my AWS account
+export AWS_CONFIG_FILE=aws-keys.conf
+
+# Release configuration
+export REL=10.0-RELEASE
+export REPO=http://people.freebsd.org/~gjb/$REL/
+export HASH_AMD64=9c377b4a4e63443c0b210080694de26133e6a276eddb07c7e00e1c9aebd84109
+export HASH_I386=2c09643b3f79c703e424c03408882369025cec655c24a6d81ee073081ee75ebc
+export TARGET_REGIONS="us-east-1 us-west-1 us-west-2 eu-west-1 ap-southeast-1 ap-northeast-1 sa-east-1 ap-southeast-2"
+export BUILD_REGION="US-W2"
+
+# Build FreeBSD disk images
+. s-build-disks
+
+# Build Amazon Machine Images from our disk images
+. s-build-images
+
+# Copy AMIs to all the other regions
+. s-copy-images
+
+# Test images
+. s-test-images
+
+# Publish images
+. s-publish-images
Added: user/cperciva/EC2-build/s-ami-test-hvm
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/EC2-build/s-ami-test-hvm Sun Feb 16 02:09:42 2014 (r261953)
@@ -0,0 +1,42 @@
+echo "Testing AMI ${AMI_FREEBSD} in ${EC2_REGION}..."
+
+# Create an SSH key pair
+export EC2_KEY=freebsd-buildtest-${AMI_FREEBSD}
+export EC2_KEYFILE=keys/${EC2_KEY}.key
+ssh-keygen -b 2048 -N '' -f ${EC2_KEYFILE} >/dev/null
+
+# Import the key
+aws ec2 import-key-pair --region ${EC2_REGION} --key-name ${EC2_KEY} \
+ --public-key-material file://`pwd`/${EC2_KEYFILE}.pub >/dev/null
+
+# Launch a FreeBSD instance
+I_FREEBSD=`aws ec2 run-instances --region ${EC2_REGION} \
+ --image-id ${AMI_FREEBSD} --instance-type ${EC2_SZ} \
+ --block-device-mappings file://\`pwd\`/block-mappings-test.conf \
+ --key-name ${EC2_KEY} --output text --query 'Instances[*].InstanceId'`
+sleep 5
+while aws ec2 describe-instances --region ${EC2_REGION} \
+ --instance-ids ${I_FREEBSD} | grep -q pending; do
+ sleep 15;
+done
+IP_FREEBSD=`aws ec2 describe-instances --region ${EC2_REGION} \
+ --instance-ids ${I_FREEBSD} --output text \
+ --query 'Reservations[*].Instances[*].PublicIpAddress'`
+
+# Read console output
+sleep 360
+aws ec2 get-console-output --region ${EC2_REGION} \
+ --instance-id ${I_FREEBSD} --output text |
+ tr -d '\r' > console.log/console.log.${EC2_REGION}.${AMI_FREEBSD}
+
+# SSH into FreeBSD instance
+ssh -o StrictHostKeyChecking=no -i ${EC2_KEYFILE} ec2-user@${IP_FREEBSD} \
+ echo "AMI ${AMI_FREEBSD}: FreeBSD \`uname -r\` \`uname -p\` running on ${EC2_SZ} in EC2 region ${EC2_REGION}" \
+ < /dev/null >> ami-test.log
+
+# Kill FreeBSD instance
+aws ec2 terminate-instances --region ${EC2_REGION} --instance-ids ${I_FREEBSD} >/dev/null
+
+# Clean up the SSH key we generated for uploading
+aws ec2 delete-key-pair --region ${EC2_REGION} --key-name ${EC2_KEY} >/dev/null
+rm ${EC2_KEYFILE} ${EC2_KEYFILE}.pub
Added: user/cperciva/EC2-build/s-build-disks
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/EC2-build/s-build-disks Sun Feb 16 02:09:42 2014 (r261953)
@@ -0,0 +1,96 @@
+# Load region configuration
+. s-config-${BUILD_REGION}
+
+echo "Setting up an SSH key..."
+
+# Create an SSH key pair
+export EC2_KEY=freebsd-build
+export EC2_KEYFILE=keys/freebsd-build-${EC2_REGION}.key
+ssh-keygen -b 2048 -N '' -f ${EC2_KEYFILE} >/dev/null
+
+# Import the key into EC2
+aws ec2 import-key-pair --region ${EC2_REGION} --key-name ${EC2_KEY} \
+ --public-key-material file://`pwd`/${EC2_KEYFILE}.pub >/dev/null
+
+# Launch a FreeBSD instance
+echo "Launching FreeBSD build host..."
+I_BUILD=`aws ec2 run-instances --region ${EC2_REGION} --image-id ${AMI_BUILD} \
+ --placement AvailabilityZone=${EC2_ZONE} --instance-type cc2.8xlarge \
+ --block-device-mappings file://\`pwd\`/block-mappings.conf \
+ --key-name ${EC2_KEY} --output text --query 'Instances[*].InstanceId'`
+while aws ec2 describe-instances --region ${EC2_REGION} \
+ --instance-ids ${I_BUILD} | grep -q pending; do
+ sleep 15;
+done
+IP_BUILD=`aws ec2 describe-instances --region ${EC2_REGION} \
+ --instance-ids ${I_BUILD} --output text \
+ --query 'Reservations[*].Instances[*].PublicIpAddress'`
+
+# Read SSH host keys from console
+sleep 240
+aws ec2 get-console-output --region ${EC2_REGION} \
+ --instance-id ${I_BUILD} --output text |
+ tr -d '\r' |
+ env PATH=$PATH:. ec2-knownhost ${IP_BUILD}
+
+# Upload new FreeBSD EC2-HVM bits and the build script
+tar -cf- ec2-bits doinit.sh dobuild.sh domarketize.sh |
+ ssh -i ${EC2_KEYFILE} ec2-user@${IP_BUILD} tar -xpof-
+
+# Prepare for builds
+echo "Preparing to build disk images..."
+ssh -i ${EC2_KEYFILE} ec2-user@$IP_BUILD -t su root -c \
+ "\"sh -e doinit.sh\"" > build-init.log 2>/dev/null
+
+# Build i386
+echo "Building i386 disk image..."
+ssh -i ${EC2_KEYFILE} ec2-user@$IP_BUILD -t su root -c \
+ "\"sh -e dobuild.sh $REL $REPO i386 xbd5 $HASH_I386\"" > build-i386.log 2>/dev/null
+
+# Build amd64
+echo "Building amd64 disk image..."
+ssh -i ${EC2_KEYFILE} ec2-user@$IP_BUILD -t su root -c \
+ "\"sh -e dobuild.sh $REL $REPO amd64 xbd6 $HASH_AMD64\"" > build-amd64.log 2>/dev/null
+
+# Build Marketplace disk
+echo "Building AWS Marketplace disk image..."
+ssh -i ${EC2_KEYFILE} ec2-user@$IP_BUILD -t su root -c \
+ "\"sh -e domarketize.sh xbd6 xbd7\"" > build-marketplace.log 2>/dev/null
+
+echo "Shutting down FreeBSD build host..."
+
+# Find the volumes we wrote root disks to.
+export VOL_HVM_32=`aws ec2 describe-volumes --region ${EC2_REGION} \
+ --filters Name=attachment.instance-id,Values=${I_BUILD} \
+ --filters Name=attachment.device,Values=/dev/sdf \
+ --query 'Volumes[*].VolumeId' --output text`
+export VOL_HVM_64=`aws ec2 describe-volumes --region ${EC2_REGION} \
+ --filters Name=attachment.instance-id,Values=${I_BUILD} \
+ --filters Name=attachment.device,Values=/dev/sdg \
+ --query 'Volumes[*].VolumeId' --output text`
+export VOL_HVM_MRKT=`aws ec2 describe-volumes --region ${EC2_REGION} \
+ --filters Name=attachment.instance-id,Values=${I_BUILD} \
+ --filters Name=attachment.device,Values=/dev/sdh \
+ --query 'Volumes[*].VolumeId' --output text`
+
+# Shut down the FreeBSD instance
+aws ec2 terminate-instances --region ${EC2_REGION} --instance-ids ${I_BUILD} >/dev/null
+
+# Wait until the volumes has finished detaching
+while aws ec2 describe-volumes --region ${EC2_REGION} \
+ --volume-ids ${VOL_HVM_32} | grep -q detaching; do
+ sleep 15;
+done
+while aws ec2 describe-volumes --region ${EC2_REGION} \
+ --volume-ids ${VOL_HVM_64} | grep -q detaching; do
+ sleep 15;
+done
+while aws ec2 describe-volumes --region ${EC2_REGION} \
+ --volume-ids ${VOL_HVM_MRKT} | grep -q detaching; do
+ sleep 15;
+done
+echo "Cleaning up..."
+
+# Clean up the SSH key we generated
+aws ec2 delete-key-pair --region ${EC2_REGION} --key-name ${EC2_KEY} >/dev/null
+rm ${EC2_KEYFILE} ${EC2_KEYFILE}.pub
Added: user/cperciva/EC2-build/s-build-images
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/EC2-build/s-build-images Sun Feb 16 02:09:42 2014 (r261953)
@@ -0,0 +1,162 @@
+describe_images_wait () {
+ while aws ec2 describe-images --region ${EC2_REGION} \
+ --image-ids $1 | grep -q $2; do
+ sleep 15;
+ done
+}
+
+describe_instances_wait () {
+ while aws ec2 describe-instances --region ${EC2_REGION} \
+ --instance-ids $1 | grep -q $2; do
+ sleep 15;
+ done
+}
+
+describe_volumes_wait () {
+ while aws ec2 describe-volumes --region ${EC2_REGION} \
+ --volume-ids $1 | grep -q $2; do
+ sleep 15;
+ done
+}
+
+describe_snapshots_wait () {
+ while aws ec2 describe-snapshots --region ${EC2_REGION} \
+ --snapshot-ids $1 | grep -q $2; do
+ sleep 15;
+ done
+}
+
+register_unixhvm () {
+ VOL_HVM="$1"
+ EC2_SZ="$2"
+ ITYPES="$3"
+ AMILOG="$4"
+
+ echo "Creating volume snapshot..."
+
+ # Create a snapshot
+ SNAP_HVM=`aws ec2 create-snapshot --region ${EC2_REGION} \
+ --volume-id ${VOL_HVM} --query 'SnapshotId' --output text`
+ describe_snapshots_wait ${SNAP_HVM} pending
+
+ echo "Registering ${ITYPES} AMI..."
+
+ # Construct name and description strings
+ NAME="FreeBSD/EC2 ${REL} ${ITYPES}"
+ DESC="FreeBSD ${REL} AMI for ${ITYPES} instances"
+
+ # Register an AMI
+ sed -e "s/@@SNAPID@@/${SNAP_HVM}/" < block-mappings-ami.conf \
+ > ${TMP}/block-mappings.conf
+ AMI_FREEBSD=`aws ec2 register-image --region ${EC2_REGION} --architecture x86_64 \
+ --virtualization-type hvm --name "${NAME}" --description "${DESC}" \
+ --block-device-mappings file://\`pwd\`/${TMP}/block-mappings.conf \
+ --root-device-name "/dev/sda1" --output text`
+
+ # Wait for the AMI to be available
+ describe_images_wait ${AMI_FREEBSD} pending
+
+ # Log the newly built image
+ echo "${EC2_REGION} ${EC2_SZ} ${AMI_FREEBSD}" >> ${AMILOG}
+}
+
+register_defenestrated () {
+ VOL_HVM="$1"
+ EC2_SZ="$2"
+ ITYPES="$3"
+ AMI_WIN="$4"
+ AMILOG="$5"
+
+ # Launch a Windows instance and stop it
+ echo "Creating victim instance for ${ITYPES} image..."
+
+ # Launch a Windows instance
+ I_WINDOWS=`aws ec2 run-instances --region ${EC2_REGION} --image-id ${AMI_WIN} \
+ --placement AvailabilityZone=${EC2_ZONE} --instance-type ${EC2_SZ} \
+ --block-device-mappings file://\`pwd\`/block-mappings-windows.conf \
+ --output text --query 'Instances[*].InstanceId'`
+ describe_instances_wait ${I_WINDOWS} pending
+
+ # Stop the instance
+ aws ec2 stop-instances --region ${EC2_REGION} \
+ --instance-ids ${I_WINDOWS} >/dev/null
+ describe_instances_wait ${I_WINDOWS} stopping
+
+ # Defenestrate the Windows instance
+ echo "Defenestrating victim instance..."
+
+ # Figure out which volume we want to remove
+ VOL_WINDOWS=`aws ec2 describe-volumes --region ${EC2_REGION} \
+ --filters Name=attachment.instance-id,Values=${I_WINDOWS} \
+ --filters Name=attachment.device,Values=/dev/sda1 \
+ --query 'Volumes[*].VolumeId' --output text`
+
+ # Detach and delete the Windows root volume
+ aws ec2 detach-volume --region ${EC2_REGION} --volume-id ${VOL_WINDOWS} > /dev/null
+ describe_volumes_wait ${VOL_WINDOWS} detaching
+ aws ec2 delete-volume --region ${EC2_REGION} --volume-id ${VOL_WINDOWS} >/dev/null
+
+ # Attach the FreeBSD/HVM volume in its place
+ aws ec2 attach-volume --region ${EC2_REGION} --volume-id ${VOL_HVM} \
+ --instance-id ${I_WINDOWS} --device "/dev/sda1" >/dev/null
+ describe_volumes_wait ${VOL_HVM} attaching
+
+ # Create an AMI from the Windows instance
+ echo "Creating ${ITYPES} AMI from defenestrated victim instance..."
+
+ # Construct name and description strings
+ NAME="FreeBSD/EC2 ${REL} ${ITYPES}"
+ DESC="FreeBSD ${REL} AMI for ${ITYPES} instances"
+
+ # Create an AMI
+ AMI_FREEBSD=`aws ec2 create-image --region ${EC2_REGION} \
+ --instance-id ${I_WINDOWS} \
+ --name "${NAME}" --description "${DESC}" \
+ --output text`
+
+ # Wait for the AMI to be available
+ describe_images_wait ${AMI_FREEBSD} pending
+
+ # Kill the Windows instance
+ echo "Shutting down victim instance..."
+
+ # Shut down the Windows instance
+ aws ec2 terminate-instances --region ${EC2_REGION} --instance-ids ${I_WINDOWS} >/dev/null
+
+ # Kill the FreeBSD root disk we attached to it
+ aws ec2 delete-volume --region ${EC2_REGION} --volume-id ${VOL_HVM} >/dev/null
+
+ echo "${EC2_REGION} ${EC2_SZ} ${AMI_FREEBSD}" >> ${AMILOG}
+}
+
+# No AMIs yet
+: > ami-built.log
+
+# Create directory for temporary files
+export TMP=`mktemp -d tmp.XXXXXX`
+
+# Create a native HVM AMI directly from the 64-bit image
+register_unixhvm ${VOL_HVM_64} m3.xlarge "m3 / c3 / i2 / HPC" ami-built.log
+
+# Create a 32-bit AMI by defenestrating a Windows instance
+register_defenestrated ${VOL_HVM_32} m1.small "32-bit m1 / c1 / t1" ${AMI_WINDOWS32} ami-built.log
+
+# Create a 64-bit AMI by defenestrating a Windows instance
+register_defenestrated ${VOL_HVM_64} m1.large "64-bit m1 / m2 / c1 / t1" ${AMI_WINDOWS64} ami-built.log
+
+# Create a "Marketplace" HVM AMI from the Marketplace image
+register_unixhvm ${VOL_HVM_MRKT} m3.xlarge "AWS Marketplace" ami-marketplace.log
+
+# Make AMI and disk snapshot visible to AWS Marketplace
+aws ec2 modify-snapshot-attribute --region ${EC2_REGION} \
+ --snapshot-id ${SNAP_HVM} --attribute createVolumePermission \
+ --operation-type add --user-ids 679593333241 >/dev/null
+aws ec2 modify-image-attribute --region ${EC2_REGION} \
+ --image-id ${AMI_FREEBSD} --attribute launchPermission \
+ --operation-type add --user-ids 679593333241 >/dev/null
+
+# Delete the Marketplace disk
+aws ec2 delete-volume --region ${EC2_REGION} --volume-id ${VOL_HVM_MRKT} >/dev/null
+
+# Clean up temporary directory
+rm -r ${TMP}
Added: user/cperciva/EC2-build/s-config-US-W2
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/EC2-build/s-config-US-W2 Sun Feb 16 02:09:42 2014 (r261953)
@@ -0,0 +1,8 @@
+echo "Setting configuration parameters for US West (Oregon) region..."
+
+# Parameters for the US-West-2 region
+export EC2_REGION=us-west-2
+export EC2_ZONE=us-west-2a
+export AMI_BUILD=ami-aa09819a # FreeBSD 9.1-RELEASE
+export AMI_WINDOWS64=ami-aa09819a
+export AMI_WINDOWS32=ami-b2098182
Added: user/cperciva/EC2-build/s-copy-images
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/EC2-build/s-copy-images Sun Feb 16 02:09:42 2014 (r261953)
@@ -0,0 +1,9 @@
+# We don't need to copy the region where we originally built these...
+cp ami-built.log ami.log
+
+# Copy into each region
+for REGION_TO in $TARGET_REGIONS; do
+ sh cpami.sh $REGION_TO &
+ sleep 1
+done
+wait
Added: user/cperciva/EC2-build/s-publish-images
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/EC2-build/s-publish-images Sun Feb 16 02:09:42 2014 (r261953)
@@ -0,0 +1,19 @@
+echo "Marking EC2 images as public..."
+
+# Mark images as public
+cat ami.log |
+ while read R S AMI; do
+ aws ec2 modify-image-attribute --region ${R} --image-id ${AMI} \
+ --attribute launchPermission --operation-type add \
+ --user-groups all >/dev/null;
+ done
+
+# Add to FreeBSD/EC2 website
+for SZ in m1.large m1.small m3.xlarge; do
+ echo $SZ
+ for R in us-east-1 us-west-1 eu-west-1 ap-southeast-1 ap-northeast-1 us-west-2 sa-east-1 ap-southeast-2; do
+ echo -n "<td>"
+ cat ami.log | grep $R | grep $SZ | cut -f 3 -d ' ' | tr -d '\n'
+ echo "</td>"
+ done
+done
Added: user/cperciva/EC2-build/s-test-images
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/EC2-build/s-test-images Sun Feb 16 02:09:42 2014 (r261953)
@@ -0,0 +1,14 @@
+# Delete old console logs and test logs
+mkdir -p console.log
+rm -f console.log/*
+: > ami-test.log
+
+# Test the AMIs one by one
+while read EC2_REGION EC2_SZ AMI_FREEBSD; do
+ ( . s-ami-test-hvm ) &
+ sleep 1
+done < ami.log
+wait
+
+# Report
+echo "`wc -l < ami-test.log` images succeeded"
Added: user/cperciva/EC2-build/ssh-knownhost
==============================================================================
--- /dev/null 00:00:00 1970 (empty, because file is newly added)
+++ user/cperciva/EC2-build/ssh-knownhost Sun Feb 16 02:09:42 2014 (r261953)
@@ -0,0 +1,56 @@
+#!/bin/sh -e
+
+# I hereby place this script in the public domain -- Colin Percival
+
+# Usage
+if [ $# -lt 1 ]; then
+ echo "usage: $0 host [fingerprint ...]" >/dev/stderr
+ exit 1;
+fi
+
+# Extract host name from command line.
+HOST=$1
+shift;
+
+# Print a warning if no fingerprints were provided.
+if [ $# -lt 1 ]; then
+ echo "$0: No fingerprints provided for host $HOST" >/dev/stderr
+ exit 0;
+fi
+
+# Create a directory for our temporary files.
+D=`mktemp -d "${TMP:-/tmp}/ssh-knownhost.XXXXXX"` || exit 1
+
+# No good keys yet.
+: > $D/goodkeys
+
+# Handle SSH keys of various sorts.
+for KTYPE in rsa1 rsa dsa ecdsa; do
+ ssh-keyscan -t $KTYPE $HOST > $D/hostkey.$KTYPE 2>/dev/null
+ if [ -s $D/hostkey.$KTYPE ]; then
+ KPRINT=`ssh-keygen -lf $D/hostkey.$KTYPE | cut -f 2 -d ' '`
+ GOODKEY=0
+ for KEY in "$@"; do
+ if [ "$KEY" = "$KPRINT" ]; then
+ GOODKEY=1
+ fi
+ done
+ if [ $GOODKEY = 1 ]; then
+ cat $D/hostkey.$KTYPE >> $D/goodkeys
+ else
+ echo "$0: $KTYPE key for $HOST not in provided list" \
+ >/dev/stderr
+ fi
+ fi
+ rm $D/hostkey.$KTYPE
+done
+
+# Add new keys to our known_hosts file.
+sort < $D/goodkeys > $D/goodkeys.tmp
+mv $D/goodkeys.tmp $D/goodkeys
+sort < ~/.ssh/known_hosts | comm -13 - $D/goodkeys > $D/newkeys
+cat $D/newkeys >> ~/.ssh/known_hosts
+
+# Clean up
+rm $D/goodkeys $D/newkeys
+rmdir $D
More information about the svn-src-user
mailing list