git: d03c82c28da8 - main - release: add optional OCI images

Go to: [ bottom of page ] [ top of archives ] [ this month ]
From: Doug Rabson <dfr_at_FreeBSD.org>
Date: Fri, 08 Nov 2024 13:25:38 UTC
The branch main has been updated by dfr:

URL: https://cgit.FreeBSD.org/src/commit/?id=d03c82c28da86e0812b98b051d24ae5980804ad7

commit d03c82c28da86e0812b98b051d24ae5980804ad7
Author:     Doug Rabson <dfr@FreeBSD.org>
AuthorDate: 2024-08-14 15:39:24 +0000
Commit:     Doug Rabson <dfr@FreeBSD.org>
CommitDate: 2024-11-08 13:23:08 +0000

    release: add optional OCI images
    
    This adds three OCI archive format files to the release containing
    FreeBSD base images suitable for static linked, dynamic linked and shell
    workloads. The shell image also contains pkg-bootstrap and can be easily
    extended by installing packages (including pkgbase packages).
    
    Reviewed by: dch, cpersiva, jlduran, zlei
    Differential Revision: https://reviews.freebsd.org/D46759
    MFC after: 2 days
---
 release/Makefile                     | 11 ++++---
 release/Makefile.oci                 | 34 +++++++++++++++++++
 release/release.conf.sample          |  4 +++
 release/release.sh                   | 47 ++++++++++++++++++++++++++-
 release/scripts/make-oci-image.sh    | 63 ++++++++++++++++++++++++++++++++++++
 release/tools/oci-image-dynamic.conf | 11 +++++++
 release/tools/oci-image-minimal.conf | 19 +++++++++++
 release/tools/oci-image-static.conf  | 42 ++++++++++++++++++++++++
 share/examples/Makefile              |  6 ++++
 share/examples/oci/Containerfile.pkg | 27 ++++++++++++++++
 share/examples/oci/README            |  7 ++++
 share/man/man7/release.7             | 14 +++++++-
 12 files changed, 279 insertions(+), 6 deletions(-)

diff --git a/release/Makefile b/release/Makefile
index be628d20ba8d..85da63b0c49a 100644
--- a/release/Makefile
+++ b/release/Makefile
@@ -7,14 +7,16 @@
 #  memstick: Builds memory stick image (memstick.img)
 #  mini-memstick: Builds minimal memory stick image (mini-memstick.img)
 #  ftp: Sets up FTP distribution area (ftp)
-#  release: Invokes real-release, vm-release, and cloudware-release targets
+#  release: Invokes real-release, vm-release, cloudware-release and oci-release targets
 #  real-release: Build all media and FTP distribution area
 #  vm-release: Build all virtual machine image targets
 #  cloudware-release: Build all cloud hosting provider targets
-#  install: Invokes the release-install and vm-install targets
+#  oci-release: Build all OCI container images
+#  install: Invokes the release-install, vm-install and oci-install targets
 #  release-install: Copies all release installation media into ${DESTDIR}
 #  vm-install: Copies all virtual machine images into ${DESTDIR}
 #  cloud-install: Copies non-uploaded cloud images into ${DESTDIR}
+#  oci-install: Copies all OCI container images into ${DESTDIR}
 #
 # Variables affecting the build process:
 #  WORLDDIR: location of src tree -- must have built world and default kernel
@@ -316,7 +318,7 @@ ftp: packagesystem
 	mkdir -p ftp
 	cp *.txz MANIFEST ftp
 
-release:	real-release vm-release cloudware-release
+release:	real-release vm-release cloudware-release oci-release
 	${MAKE} -C ${.CURDIR} ${.MAKEFLAGS} release-done
 	true
 
@@ -327,7 +329,7 @@ real-release:
 	${MAKE} -C ${.CURDIR} ${.MAKEFLAGS} obj
 	${MAKE} -C ${.CURDIR} ${.MAKEFLAGS} ${RELEASE_TARGETS}
 
-install:	release-install vm-install .WAIT cloud-install
+install:	release-install vm-install oci-install .WAIT cloud-install
 
 release-install:
 .if defined(DESTDIR) && !empty(DESTDIR)
@@ -347,3 +349,4 @@ release-install:
 
 .include "${.CURDIR}/Makefile.inc1"
 .include "${.CURDIR}/Makefile.vm"
+.include "${.CURDIR}/Makefile.oci"
diff --git a/release/Makefile.oci b/release/Makefile.oci
new file mode 100644
index 000000000000..461c95f49636
--- /dev/null
+++ b/release/Makefile.oci
@@ -0,0 +1,34 @@
+#
+#
+#
+# Makefile for building OCI container images.
+#
+
+.if defined(WITH_OCIIMAGES) && !empty(WITH_OCIIMAGES)
+OCI_IMAGES= static dynamic minimal
+.endif
+
+oci-install:
+.if defined(WITH_OCIIMAGES) && !empty(WITH_OCIIMAGES)
+	mkdir -p ${DESTDIR}/ociimages
+. for _IMG in ${OCI_IMAGES}
+	cp -p ${.OBJDIR}/container-image-${_IMG}.txz ${DESTDIR}/ociimages
+. endfor
+.endif
+
+OCI_TARGETS=
+OCI_DEPS_static=
+OCI_DEPS_dynamic= container-image-static.txz
+OCI_DEPS_minimal= container-image-dynamic.txz
+
+.for _IMG in ${OCI_IMAGES}
+OCI_TARGETS+= container-image-${_IMG}.txz
+container-image-${_IMG}.txz: ${OCI_DEPS_${_IMG}}
+	sh ${.CURDIR}/scripts/make-oci-image.sh ${.CURDIR} ${REVISION} ${BRANCH} ${TARGET_ARCH} ${_IMG}
+	skopeo copy \
+		containers-storage:localhost/freebsd${REVISION:R}-${_IMG}:latest \
+		oci-archive:${.OBJDIR}/container-image-${_IMG}.tar:freebsd${REVISION:R}-${_IMG}:${REVISION}-${BRANCH}-${TARGET_ARCH}
+	${XZ_CMD} < ${.OBJDIR}/container-image-${_IMG}.tar > ${.OBJDIR}/container-image-${_IMG}.txz
+.endfor
+
+oci-release: ${OCI_TARGETS}
diff --git a/release/release.conf.sample b/release/release.conf.sample
index c306efb13b65..e583e49828d4 100644
--- a/release/release.conf.sample
+++ b/release/release.conf.sample
@@ -114,3 +114,7 @@ PORTBRANCH="main"
 ## If WITH_CLOUDWARE is set to a non-empty value, this is a list of providers
 ## to create disk images.
 #CLOUDWARE="EC2 GCE ORACLE VAGRANT-VIRTUALBOX VAGRANT-VMWARE"
+
+## If WITH_OCIIMAGES is set to a non-empty value, build Open Container
+## Initiative (OCI) base images as part of the release.
+#WITH_OCIIMAGES=
diff --git a/release/release.sh b/release/release.sh
index cee0fbd5643d..d6752e016994 100755
--- a/release/release.sh
+++ b/release/release.sh
@@ -120,6 +120,9 @@ env_setup() {
 	# cloud providers as part of the release.
 	WITH_CLOUDWARE=
 
+	# Set to non-empty to build OCI images as part of the release
+	WITH_OCIIMAGES=
+
 	return 0
 } # env_setup()
 
@@ -195,7 +198,8 @@ env_check() {
 	RELEASE_RMAKEFLAGS="${ARCH_FLAGS} ${RELEASE_FLAGS} \
 		KERNCONF=\"${KERNEL}\" ${CONF_FILES} ${SRCPORTS} \
 		WITH_DVD=${WITH_DVD} WITH_VMIMAGES=${WITH_VMIMAGES} \
-		WITH_CLOUDWARE=${WITH_CLOUDWARE} XZ_THREADS=${XZ_THREADS}"
+		WITH_CLOUDWARE=${WITH_CLOUDWARE} WITH_OCIIMAGES=${WITH_OCIIMAGES} \
+		XZ_THREADS=${XZ_THREADS}"
 
 	return 0
 } # env_check()
@@ -288,6 +292,44 @@ extra_chroot_setup() {
 		fi
 	fi
 
+	if [ ! -z "${WITH_OCIIMAGES}" ]; then
+		# Install buildah and skopeo from ports if the ports tree is available;
+		# otherwise install the pkg.
+		if [ -d ${CHROOTDIR}/usr/ports ]; then
+			# Trick the ports 'run-autotools-fixup' target to do the right
+			# thing.
+			_OSVERSION=$(chroot ${CHROOTDIR} /usr/bin/uname -U)
+			REVISION=$(chroot ${CHROOTDIR} make -C /usr/src/release -V REVISION)
+			BRANCH=$(chroot ${CHROOTDIR} make -C /usr/src/release -V BRANCH)
+			UNAME_r=${REVISION}-${BRANCH}
+			GITUNSETOPTS="CONTRIB CURL CVS GITWEB GUI HTMLDOCS"
+			GITUNSETOPTS="${GITUNSETOPTS} ICONV NLS P4 PERL"
+			GITUNSETOPTS="${GITUNSETOPTS} SEND_EMAIL SUBTREE SVN"
+			GITUNSETOPTS="${GITUNSETOPTS} PCRE PCRE2"
+			PBUILD_FLAGS="OSVERSION=${_OSVERSION} BATCH=yes"
+			PBUILD_FLAGS="${PBUILD_FLAGS} UNAME_r=${UNAME_r}"
+			PBUILD_FLAGS="${PBUILD_FLAGS} OSREL=${REVISION}"
+			PBUILD_FLAGS="${PBUILD_FLAGS} WRKDIRPREFIX=/tmp/ports"
+			PBUILD_FLAGS="${PBUILD_FLAGS} DISTDIR=/tmp/distfiles"
+			for _PORT in sysutils/buildah sysutils/skopeo; do
+				eval chroot ${CHROOTDIR} env ${PBUILD_FLAGS} make -C \
+				     /usr/ports/${_PORT} \
+				     FORCE_PKG_REGISTER=1 deinstall install clean distclean
+			done
+		else
+			eval chroot ${CHROOTDIR} env ASSUME_ALWAYS_YES=yes \
+				pkg install -y sysutils/buildah sysutils/skopeo
+			eval chroot ${CHROOTDIR} env ASSUME_ALWAYS_YES=yes \
+				pkg clean -y
+		fi
+		# Use the vfs storage driver so that this works whether or not
+		# the build directory is on ZFS. The images are small so the
+		# performance difference is negligible.
+		eval chroot ${CHROOTDIR} sed -I .bak -e '/^driver/s/zfs/vfs/' /usr/local/etc/containers/storage.conf
+		# Remove any stray images from previous builds
+		eval chroot ${CHROOTDIR} buildah rmi -af
+	fi
+
 	if [ ! -z "${EMBEDDEDPORTS}" ]; then
 		_OSVERSION=$(chroot ${CHROOTDIR} /usr/bin/uname -U)
 		REVISION=$(chroot ${CHROOTDIR} make -C /usr/src/release -V REVISION)
@@ -323,6 +365,9 @@ chroot_build_target() {
 	fi
 	eval chroot ${CHROOTDIR} make -C /usr/src ${RELEASE_WMAKEFLAGS} buildworld
 	eval chroot ${CHROOTDIR} make -C /usr/src ${RELEASE_KMAKEFLAGS} buildkernel
+	if [ ! -z "${WITH_OCIIMAGES}" ]; then
+		eval chroot ${CHROOTDIR} make -C /usr/src ${RELEASE_WMAKEFLAGS} packages
+	fi
 
 	return 0
 } # chroot_build_target
diff --git a/release/scripts/make-oci-image.sh b/release/scripts/make-oci-image.sh
new file mode 100644
index 000000000000..a139a38d1930
--- /dev/null
+++ b/release/scripts/make-oci-image.sh
@@ -0,0 +1,63 @@
+#! /bin/sh
+
+# Build an Open Container Initiative (OCI) container image
+
+curdir=$1; shift
+rev=$1; shift
+branch=$1; shift
+arch=$1; shift
+image=$1; shift
+
+major=${rev%.*}
+minor=${rev#*.}
+
+abi=FreeBSD:${major}:${arch}
+
+echo "Building OCI freebsd${major}-${image} image for ${abi}"
+
+. ${curdir}/tools/oci-image-${image}.conf
+
+init_workdir() {
+	local abi=$1; shift
+	local workdir=$(mktemp -d -t oci-images)
+
+	mkdir ${workdir}/repos
+	cat > ${workdir}/repos/base.conf <<EOF
+FreeBSD-base: {
+  url: "file:///usr/obj/usr/src/repo/${abi}/latest"
+  signature_type: "none"
+  fingerprints: "none"
+}
+EOF
+	cp /etc/pkg/FreeBSD.conf ${workdir}/repos
+	echo ${workdir}
+}
+
+install_packages() {
+	local abi=$1; shift
+	local workdir=$1; shift
+	local rootdir=$1; shift
+	if [ ! -d ${rootdir}/usr/share/keys/pkg/trusted ]; then
+		mkdir -p ${rootdir}/usr/share/keys/pkg/trusted
+	fi
+	cp /usr/share/keys/pkg/trusted/* ${rootdir}/usr/share/keys/pkg/trusted
+	# We install the packages and then remove repository metadata (keeping the
+	# metadata for what was installed). This trims more than 40Mb from the
+	# resulting image.
+	env IGNORE_OSVERSION=yes ABI=${abi} pkg --rootdir ${rootdir} --repo-conf-dir ${workdir}/repos \
+		install -yq "$@" || exit $?
+	rm -rf ${rootdir}/var/db/pkg/repos
+}
+
+workdir=$(init_workdir ${abi})
+if [ -n "${OCI_BASE_IMAGE}" ]; then
+	base_image=freebsd${major}-${OCI_BASE_IMAGE}
+else
+	base_image=scratch
+fi
+
+c=$(buildah from ${base_image})
+m=$(buildah mount $c)
+oci_image_build
+buildah unmount $c
+buildah commit --rm $c freebsd${major}-${image}:latest
diff --git a/release/tools/oci-image-dynamic.conf b/release/tools/oci-image-dynamic.conf
new file mode 100644
index 000000000000..b146ff2cf7c3
--- /dev/null
+++ b/release/tools/oci-image-dynamic.conf
@@ -0,0 +1,11 @@
+#! /bin/sh
+
+# Build Open Container Initiative (OCI) container image suitable as a base for
+# dynamic-linked workloads. This adds libraries from the FreeBSD-clibs and
+# FreeBSD-openssl-lib packages.
+
+OCI_BASE_IMAGE=static
+
+oci_image_build() {
+	install_packages ${abi} ${workdir} $m FreeBSD-clibs FreeBSD-openssl-lib
+}
diff --git a/release/tools/oci-image-minimal.conf b/release/tools/oci-image-minimal.conf
new file mode 100644
index 000000000000..122da1905436
--- /dev/null
+++ b/release/tools/oci-image-minimal.conf
@@ -0,0 +1,19 @@
+#! /bin/sh
+
+# Build Open Container Initiative (OCI) container image suitable as a base for
+# shell-based workloads. This adds FreeBSD-runtime, FreeBSD-pkg-bootstrap and a
+# handful of others packages to create a small image which can be easily
+# extended by installing packages.
+
+OCI_BASE_IMAGE=dynamic
+
+oci_image_build() {
+	install_packages ${abi} ${workdir} $m \
+			 FreeBSD-runtime \
+			 FreeBSD-certctl \
+			 FreeBSD-kerberos-lib \
+			 FreeBSD-libexecinfo \
+			 FreeBSD-rc \
+			 FreeBSD-pkg-bootstrap \
+			 FreeBSD-mtree
+}
diff --git a/release/tools/oci-image-static.conf b/release/tools/oci-image-static.conf
new file mode 100644
index 000000000000..27cfb3c6778c
--- /dev/null
+++ b/release/tools/oci-image-static.conf
@@ -0,0 +1,42 @@
+#! /bin/sh
+
+# Build Open Container Initiative (OCI) container image suitable as a base for
+# static-linked workloads. This contains mtree directories, SSL certificates and
+# a few other config files.
+
+OCI_BASE_IMAGE=
+
+oci_image_build() {
+	mtree -deU -p $m/ -f /etc/mtree/BSD.root.dist > /dev/null
+	mtree -deU -p $m/var -f /etc/mtree/BSD.var.dist > /dev/null
+	mtree -deU -p $m/usr -f /etc/mtree/BSD.usr.dist > /dev/null
+	mtree -deU -p $m/usr/include -f /etc/mtree/BSD.include.dist > /dev/null
+	mtree -deU -p $m/usr/lib -f /etc/mtree/BSD.debug.dist > /dev/null
+	install_packages ${abi} ${workdir} $m FreeBSD-caroot FreeBSD-zoneinfo
+	cp /etc/master.passwd $m/etc
+	pwd_mkdb -p -d $m/etc $m/etc/master.passwd || return $?
+	cp /etc/group $m/etc || return $?
+	cp /etc/termcap.small $m/etc/termcap.small || return $?
+	cp /etc/termcap.small $m/usr/share/misc/termcap || return $?
+	env DESTDIR=$m /usr/sbin/certctl rehash
+	# Generate a suitable repo config for pkgbase
+	case ${branch} in
+		CURRENT|STABLE|BETA*)
+			repo=base_latest
+			;;
+		*)
+			repo=base_release_${minor}
+			;;
+	esac
+	mkdir -p $m/usr/local/etc/pkg/repos
+	cat > $m/usr/local/etc/pkg/repos/base.conf <<EOF
+FreeBSD-base: {
+  url: "https://pkg.FreeBSD.org/\${ABI}/${repo}",
+  mirror_type: "srv",
+  signature_type: "fingerprints",
+  fingerprints: "/usr/share/keys/pkg",
+  enabled: yes
+}
+EOF
+
+}
diff --git a/share/examples/Makefile b/share/examples/Makefile
index 58f5a2109ec3..560fdae6de5b 100644
--- a/share/examples/Makefile
+++ b/share/examples/Makefile
@@ -21,6 +21,7 @@ LDIRS=	BSD_daemon \
 	libvgl \
 	mdoc \
 	netgraph \
+	oci \
 	perfmon \
 	ppi \
 	ppp \
@@ -199,6 +200,11 @@ SE_NETGRAPH= \
 	virtual.chain \
 	virtual.lan \
 
+SE_DIRS+=	oci
+SE_OCI= \
+	README \
+	Containerfile.pkg
+
 SE_DIRS+=	perfmon
 SE_PERFMON= \
 	Makefile \
diff --git a/share/examples/oci/Containerfile.pkg b/share/examples/oci/Containerfile.pkg
new file mode 100644
index 000000000000..ed8ac7caf4b6
--- /dev/null
+++ b/share/examples/oci/Containerfile.pkg
@@ -0,0 +1,27 @@
+# This is an example showing how to extend the freebsd-minimal OCI image by
+# installing additional packages while keeping the resulting image as small as
+# possible.
+
+# The OS version matching the desired freebsd-minimal image
+ARG version=15.0-CURRENT-amd64
+
+# Select freebsd-minimal as our starting point.
+FROM localhost/freebsd-minimal:${version}
+
+# A list of package(s) to install
+ARG packages
+
+# Install package management tools. We specify 'FreeBSD' as the repository to
+# use for downloading pkg since the freebsd-minimal image has both FreeBSD and
+# FreeBSD-base pkg repo configs installed and FreeBSD-base does not contain the
+# pkg package.
+RUN env ASSUME_ALWAYS_YES=yes pkg bootstrap -r FreeBSD && pkg update
+
+# Install some package(s).
+RUN pkg install -y ${packages}
+
+# Clean up and remove package management overhead. We delete downloaded
+# packages, uninstall pkg and delete the repository metadata downloaded by 'pkg
+# install'.  This retains the record of which packages are installed in the
+# image.
+RUN pkg clean -ay && pkg delete -fy pkg && rm -rf /var/db/pkg/repos
diff --git a/share/examples/oci/README b/share/examples/oci/README
new file mode 100644
index 000000000000..212687251754
--- /dev/null
+++ b/share/examples/oci/README
@@ -0,0 +1,7 @@
+This example Containerfile shows how to add packages to freebsd-minimal while
+minimising the package metadata overhead.
+
+For instance, To build a new image called 'my-new-image:latest' containing the
+nginx package:
+
+# podman build --squash --build-arg packages=nginx --tag my-new-image:latest -f Containerfile.pkg
diff --git a/share/man/man7/release.7 b/share/man/man7/release.7
index c81f540d9463..5c5c11efd0fb 100644
--- a/share/man/man7/release.7
+++ b/share/man/man7/release.7
@@ -22,7 +22,7 @@
 .\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 .\" SUCH DAMAGE.
 .\"
-.Dd August 6, 2023
+.Dd September 26, 2024
 .Dt RELEASE 7
 .Os
 .Sh NAME
@@ -443,6 +443,18 @@ values, run:
 cd /usr/src
 make -C release list-cloudware
 .Ed
+.Sh OCI IMAGES
+The
+.Fx
+release build tools have experimental support for building
+Open Container Initiative (OCI) format container base images.
+This is enabled using a
+.Fa release.conf
+variable:
+.Bl -tag -width Ev
+.It Va WITH_OCIIMAGES
+Set to a non-null value to build OCI base images.
+.El
 .Sh MAKEFILE TARGETS
 The release makefile
 .Pq Pa src/release/Makefile