[Bug 204484] ZFS caches on 4K GELI devices accumulate inordinate write errors
bugzilla-noreply at freebsd.org
bugzilla-noreply at freebsd.org
Sun Jan 17 20:02:42 UTC 2016
https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=204484
Matthew Dresden <matthew at dresdencraft.com> changed:
What |Removed |Added
----------------------------------------------------------------------------
CC| |matthew at dresdencraft.com
--- Comment #2 from Matthew Dresden <matthew at dresdencraft.com> ---
I have not tried to reproduce this with 512 alignment, but it is still an
current issue affecting the stable release of 10.2. Is this bug getting any
priority currently.
Using mfsbsd boot and installing from ftp.freebsd.org with the script below,
these errors appear on ever cache immediately.
Example was a machine with:
ada0 - 120GB ssd
ada1 - 2tb
ada2 - 2tb
ada3 - 2tb
ada4 - 2tb
add5 - 2tb
ada6 - 127GB ssd
This occurs for me on virtual and physical machines:
Here is a semi automated install script to reproduce my scenario.
Instructions to use:
Boot from mfsbsd usb imaged.
cd /tmp
fetch http://<This Script>
sh install-script.sh
Enter encryption password.
Install completes
take zpool status
reboot
take zpool status
Script Follows:
#!/bin/sh
cd /tmp
read -p "Enter a disk encryption password: " password </dev/tty
echo "${password}" > passfile.txt
PASSFILE="/tmp/passfile.txt"
fetch http://ftp.freebsd.org/pub/FreeBSD/releases/amd64/10.2-RELEASE/kernel.txz
tar xvfz kernel.txz
sysctl kern.geom.debugflags=0x10
dd bs=512 if=/dev/zero of=/dev/ada0 count=10
dd bs=512 if=/dev/zero of=/dev/ada1 count=10
dd bs=512 if=/dev/zero of=/dev/ada2 count=10
dd bs=512 if=/dev/zero of=/dev/ada3 count=10
dd bs=512 if=/dev/zero of=/dev/ada4 count=10
dd bs=512 if=/dev/zero of=/dev/ada5 count=10
dd bs=512 if=/dev/zero of=/dev/ada6 count=10
dd bs=1m if=/dev/zero of=/dev/ada0 &
pid0=$(echo $!)
dd bs=1m if=/dev/zero of=/dev/ada1 &
pid1=$(echo $!)
dd bs=1m if=/dev/zero of=/dev/ada2 &
pid2=$(echo $!)
dd bs=1m if=/dev/zero of=/dev/ada3 &
pid3=$(echo $!)
dd bs=1m if=/dev/zero of=/dev/ada4 &
pid4=$(echo $!)
dd bs=1m if=/dev/zero of=/dev/ada5 &
pid5=$(echo $!)
dd bs=1m if=/dev/zero of=/dev/ada6 &
pid6=$(echo $!)
printf "\nLetting dd do some work!\n"
printf "\n6 minutes to go"
sleep 60
printf "\n5 minutes to go"
sleep 60
printf "\n4 minutes to go"
sleep 60
printf "\n3 minutes to go"
sleep 60
printf "\n2 minutes to go"
sleep 60
printf "\n1 minutes to go"
sleep 60
for i in "$pid0" "$pid1" "$pid2" "$pid3" "$pid4" "$pid5" "$pid6"
do
printf "\nWe don't need to wait any longer -- Killing pid ${i}\n"
kill "$i"
done
kern_mod="/tmp/boot/kernel"
kldload ${kern_mod}/aesni.ko
kldload ${kern_mod}/geom_eli.ko
kldload ${kern_mod}/zfs.ko
rm -Rf "/tmp/boot"
gpart create -s gpt ada0
gpart create -s gpt ada1
gpart create -s gpt ada2
gpart create -s gpt ada3
gpart create -s gpt ada4
gpart create -s gpt ada5
gpart create -s gpt ada6
# Add p1 - non encrpyted
gpart add -s 64K -a 4k -t freebsd-boot ada1 #p1
gpart add -s 64K -a 4k -t freebsd-boot ada2 #p1
gpart add -s 64K -a 4k -t freebsd-boot ada3 #p1
gpart add -s 64K -a 4k -t freebsd-boot ada4 #p1
gpart add -s 64K -a 4k -t freebsd-boot ada5 #p1
# Add p2 - non encrypted boot
gpart add -s 4g -a 4k -t freebsd-zfs -l boot-zfs1 ada1 #p2
gpart add -s 4g -a 4k -t freebsd-zfs -l boot-zfs2 ada2 #p2
gpart add -s 4g -a 4k -t freebsd-zfs -l boot-zfs3 ada3 #p2
gpart add -s 4g -a 4k -t freebsd-zfs -l boot-zfs4 ada4 #p2
gpart add -s 4g -a 4k -t freebsd-zfs -l boot-zfs5 ada5 #p2
# Add p1 - encrypted swap
gpart add -s 20g -a 4k -t freebsd-swap -l swap1 ada0 #p1
gpart add -s 13g -a 4k -t freebsd-swap -l swap2 ada6 #p1
# Add p3 - encrypted zroot
gpart add -s 16g -a 4k -t freebsd-zfs -l zroot-zfs1 ada1 #p3
gpart add -s 16g -a 4k -t freebsd-zfs -l zroot-zfs2 ada2 #p3
gpart add -s 16g -a 4k -t freebsd-zfs -l zroot-zfs3 ada3 #p3
gpart add -s 16g -a 4k -t freebsd-zfs -l zroot-zfs4 ada4 #p3
gpart add -s 16g -a 4k -t freebsd-zfs -l zroot-zfs5 ada5 #p3
# Add p2 - encrypted zroot logzil
gpart add -s 8g -a 4k -t freebsd-zfs -l zroot-log1 ada0 #p2
gpart add -s 8g -a 4k -t freebsd-zfs -l zroot-log2 ada6 #p2
# Add p3 - encrypted zroot cache
gpart add -s 8g -a 4k -t freebsd-zfs -l zroot-cache1 ada0 #p3
gpart add -s 8g -a 4k -t freebsd-zfs -l zroot-cache2 ada6 #p3
# Add p4 - encrypted datastore
gpart add -a 4k -t freebsd-zfs -l datastore-zfs1 ada1 #p4
gpart add -a 4k -t freebsd-zfs -l datastore-zfs2 ada2 #p4
gpart add -a 4k -t freebsd-zfs -l datastore-zfs3 ada3 #p4
gpart add -a 4k -t freebsd-zfs -l datastore-zfs4 ada4 #p4
gpart add -a 4k -t freebsd-zfs -l datastore-zfs5 ada5 #p4
# Add p4 - encrypted datastore logzil
gpart add -s 8g -a 4k -t freebsd-zfs -l datastore-log1 ada0 #p4
gpart add -s 8g -a 4k -t freebsd-zfs -l datastore-log2 ada6 #p4
# Add p5 - encrypted datastore cache
gpart add -a 4k -t freebsd-zfs -l datastore-cache1 ada0 #p5
gpart add -a 4k -t freebsd-zfs -l datastore-cache2 ada6 #p5
gpart set -a bootme -i 2 ada1
gpart set -a bootme -i 2 ada2
gpart set -a bootme -i 2 ada3
gpart set -a bootme -i 2 ada4
gpart set -a bootme -i 2 ada5
gpart bootcode -b /boot/pmbr -p /boot/gptzfsboot -i 1 ada1
gpart bootcode -b /boot/pmbr -p /boot/gptzfsboot -i 1 ada2
gpart bootcode -b /boot/pmbr -p /boot/gptzfsboot -i 1 ada3
gpart bootcode -b /boot/pmbr -p /boot/gptzfsboot -i 1 ada4
gpart bootcode -b /boot/pmbr -p /boot/gptzfsboot -i 1 ada5
mkdir /root/keys
dd if=/dev/random of=/root/keys/boot.key bs=128k count=1
geli init -J $PASSFILE -b -K /root/keys/boot.key -s 4096 -l 256 /dev/gpt/swap1
geli init -J $PASSFILE -b -K /root/keys/boot.key -s 4096 -l 256 /dev/gpt/swap2
geli init -J $PASSFILE -b -K /root/keys/boot.key -s 4096 -l 256
/dev/gpt/zroot-zfs1
geli init -J $PASSFILE -b -K /root/keys/boot.key -s 4096 -l 256
/dev/gpt/zroot-zfs2
geli init -J $PASSFILE -b -K /root/keys/boot.key -s 4096 -l 256
/dev/gpt/zroot-zfs3
geli init -J $PASSFILE -b -K /root/keys/boot.key -s 4096 -l 256
/dev/gpt/zroot-zfs4
geli init -J $PASSFILE -b -K /root/keys/boot.key -s 4096 -l 256
/dev/gpt/zroot-zfs5
geli init -J $PASSFILE -b -K /root/keys/boot.key -s 4096 -l 256
/dev/gpt/zroot-log1
geli init -J $PASSFILE -b -K /root/keys/boot.key -s 4096 -l 256
/dev/gpt/zroot-log2
geli init -J $PASSFILE -b -K /root/keys/boot.key -s 4096 -l 256
/dev/gpt/zroot-cache1
geli init -J $PASSFILE -b -K /root/keys/boot.key -s 4096 -l 256
/dev/gpt/zroot-cache2
geli init -J $PASSFILE -b -K /root/keys/boot.key -s 4096 -l 256
/dev/gpt/datastore-zfs1
geli init -J $PASSFILE -b -K /root/keys/boot.key -s 4096 -l 256
/dev/gpt/datastore-zfs2
geli init -J $PASSFILE -b -K /root/keys/boot.key -s 4096 -l 256
/dev/gpt/datastore-zfs3
geli init -J $PASSFILE -b -K /root/keys/boot.key -s 4096 -l 256
/dev/gpt/datastore-zfs4
geli init -J $PASSFILE -b -K /root/keys/boot.key -s 4096 -l 256
/dev/gpt/datastore-zfs5
geli init -J $PASSFILE -b -K /root/keys/boot.key -s 4096 -l 256
/dev/gpt/datastore-log1
geli init -J $PASSFILE -b -K /root/keys/boot.key -s 4096 -l 256
/dev/gpt/datastore-log2
geli init -J $PASSFILE -b -K /root/keys/boot.key -s 4096 -l 256
/dev/gpt/datastore-cache1
geli init -J $PASSFILE -b -K /root/keys/boot.key -s 4096 -l 256
/dev/gpt/datastore-cache2
geli attach -j $PASSFILE -k /root/keys/boot.key /dev/gpt/swap1
geli attach -j $PASSFILE -k /root/keys/boot.key /dev/gpt/swap2
geli attach -j $PASSFILE -k /root/keys/boot.key /dev/gpt/zroot-zfs1
geli attach -j $PASSFILE -k /root/keys/boot.key /dev/gpt/zroot-zfs2
geli attach -j $PASSFILE -k /root/keys/boot.key /dev/gpt/zroot-zfs3
geli attach -j $PASSFILE -k /root/keys/boot.key /dev/gpt/zroot-zfs4
geli attach -j $PASSFILE -k /root/keys/boot.key /dev/gpt/zroot-zfs5
geli attach -j $PASSFILE -k /root/keys/boot.key /dev/gpt/zroot-log1
geli attach -j $PASSFILE -k /root/keys/boot.key /dev/gpt/zroot-log2
geli attach -j $PASSFILE -k /root/keys/boot.key /dev/gpt/zroot-cache1
geli attach -j $PASSFILE -k /root/keys/boot.key /dev/gpt/zroot-cache2
geli attach -j $PASSFILE -k /root/keys/boot.key /dev/gpt/datastore-zfs1
geli attach -j $PASSFILE -k /root/keys/boot.key /dev/gpt/datastore-zfs2
geli attach -j $PASSFILE -k /root/keys/boot.key /dev/gpt/datastore-zfs3
geli attach -j $PASSFILE -k /root/keys/boot.key /dev/gpt/datastore-zfs4
geli attach -j $PASSFILE -k /root/keys/boot.key /dev/gpt/datastore-zfs5
geli attach -j $PASSFILE -k /root/keys/boot.key /dev/gpt/datastore-log1
geli attach -j $PASSFILE -k /root/keys/boot.key /dev/gpt/datastore-log2
geli attach -j $PASSFILE -k /root/keys/boot.key /dev/gpt/datastore-cache1
geli attach -j $PASSFILE -k /root/keys/boot.key /dev/gpt/datastore-cache2
zpool create zboot raidz1 gpt/boot-zfs1 gpt/boot-zfs2 gpt/boot-zfs3
gpt/boot-zfs4 gpt/boot-zfs5
zfs set checksum=fletcher4 zboot
zfs set atime=off zboot
zpool create -f \
-o altroot=/mnt \
-O canmount=off \
-m none \
zroot raidz1 \
/dev/gpt/zroot-zfs1.eli \
/dev/gpt/zroot-zfs2.eli \
/dev/gpt/zroot-zfs3.eli \
/dev/gpt/zroot-zfs4.eli \
/dev/gpt/zroot-zfs5.eli
zpool add zroot log mirror gpt/zroot-log1.eli gpt/zroot-log2.eli
zpool add zroot cache gpt/zroot-cache1.eli gpt/zroot-cache2.eli
zfs set checksum=fletcher4 zroot
zfs set atime=off zroot
zfs create -o mountpoint=none zroot/ROOT
zfs create -o mountpoint=/ zroot/ROOT/default
zfs create -o mountpoint=/tmp -o compression=lzjb -o setuid=off zroot/tmp
chmod 1777 /mnt/tmp
zfs create -o mountpoint=/usr zroot/usr
zfs create zroot/usr/local
zfs create -o compression=lzjb -o setuid=off zroot/usr/ports
zfs create -o compression=off -o exec=off -o setuid=off
zroot/usr/ports/distfiles
zfs create -o compression=off -o exec=off -o setuid=off
zroot/usr/ports/packages
zfs create -o compression=lzjb -o exec=off -o setuid=off zroot/usr/src
zfs create zroot/usr/obj
zfs create -o mountpoint=/var zroot/var
zfs create -o compression=lzjb -o exec=off -o setuid=off zroot/var/crash
zfs create -o exec=off -o setuid=off zroot/var/db
zfs create -o compression=lzjb -o exec=on -o setuid=off zroot/var/db/pkg
zfs create -o exec=off -o setuid=off zroot/var/empty
zfs create -o compression=lzjb -o exec=off -o setuid=off zroot/var/log
zfs create -o compression=gzip -o exec=off -o setuid=off zroot/var/mail
zfs create -o exec=off -o setuid=off zroot/var/run
zfs create -o compression=lzjb -o exec=on -o setuid=off zroot/var/tmp
chmod 1777 /mnt/var/tmp
zpool create -m /mnt/datastore datastore raidz1 gpt/datastore-zfs1.eli
gpt/datastore-zfs2.eli gpt/datastore-zfs3.eli gpt/datastore-zfs4.eli
gpt/datastore-zfs5.eli
zpool add datastore log mirror gpt/datastore-log1.eli gpt/datastore-log2.eli
zpool add datastore cache gpt/datastore-cache1.eli gpt/datastore-cache2.eli
zfs set checksum=fletcher4 datastore
zfs set atime=off datastore
zfs set compression=lz4
zfs create -o mountpoint=/home -o setuid=off datastore/home
cd /tmp
fetch http://ftp.freebsd.org/pub/FreeBSD/releases/amd64/10.2-RELEASE/base.txz
fetch http://ftp.freebsd.org/pub/FreeBSD/releases/amd64/10.2-RELEASE/doc.txz
fetch http://ftp.freebsd.org/pub/FreeBSD/releases/amd64/10.2-RELEASE/lib32.txz
fetch http://ftp.freebsd.org/pub/FreeBSD/releases/amd64/10.2-RELEASE/ports.txz
fetch http://ftp.freebsd.org/pub/FreeBSD/releases/amd64/10.2-RELEASE/src.txz
for i in base doc kernel lib32 ports src; do \
xz -d -c $i.txz | tar -C /mnt/ -xf - ; \
done
cat << EOF > /mnt/etc/fstab
/dev/ada0p1.eli none swap sw 0 0
/dev/ada6p1.eli none swap sw 0 0
EOF
cat << EOF > /boot/loader.conf
# enable AHCI driver
ahci_load="YES"
# fs modules
zfs_load="YES"
fdescfs_load="YES"
nullfs_load="YES"
# CTL ISCSI
ctl_load="YES"
# Lagg support
if_lagg_load="YES"
# Crypto stuff
geom_eli_load="YES"
crypto_load="YES"
aesni_load="YES"
cryptodev_load="YES"
# pf stuff
pf_load="YES"
pflog_load="YES"
# zfs tuning
vm.kmem_size="8192M"
vm.kmem_size_max="8192M"
vfs.zfs.arc_max="8192M"
vfs.zfs.prefetch_disable="1"
vfs.zfs.zio.use_uma="0"
vfs.zfs.txg.timeout="5"
vfs.zfs.txg.write_limit_override=1073741824
# disk encryption key load
geli_ada0p1_keyfile0_load="YES"
geli_ada0p1_keyfile0_type="ada0p1:geli_keyfile0"
geli_ada0p1_keyfile0_name="/boot/keys/boot.key"
geli_ada0p2_keyfile0_load="YES"
geli_ada0p2_keyfile0_type="ada0p2:geli_keyfile0"
geli_ada0p2_keyfile0_name="/boot/keys/boot.key"
geli_ada0p3_keyfile0_load="YES"
geli_ada0p3_keyfile0_type="ada0p3:geli_keyfile0"
geli_ada0p3_keyfile0_name="/boot/keys/boot.key"
geli_ada0p4_keyfile0_load="YES"
geli_ada0p4_keyfile0_type="ada0p4:geli_keyfile0"
geli_ada0p4_keyfile0_name="/boot/keys/boot.key"
geli_ada0p5_keyfile0_load="YES"
geli_ada0p5_keyfile0_type="ada0p5:geli_keyfile0"
geli_ada0p5_keyfile0_name="/boot/keys/boot.key"
geli_ada6p1_keyfile0_load="YES"
geli_ada6p1_keyfile0_type="ada6p1:geli_keyfile0"
geli_ada6p1_keyfile0_name="/boot/keys/boot.key"
geli_ada6p2_keyfile0_load="YES"
geli_ada6p2_keyfile0_type="ada6p2:geli_keyfile0"
geli_ada6p2_keyfile0_name="/boot/keys/boot.key"
geli_ada6p3_keyfile0_load="YES"
geli_ada6p3_keyfile0_type="ada6p3:geli_keyfile0"
geli_ada6p3_keyfile0_name="/boot/keys/boot.key"
geli_ada6p4_keyfile0_load="YES"
geli_ada6p4_keyfile0_type="ada6p4:geli_keyfile0"
geli_ada6p4_keyfile0_name="/boot/keys/boot.key"
geli_ada6p5_keyfile0_load="YES"
geli_ada6p5_keyfile0_type="ada6p5:geli_keyfile0"
geli_ada6p5_keyfile0_name="/boot/keys/boot.key"
geli_ada1p2_keyfile0_load="YES"
geli_ada1p2_keyfile0_type="ada1p2:geli_keyfile0"
geli_ada1p2_keyfile0_name="/boot/keys/boot.key"
geli_ada1p3_keyfile0_load="YES"
geli_ada1p3_keyfile0_type="ada1p3:geli_keyfile0"
geli_ada1p3_keyfile0_name="/boot/keys/boot.key"
geli_ada1p4_keyfile0_load="YES"
geli_ada1p4_keyfile0_type="ada1p4:geli_keyfile0"
geli_ada1p4_keyfile0_name="/boot/keys/boot.key"
geli_ada2p2_keyfile0_load="YES"
geli_ada2p2_keyfile0_type="ada2p2:geli_keyfile0"
geli_ada2p2_keyfile0_name="/boot/keys/boot.key"
geli_ada2p3_keyfile0_load="YES"
geli_ada2p3_keyfile0_type="ada2p3:geli_keyfile0"
geli_ada2p3_keyfile0_name="/boot/keys/boot.key"
geli_ada2p4_keyfile0_load="YES"
geli_ada2p4_keyfile0_type="ada2p4:geli_keyfile0"
geli_ada2p4_keyfile0_name="/boot/keys/boot.key"
geli_ada3p2_keyfile0_load="YES"
geli_ada3p2_keyfile0_type="ada3p2:geli_keyfile0"
geli_ada3p2_keyfile0_name="/boot/keys/boot.key"
geli_ada3p3_keyfile0_load="YES"
geli_ada3p3_keyfile0_type="ada3p3:geli_keyfile0"
geli_ada3p3_keyfile0_name="/boot/keys/boot.key"
geli_ada3p4_keyfile0_load="YES"
geli_ada3p4_keyfile0_type="ada3p4:geli_keyfile0"
geli_ada3p4_keyfile0_name="/boot/keys/boot.key"
geli_ada4p2_keyfile0_load="YES"
geli_ada4p2_keyfile0_type="ada4p2:geli_keyfile0"
geli_ada4p2_keyfile0_name="/boot/keys/boot.key"
geli_ada4p3_keyfile0_load="YES"
geli_ada4p3_keyfile0_type="ada4p3:geli_keyfile0"
geli_ada4p3_keyfile0_name="/boot/keys/boot.key"
geli_ada4p4_keyfile0_load="YES"
geli_ada4p4_keyfile0_type="ada4p4:geli_keyfile0"
geli_ada4p4_keyfile0_name="/boot/keys/boot.key"
geli_ada5p2_keyfile0_load="YES"
geli_ada5p2_keyfile0_type="ada5p2:geli_keyfile0"
geli_ada5p2_keyfile0_name="/boot/keys/boot.key"
geli_ada5p3_keyfile0_load="YES"
geli_ada5p3_keyfile0_type="ada5p3:geli_keyfile0"
geli_ada5p3_keyfile0_name="/boot/keys/boot.key"
geli_ada5p4_keyfile0_load="YES"
geli_ada5p4_keyfile0_type="ada5p4:geli_keyfile0"
geli_ada5p4_keyfile0_name="/boot/keys/boot.key"
# Mount root
vfs.root.mountfrom="zfs:zroot/ROOT/default"
EOF
cat << EOF > /mnt/etc/rc.conf
# Network
hostname="example.example.com"
ifconfig_em0="DHCP"
# zfs
zfs_enable="YES"
# ssh server
sshd_enable="YES"
# ntp client
ntpd_enable="YES"
ntpd_sync_on_start="YES"
# NFS Server
rpcbind_enable="YES"
rpc_lockd_enable="YES"
rpc_statd_enable="YES"
nfs_server_enable="YES"
# the number of nfsds to run [-n] should be: (Number-of-clients + 1) * 4
nfs_server_flags="-u -t -n 100"
mountd_enable="YES"
mountd_flags="-r -p 100"
#ISCSI Target Native Service
ctld_enable="YES"
# disable Sendmail
sendmail_enable="NO"
sendmail_submit_enable="NO"
sendmail_outbound_enable="NO"
sendmail_msp_queue_enable="NO"
# Set dumpdev to "AUTO" to enable crash dumps, "NO" to disable
dumpdev="AUTO"
# powerd
#powerd_enable="YES"
# motd
motd_enable="yes"
EOF
cat << EOF >> /mnt/etc/sysctl.conf
kern.maxvnodes=260000
EOF
touch /mnt/etc/ctl.conf
touch /mnt/etc/exports
# Final work
cp /boot/zfs/zpool.cache /mnt/boot/zfs/
cp -pR /root/keys /mnt/boot/
mkdir -p /zboot/boot
cp -pR /mnt/boot/* /zboot/boot/
cp /boot/loader.conf /zboot/boot/
# Command to run following an freebsd update
# rsync -alv --human-readable --delete -c -i /boot/ /zboot/boot/ --exclude
loader.conf --exclude keys --exclude boot.key
--
You are receiving this mail because:
You are the assignee for the bug.
More information about the freebsd-fs
mailing list