svn commit: r339299 - head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs

[Allan Jude]

Author: allanjude
Date: Wed Oct 10 22:59:15 2018
New Revision: 339299
URL: https://svnweb.freebsd.org/changeset/base/339299

Log:
  Pull in a follow-on commit to resolve a deadlock in ZFS sequential
  resilver (r334844)
  
  MFV/ZoL: Fix deadlock in IO pipeline
  
  commit a76f3d0437e5e974f0f748f8735af3539443b388
  Author: Brian Behlendorf <behlendorf1 at llnl.gov>
  Date:   Fri Mar 16 16:46:06 2018 -0700
  
      Fix deadlock in IO pipeline
  
      In vdev_queue_aggregate() the zio_execute() bypass should not be
      called under the vdev queue lock.  This can result in a deadlock
      as shown in the stack traces below.
  
      Drop the vdev queue lock then walk the parents of the aggregate IO
      to determine the list of component IOs to be bypassed.  This can
      be done safely without holding the io_lock since the new aggregate
      IO has not yet been returned and its parents cannot change.
  
      ---  THREAD 1 ---
      arc_read()
        zio_nowait()
          zio_vdev_io_start()
            vdev_queue_io() <--- mutex_enter(vq->vq_lock)
              vdev_queue_io_to_issue()
                vdev_queue_aggregate()
                  zio_execute()
              vdev_queue_io_to_issue()
                vdev_queue_aggregate()
                  zio_execute()
                    zio_vdev_io_assess()
                      zio_wait_for_children() <- mutex_enter(zio->io_lock)
  
      --- THREAD 2 --- (inverse order)
      arc_read()
        zio_change_priority() <- mutex_enter(zio->zio_lock)
          vdev_queue_change_io_priority() <- mutex_enter(vq->vq_lock)
  
      Reviewed-by: Tom Caputi <tcaputi at datto.com>
      Reviewed-by: Don Brady <don.brady at delphix.com>
      Signed-off-by: Brian Behlendorf <behlendorf1 at llnl.gov>
  
  Reported by:	ZFS Leadership Meeting
  Reviewed by:	mav
  Approved by:	re (kib)
  Obtained from:	ZFS-on-Linux
  MFC after:	2 weeks
  Sponsored by:	Klara Systems
  Differential Revision:	https://reviews.freebsd.org/D17495

Modified:
  head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_queue.c

Modified: head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_queue.c
==============================================================================
--- head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_queue.c	Wed Oct 10 22:55:31 2018	(r339298)
+++ head/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/vdev_queue.c	Wed Oct 10 22:59:15 2018	(r339299)
@@ -670,6 +670,7 @@ static zio_t *
 vdev_queue_aggregate(vdev_queue_t *vq, zio_t *zio)
 {
 	zio_t *first, *last, *aio, *dio, *mandatory, *nio;
+	zio_link_t *zl = NULL;
 	uint64_t maxgap = 0;
 	uint64_t size;
 	boolean_t stretch;
@@ -813,9 +814,18 @@ vdev_queue_aggregate(vdev_queue_t *vq, zio_t *zio)
 
 		zio_add_child(dio, aio);
 		vdev_queue_io_remove(vq, dio);
+	} while (dio != last);
+
+	/*
+	 * We need to drop the vdev queue's lock to avoid a deadlock that we
+	 * could encounter since this I/O will complete immediately.
+	 */
+	mutex_exit(&vq->vq_lock);
+	while ((dio = zio_walk_parents(aio, &zl)) != NULL) {
 		zio_vdev_io_bypass(dio);
 		zio_execute(dio);
-	} while (dio != last);
+	}
+	mutex_enter(&vq->vq_lock);
 
 	return (aio);
 }