svn commit: r359469 - head/sys/kern

[Konstantin Belousov]

Author: kib
Date: Mon Mar 30 21:57:28 2020
New Revision: 359469
URL: https://svnweb.freebsd.org/changeset/base/359469

Log:
  kern_sendfile.c: wait for all in-flight ios completion before unwiring pages.
  
  Reviewed by:	glebius, markj
  Tested by:	pho
  Sponsored by:	The FreeBSD Foundation
  MFC after:	1 week
  Differential revision:	https://reviews.freebsd.org/D24038

Modified:
  head/sys/kern/kern_sendfile.c

Modified: head/sys/kern/kern_sendfile.c
==============================================================================
--- head/sys/kern/kern_sendfile.c	Mon Mar 30 21:50:51 2020	(r359468)
+++ head/sys/kern/kern_sendfile.c	Mon Mar 30 21:57:28 2020	(r359469)
@@ -256,6 +256,17 @@ fixspace(int old, int new, off_t off, int *space)
 }
 
 /*
+ * Wait for all in-flight ios to complete, we must not unwire pages
+ * under them.
+ */
+static void
+sendfile_iowait(struct sf_io *sfio, const char *wmesg)
+{
+	while (atomic_load_int(&sfio->nios) != 1)
+		pause(wmesg, 1);
+}
+
+/*
  * I/O completion callback.
  */
 static void
@@ -437,6 +448,8 @@ sendfile_swapin(vm_object_t obj, struct sf_io *sfio, i
 		    i + count == npages ? &rhpages : NULL,
 		    &sendfile_iodone, sfio);
 		if (__predict_false(rv != VM_PAGER_OK)) {
+			sendfile_iowait(sfio, "sferrio");
+
 			/*
 			 * Perform full pages recovery before returning EIO.
 			 * Pages from 0 to npages are wired.
@@ -967,6 +980,7 @@ retry_space:
 			    m != NULL ? SFB_NOWAIT : SFB_CATCH);
 			if (sf == NULL) {
 				SFSTAT_INC(sf_allocfail);
+				sendfile_iowait(sfio, "sfnosf");
 				for (int j = i; j < npages; j++)
 					vm_page_unwire(pa[j], PQ_INACTIVE);
 				if (m == NULL)