git: fe6953f70fab - stable/13 - aio: Fix more synchronization issues in aio_biowakeup.

The branch stable/13 has been updated by jhb:

URL: https://cgit.FreeBSD.org/src/commit/?id=fe6953f70fab1f89de198d3dbc885d6f89a44883

commit fe6953f70fab1f89de198d3dbc885d6f89a44883
Author:     John Baldwin <jhb@FreeBSD.org>
AuthorDate: 2023-02-15 21:32:52 +0000
Commit:     John Baldwin <jhb@FreeBSD.org>
CommitDate: 2023-09-06 21:56:09 +0000

    aio: Fix more synchronization issues in aio_biowakeup.
    
    - Use atomic_store to set job->error.  atomic_set does an or
      operation, not assignment.
    
    - Use refcount_* to manage job->nbio.
    
      This ensures proper memory barriers are present so that the last bio
      won't see a possibly stale value of job->error.
    
    - Don't re-read job->error after reading it via atomic_load.
    
    Reported by:    markj (1)
    Reviewed by:    mjg, markj
    Differential Revision:  https://reviews.freebsd.org/D38611
    
    (cherry picked from commit 98844e99d40a90ae89d84762e07150af3a8f89bd)
---
 sys/kern/vfs_aio.c | 11 ++++++-----
 sys/sys/aio.h      |  2 +-
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/sys/kern/vfs_aio.c b/sys/kern/vfs_aio.c
index 0fbe094d8f44..1bdd32ed7969 100644
--- a/sys/kern/vfs_aio.c
+++ b/sys/kern/vfs_aio.c
@@ -1279,7 +1279,7 @@ aio_qbio(struct proc *p, struct kaiocb *job)
 	}
 
 	bios = malloc(sizeof(struct bio *) * iovcnt, M_TEMP, M_WAITOK);
-	atomic_store_int(&job->nbio, iovcnt);
+	refcount_init(&job->nbio, iovcnt);
 	for (i = 0; i < iovcnt; i++) {
 		struct vm_page** pages;
 		struct bio *bp;
@@ -2487,15 +2487,16 @@ aio_biowakeup(struct bio *bp)
 	 * error of whichever failed bio completed last.
 	 */
 	if (flags & BIO_ERROR)
-		atomic_set_int(&job->error, bio_error);
+		atomic_store_int(&job->error, bio_error);
 	if (opcode & LIO_WRITE)
 		atomic_add_int(&job->outblock, nblks);
 	else
 		atomic_add_int(&job->inblock, nblks);
 
-	if (atomic_fetchadd_int(&job->nbio, -1) == 1) {
-		if (atomic_load_int(&job->error))
-			aio_complete(job, -1, job->error);
+	if (refcount_release(&job->nbio)) {
+		bio_error = atomic_load_int(&job->error);
+		if (bio_error != 0)
+			aio_complete(job, -1, bio_error);
 		else
 			aio_complete(job, atomic_load_long(&job->nbytes), 0);
 	}
diff --git a/sys/sys/aio.h b/sys/sys/aio.h
index e9c5bdc63c87..a1aa96efed09 100644
--- a/sys/sys/aio.h
+++ b/sys/sys/aio.h
@@ -149,7 +149,7 @@ struct kaiocb {
 	aio_handle_fn_t *handle_fn;	/* (c) backend handle function */
 	union {				/* Backend-specific data fields */
 		struct {		/* BIO backend */
-			int	nbio;	/* Number of remaining bios */
+			volatile u_int nbio; /* Number of remaining bios */
 			int	error;	/* Worst error of all bios */
 			long	nbytes;	/* Bytes completed so far */
 		};