git: 7adb9583bfac - stable/13 - GEOM: Relax direct dispatch for GEOM threads.

The branch stable/13 has been updated by mav:

URL: https://cgit.FreeBSD.org/src/commit/?id=7adb9583bfac505f5a5bc79ab9635c7944be8f59

commit 7adb9583bfac505f5a5bc79ab9635c7944be8f59
Author:     Alexander Motin <mav@FreeBSD.org>
AuthorDate: 2022-01-28 19:12:29 +0000
Commit:     Alexander Motin <mav@FreeBSD.org>
CommitDate: 2022-02-11 00:50:45 +0000

    GEOM: Relax direct dispatch for GEOM threads.
    
    The only cases when direct dispatch does not make sense is for I/O
    submission from down thread and for completion from up thread.  In
    all other cases, if both consumer and producer are OK about it, we
    can save on context switches.
    
    MFC after:      2 weeks
    
    (cherry picked from commit ffc1cc95e78ec05a3e1a0aed869e33a44d9f6641)
---
 sys/geom/geom_int.h  | 2 ++
 sys/geom/geom_io.c   | 4 ++--
 sys/geom/geom_kern.c | 4 ++--
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/sys/geom/geom_int.h b/sys/geom/geom_int.h
index 9f2a011b23f0..67c46d715885 100644
--- a/sys/geom/geom_int.h
+++ b/sys/geom/geom_int.h
@@ -69,6 +69,8 @@ void g_io_schedule_up(struct thread *tp);
 
 /* geom_kern.c / geom_kernsim.c */
 void g_init(void);
+extern struct thread *g_up_td;
+extern struct thread *g_down_td;
 extern int g_shutdown;
 extern int g_notaste;
 
diff --git a/sys/geom/geom_io.c b/sys/geom/geom_io.c
index 0ccbf00c1058..7c9dc4164767 100644
--- a/sys/geom/geom_io.c
+++ b/sys/geom/geom_io.c
@@ -561,7 +561,7 @@ g_io_request(struct bio *bp, struct g_consumer *cp)
 
 	direct = (cp->flags & G_CF_DIRECT_SEND) != 0 &&
 	    (pp->flags & G_PF_DIRECT_RECEIVE) != 0 &&
-	    !g_is_geom_thread(curthread) &&
+	    curthread != g_down_td &&
 	    ((pp->flags & G_PF_ACCEPT_UNMAPPED) != 0 ||
 	    (bp->bio_flags & BIO_UNMAPPED) == 0 || THREAD_CAN_SLEEP()) &&
 	    pace == 0;
@@ -653,7 +653,7 @@ g_io_deliver(struct bio *bp, int error)
 
 	direct = (pp->flags & G_PF_DIRECT_SEND) &&
 		 (cp->flags & G_CF_DIRECT_RECEIVE) &&
-		 !g_is_geom_thread(curthread);
+		 curthread != g_up_td;
 	if (direct) {
 		/* Block direct execution if less then half of stack left. */
 		size_t	st, su;
diff --git a/sys/geom/geom_kern.c b/sys/geom/geom_kern.c
index 4b7219591dce..429bd7a05a4b 100644
--- a/sys/geom/geom_kern.c
+++ b/sys/geom/geom_kern.c
@@ -61,8 +61,8 @@ MALLOC_DEFINE(M_GEOM, "GEOM", "Geom data structures");
 struct sx topology_lock;
 
 static struct proc *g_proc;
-static struct thread __read_mostly *g_up_td;
-static struct thread __read_mostly *g_down_td;
+struct thread __read_mostly *g_up_td;
+struct thread __read_mostly *g_down_td;
 static struct thread __read_mostly *g_event_td;
 
 int __read_mostly g_debugflags;