git: a58590631ccc - main - taskqueue: Avoid unbounded epoch read sections

The branch main has been updated by markj:

URL: https://cgit.FreeBSD.org/src/commit/?id=a58590631ccc0fa5bdbbdf88021c6878d644d128

commit a58590631ccc0fa5bdbbdf88021c6878d644d128
Author:     Mark Johnston <markj@FreeBSD.org>
AuthorDate: 2026-07-08 17:12:54 +0000
Commit:     Mark Johnston <markj@FreeBSD.org>
CommitDate: 2026-07-09 13:41:27 +0000

    taskqueue: Avoid unbounded epoch read sections
    
    The taskqueue thread loop tries to avoid entering and exiting net epoch
    read sections for every task.  This reduces the overhead of net epoch
    integration, but the implementation wasn't bounding the length of the
    read section, so a busy taskqueue thread could hold an epoch open for an
    unbounded period.  This is easy to achieve with the epair task, for
    instance.
    
    Bound the number of tasks that we'll execute without observing the
    global epoch, and provide a sysctl to control it.  Let the default bound
    be eight.
    
    Reviewed by:    glebius
    MFC after:      2 weeks
    Differential Revision:  https://reviews.freebsd.org/D58031
---
 sys/kern/subr_gtaskqueue.c | 20 ++++++++++++--------
 sys/kern/subr_taskqueue.c  | 34 ++++++++++++++++++++++++++--------
 sys/sys/taskqueue.h        |  2 ++
 3 files changed, 40 insertions(+), 16 deletions(-)

diff --git a/sys/kern/subr_gtaskqueue.c b/sys/kern/subr_gtaskqueue.c
index e13721ffe1f2..4b0600101b64 100644
--- a/sys/kern/subr_gtaskqueue.c
+++ b/sys/kern/subr_gtaskqueue.c
@@ -342,14 +342,14 @@ gtaskqueue_run_locked(struct gtaskqueue *queue)
 	struct epoch_tracker et;
 	struct gtaskqueue_busy tb;
 	struct gtask *gtask;
-	bool in_net_epoch;
+	unsigned int epochtasks;
 
 	KASSERT(queue != NULL, ("tq is NULL"));
 	TQ_ASSERT_LOCKED(queue);
 	tb.tb_running = NULL;
 	LIST_INSERT_HEAD(&queue->tq_active, &tb, tb_link);
-	in_net_epoch = false;
 
+	epochtasks = 0;
 	while ((gtask = STAILQ_FIRST(&queue->tq_queue)) != NULL) {
 		STAILQ_REMOVE_HEAD(&queue->tq_queue, ta_link);
 		gtask->ta_flags &= ~TASK_ENQUEUED;
@@ -358,19 +358,23 @@ gtaskqueue_run_locked(struct gtaskqueue *queue)
 		TQ_UNLOCK(queue);
 
 		KASSERT(gtask->ta_func != NULL, ("task->ta_func is NULL"));
-		if (!in_net_epoch && TASK_IS_NET(gtask)) {
-			in_net_epoch = true;
-			NET_EPOCH_ENTER(et);
-		} else if (in_net_epoch && !TASK_IS_NET(gtask)) {
+		if (TASK_IS_NET(gtask)) {
+			if (epochtasks++ == 0)
+				NET_EPOCH_ENTER(et);
+		} else if (epochtasks > 0) {
 			NET_EPOCH_EXIT(et);
-			in_net_epoch = false;
+			epochtasks = 0;
 		}
 		gtask->ta_func(gtask->ta_context);
+		if (epochtasks > net_epoch_task_limit) {
+			NET_EPOCH_EXIT(et);
+			epochtasks = 0;
+		}
 
 		TQ_LOCK(queue);
 		wakeup(gtask);
 	}
-	if (in_net_epoch)
+	if (epochtasks > 0)
 		NET_EPOCH_EXIT(et);
 	LIST_REMOVE(&tb, tb_link);
 }
diff --git a/sys/kern/subr_taskqueue.c b/sys/kern/subr_taskqueue.c
index f17e59a441a6..b337aa83d69d 100644
--- a/sys/kern/subr_taskqueue.c
+++ b/sys/kern/subr_taskqueue.c
@@ -43,6 +43,7 @@
 #include <sys/sched.h>
 #include <sys/smp.h>
 #include <sys/stdarg.h>
+#include <sys/sysctl.h>
 #include <sys/taskqueue.h>
 #include <sys/unistd.h>
 
@@ -78,6 +79,19 @@ struct taskqueue {
 	void			*tq_cb_contexts[TASKQUEUE_NUM_CALLBACKS];
 };
 
+static SYSCTL_NODE(_kern, OID_AUTO, taskqueue, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
+    "taskqueue information");
+
+/*
+ * Limit on the number of tasks that may be run in a single epoch section.
+ * It's profitable to batch tasks together, but there must be a bound in order
+ * to maintain system liveness.
+ */
+unsigned int net_epoch_task_limit = 8;
+SYSCTL_UINT(_kern_taskqueue, OID_AUTO, net_epoch_task_limit, CTLFLAG_RWTUN,
+    &net_epoch_task_limit, 0,
+    "Maximum number of tasks to run in an epoch section");
+
 #define	TQ_FLAGS_ACTIVE		(1 << 0)
 #define	TQ_FLAGS_BLOCKED	(1 << 1)
 #define	TQ_FLAGS_UNLOCKED_ENQUEUE	(1 << 2)
@@ -486,15 +500,15 @@ taskqueue_run_locked(struct taskqueue *queue)
 	struct epoch_tracker et;
 	struct taskqueue_busy tb;
 	struct task *task;
-	bool in_net_epoch;
+	unsigned int epochtasks;
 	int pending;
 
 	KASSERT(queue != NULL, ("tq is NULL"));
 	TQ_ASSERT_LOCKED(queue);
 	tb.tb_running = NULL;
 	LIST_INSERT_HEAD(&queue->tq_active, &tb, tb_link);
-	in_net_epoch = false;
 
+	epochtasks = 0;
 	while ((task = STAILQ_FIRST(&queue->tq_queue)) != NULL) {
 		STAILQ_REMOVE_HEAD(&queue->tq_queue, ta_link);
 		if (queue->tq_hint == task)
@@ -507,19 +521,23 @@ taskqueue_run_locked(struct taskqueue *queue)
 		TQ_UNLOCK(queue);
 
 		KASSERT(task->ta_func != NULL, ("task->ta_func is NULL"));
-		if (!in_net_epoch && TASK_IS_NET(task)) {
-			in_net_epoch = true;
-			NET_EPOCH_ENTER(et);
-		} else if (in_net_epoch && !TASK_IS_NET(task)) {
+		if (TASK_IS_NET(task)) {
+			if (epochtasks++ == 0)
+				NET_EPOCH_ENTER(et);
+		} else if (epochtasks > 0) {
 			NET_EPOCH_EXIT(et);
-			in_net_epoch = false;
+			epochtasks = 0;
 		}
 		task->ta_func(task->ta_context, pending);
+		if (epochtasks > net_epoch_task_limit) {
+			NET_EPOCH_EXIT(et);
+			epochtasks = 0;
+		}
 
 		TQ_LOCK(queue);
 		wakeup(task);
 	}
-	if (in_net_epoch)
+	if (epochtasks > 0)
 		NET_EPOCH_EXIT(et);
 	LIST_REMOVE(&tb, tb_link);
 }
diff --git a/sys/sys/taskqueue.h b/sys/sys/taskqueue.h
index 9da364a1a716..b9d4ebe98be4 100644
--- a/sys/sys/taskqueue.h
+++ b/sys/sys/taskqueue.h
@@ -221,4 +221,6 @@ struct taskqueue *taskqueue_create_fast(const char *name, int mflags,
  */
 TASKQUEUE_DECLARE(bus);
 
+extern unsigned int net_epoch_task_limit;
+
 #endif /* !_SYS_TASKQUEUE_H_ */