svn commit: r357771 - in head/sys: kern sys
Gleb Smirnoff
glebius at FreeBSD.org
Tue Feb 11 18:48:09 UTC 2020
Author: glebius
Date: Tue Feb 11 18:48:07 2020
New Revision: 357771
URL: https://svnweb.freebsd.org/changeset/base/357771
Log:
Add flag to struct task to mark the task as requiring network epoch.
When processing a taskqueue and a task has associated epoch, then
enter for duration of the task. If consecutive tasks belong to the
same epoch, batch them. Now we are talking about the network epoch
only.
Shrink the ta_priority size to 8-bits. No current consumers use
a priority that won't fit into 8 bits. Also complexity of
taskqueue_enqueue() is a square of maximum value of priority, so
we unlikely ever want to go over UCHAR_MAX here.
Reviewed by: hselasky
Differential Revision: https://reviews.freebsd.org/D23518
Modified:
head/sys/kern/subr_gtaskqueue.c
head/sys/kern/subr_taskqueue.c
head/sys/sys/_task.h
head/sys/sys/epoch.h
head/sys/sys/gtaskqueue.h
head/sys/sys/taskqueue.h
Modified: head/sys/kern/subr_gtaskqueue.c
==============================================================================
--- head/sys/kern/subr_gtaskqueue.c Tue Feb 11 18:19:56 2020 (r357770)
+++ head/sys/kern/subr_gtaskqueue.c Tue Feb 11 18:48:07 2020 (r357771)
@@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$");
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/proc.h>
+#include <sys/epoch.h>
#include <sys/sched.h>
#include <sys/smp.h>
#include <sys/gtaskqueue.h>
@@ -342,13 +343,16 @@ gtaskqueue_unblock(struct gtaskqueue *queue)
static void
gtaskqueue_run_locked(struct gtaskqueue *queue)
{
+ struct epoch_tracker et;
struct gtaskqueue_busy tb;
struct gtask *gtask;
+ bool in_net_epoch;
KASSERT(queue != NULL, ("tq is NULL"));
TQ_ASSERT_LOCKED(queue);
tb.tb_running = NULL;
LIST_INSERT_HEAD(&queue->tq_active, &tb, tb_link);
+ in_net_epoch = false;
while ((gtask = STAILQ_FIRST(&queue->tq_queue)) != NULL) {
STAILQ_REMOVE_HEAD(&queue->tq_queue, ta_link);
@@ -358,11 +362,20 @@ gtaskqueue_run_locked(struct gtaskqueue *queue)
TQ_UNLOCK(queue);
KASSERT(gtask->ta_func != NULL, ("task->ta_func is NULL"));
+ if (!in_net_epoch && TASK_IS_NET(gtask)) {
+ in_net_epoch = true;
+ NET_EPOCH_ENTER(et);
+ } else if (in_net_epoch && !TASK_IS_NET(gtask)) {
+ NET_EPOCH_EXIT(et);
+ in_net_epoch = false;
+ }
gtask->ta_func(gtask->ta_context);
TQ_LOCK(queue);
wakeup(gtask);
}
+ if (in_net_epoch)
+ NET_EPOCH_EXIT(et);
LIST_REMOVE(&tb, tb_link);
}
Modified: head/sys/kern/subr_taskqueue.c
==============================================================================
--- head/sys/kern/subr_taskqueue.c Tue Feb 11 18:19:56 2020 (r357770)
+++ head/sys/kern/subr_taskqueue.c Tue Feb 11 18:48:07 2020 (r357771)
@@ -42,6 +42,7 @@ __FBSDID("$FreeBSD$");
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/proc.h>
+#include <sys/epoch.h>
#include <sys/sched.h>
#include <sys/smp.h>
#include <sys/taskqueue.h>
@@ -371,7 +372,7 @@ taskqueue_drain_tq_queue(struct taskqueue *queue)
* anyway) so just insert it at tail while we have the
* queue lock.
*/
- TASK_INIT(&t_barrier, USHRT_MAX, taskqueue_task_nop_fn, &t_barrier);
+ TASK_INIT(&t_barrier, UCHAR_MAX, taskqueue_task_nop_fn, &t_barrier);
STAILQ_INSERT_TAIL(&queue->tq_queue, &t_barrier, ta_link);
queue->tq_hint = &t_barrier;
t_barrier.ta_pending = 1;
@@ -442,14 +443,17 @@ taskqueue_unblock(struct taskqueue *queue)
static void
taskqueue_run_locked(struct taskqueue *queue)
{
+ struct epoch_tracker et;
struct taskqueue_busy tb;
struct task *task;
+ bool in_net_epoch;
int pending;
KASSERT(queue != NULL, ("tq is NULL"));
TQ_ASSERT_LOCKED(queue);
tb.tb_running = NULL;
LIST_INSERT_HEAD(&queue->tq_active, &tb, tb_link);
+ in_net_epoch = false;
while ((task = STAILQ_FIRST(&queue->tq_queue)) != NULL) {
STAILQ_REMOVE_HEAD(&queue->tq_queue, ta_link);
@@ -462,11 +466,20 @@ taskqueue_run_locked(struct taskqueue *queue)
TQ_UNLOCK(queue);
KASSERT(task->ta_func != NULL, ("task->ta_func is NULL"));
+ if (!in_net_epoch && TASK_IS_NET(task)) {
+ in_net_epoch = true;
+ NET_EPOCH_ENTER(et);
+ } else if (in_net_epoch && !TASK_IS_NET(task)) {
+ NET_EPOCH_EXIT(et);
+ in_net_epoch = false;
+ }
task->ta_func(task->ta_context, pending);
TQ_LOCK(queue);
wakeup(task);
}
+ if (in_net_epoch)
+ NET_EPOCH_EXIT(et);
LIST_REMOVE(&tb, tb_link);
}
Modified: head/sys/sys/_task.h
==============================================================================
--- head/sys/sys/_task.h Tue Feb 11 18:19:56 2020 (r357770)
+++ head/sys/sys/_task.h Tue Feb 11 18:48:07 2020 (r357771)
@@ -48,10 +48,17 @@ typedef void task_fn_t(void *context, int pending);
struct task {
STAILQ_ENTRY(task) ta_link; /* (q) link for queue */
uint16_t ta_pending; /* (q) count times queued */
- u_short ta_priority; /* (c) Priority */
+ uint8_t ta_priority; /* (c) Priority */
+ uint8_t ta_flags; /* (c) Flags */
task_fn_t *ta_func; /* (c) task handler */
void *ta_context; /* (c) argument for handler */
};
+
+#define TASK_ENQUEUED 0x1
+#define TASK_NOENQUEUE 0x2
+#define TASK_NETWORK 0x4
+
+#define TASK_IS_NET(ta) ((ta)->ta_flags & TASK_NETWORK)
#ifdef _KERNEL
Modified: head/sys/sys/epoch.h
==============================================================================
--- head/sys/sys/epoch.h Tue Feb 11 18:19:56 2020 (r357770)
+++ head/sys/sys/epoch.h Tue Feb 11 18:48:07 2020 (r357771)
@@ -104,6 +104,9 @@ extern epoch_t net_epoch_preempt;
#define NET_EPOCH_WAIT() epoch_wait_preempt(net_epoch_preempt)
#define NET_EPOCH_CALL(f, c) epoch_call(net_epoch_preempt, (f), (c))
#define NET_EPOCH_ASSERT() MPASS(in_epoch(net_epoch_preempt))
+#define NET_TASK_INIT(t, p, f, c) TASK_INIT_FLAGS(t, p, f, c, TASK_NETWORK)
+#define NET_GROUPTASK_INIT(gtask, prio, func, ctx) \
+ GTASK_INIT(&(gtask)->gt_task, TASK_NETWORK, (prio), (func), (ctx))
#endif /* _KERNEL */
#endif /* _SYS_EPOCH_H_ */
Modified: head/sys/sys/gtaskqueue.h
==============================================================================
--- head/sys/sys/gtaskqueue.h Tue Feb 11 18:19:56 2020 (r357770)
+++ head/sys/sys/gtaskqueue.h Tue Feb 11 18:48:07 2020 (r357771)
@@ -84,10 +84,6 @@ void taskqgroup_config_gtask_init(void *ctx, struct gr
gtask_fn_t *fn, const char *name);
void taskqgroup_config_gtask_deinit(struct grouptask *gtask);
-#define TASK_ENQUEUED 0x1
-#define TASK_SKIP_WAKEUP 0x2
-#define TASK_NOENQUEUE 0x4
-
#define GTASK_INIT(gtask, flags, priority, func, context) do { \
(gtask)->ta_flags = flags; \
(gtask)->ta_priority = (priority); \
@@ -96,7 +92,7 @@ void taskqgroup_config_gtask_deinit(struct grouptask *
} while (0)
#define GROUPTASK_INIT(gtask, priority, func, context) \
- GTASK_INIT(&(gtask)->gt_task, TASK_SKIP_WAKEUP, priority, func, context)
+ GTASK_INIT(&(gtask)->gt_task, 0, priority, func, context)
#define GROUPTASK_ENQUEUE(gtask) \
grouptaskqueue_enqueue((gtask)->gt_taskqueue, &(gtask)->gt_task)
Modified: head/sys/sys/taskqueue.h
==============================================================================
--- head/sys/sys/taskqueue.h Tue Feb 11 18:19:56 2020 (r357770)
+++ head/sys/sys/taskqueue.h Tue Feb 11 18:48:07 2020 (r357771)
@@ -107,8 +107,7 @@ void taskqueue_set_callback(struct taskqueue *queue,
taskqueue_callback_fn callback, void *context);
#define TASK_INITIALIZER(priority, func, context) \
- { .ta_pending = 0, \
- .ta_priority = (priority), \
+ { .ta_priority = (priority), \
.ta_func = (func), \
.ta_context = (context) }
@@ -121,18 +120,25 @@ void taskqueue_thread_enqueue(void *context);
/*
* Initialise a task structure.
*/
-#define TASK_INIT(task, priority, func, context) do { \
- (task)->ta_pending = 0; \
- (task)->ta_priority = (priority); \
- (task)->ta_func = (func); \
- (task)->ta_context = (context); \
+#define TASK_INIT_FLAGS(task, priority, func, context, flags) do { \
+ MPASS((priority) >= 0 && (priority) <= 255); \
+ (task)->ta_pending = 0; \
+ (task)->ta_priority = (priority); \
+ (task)->ta_flags = (flags); \
+ (task)->ta_func = (func); \
+ (task)->ta_context = (context); \
} while (0)
+#define TASK_INIT(t, p, f, c) TASK_INIT_FLAGS(t, p, f, c, 0)
+
void _timeout_task_init(struct taskqueue *queue,
struct timeout_task *timeout_task, int priority, task_fn_t func,
void *context);
-#define TIMEOUT_TASK_INIT(queue, timeout_task, priority, func, context) \
- _timeout_task_init(queue, timeout_task, priority, func, context);
+#define TIMEOUT_TASK_INIT(queue, timeout_task, priority, func, context) do { \
+ _Static_assert((priority) >= 0 && (priority) <= 255, \
+ "struct task priority is 8 bit in size"); \
+ _timeout_task_init(queue, timeout_task, priority, func, context); \
+} while (0)
/*
* Declare a reference to a taskqueue.
More information about the svn-src-head
mailing list