git: 177772088060 - main - Reduce chance of RCU deadlock in the LinuxKPI by implementing the section feature of the concurrency kit, CK.

[Hans Petter Selasky]

The branch main has been updated by hselasky:

URL: https://cgit.FreeBSD.org/src/commit/?id=177772088060ab0f41bcdbdd81c4712e7f1c7621

commit 177772088060ab0f41bcdbdd81c4712e7f1c7621
Author:     Hans Petter Selasky <hselasky at FreeBSD.org>
AuthorDate: 2021-03-28 07:36:48 +0000
Commit:     Hans Petter Selasky <hselasky at FreeBSD.org>
CommitDate: 2021-03-29 08:55:14 +0000

    Reduce chance of RCU deadlock in the LinuxKPI by implementing the section
    feature of the concurrency kit, CK.
    
    Differential Revision:  https://reviews.freebsd.org/D29467
    Reviewed by:    kib@ and markj@
    MFC after:      1 week
    Sponsored by:   Mellanox Technologies // NVIDIA Networking
---
 sys/compat/linuxkpi/common/include/linux/sched.h |  1 +
 sys/compat/linuxkpi/common/src/linux_rcu.c       | 43 ++++++++++++++++++------
 2 files changed, 33 insertions(+), 11 deletions(-)

diff --git a/sys/compat/linuxkpi/common/include/linux/sched.h b/sys/compat/linuxkpi/common/include/linux/sched.h
index da38d89eb639..937e9f27870c 100644
--- a/sys/compat/linuxkpi/common/include/linux/sched.h
+++ b/sys/compat/linuxkpi/common/include/linux/sched.h
@@ -82,6 +82,7 @@ struct task_struct {
 	int bsd_interrupt_value;
 	struct work_struct *work;	/* current work struct, if set */
 	struct task_struct *group_leader;
+  	unsigned rcu_section[TS_RCU_TYPE_MAX];
 };
 
 #define	current	({ \
diff --git a/sys/compat/linuxkpi/common/src/linux_rcu.c b/sys/compat/linuxkpi/common/src/linux_rcu.c
index 86ec193aa4e4..404c5cec4ae4 100644
--- a/sys/compat/linuxkpi/common/src/linux_rcu.c
+++ b/sys/compat/linuxkpi/common/src/linux_rcu.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2016 Matthew Macy (mmacy at mattmacy.io)
- * Copyright (c) 2017-2020 Hans Petter Selasky (hselasky at freebsd.org)
+ * Copyright (c) 2017-2021 Hans Petter Selasky (hselasky at freebsd.org)
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -85,6 +85,15 @@ struct linux_epoch_record {
  */
 CTASSERT(sizeof(struct rcu_head) == sizeof(struct callback_head));
 
+/*
+ * Verify that "rcu_section[0]" has the same size as
+ * "ck_epoch_section_t". This has been done to avoid having to add
+ * special compile flags for including ck_epoch.h to all clients of
+ * the LinuxKPI.
+ */
+CTASSERT(sizeof(((struct task_struct *)0)->rcu_section[0] ==
+    sizeof(ck_epoch_section_t)));
+
 /*
  * Verify that "epoch_record" is at beginning of "struct
  * linux_epoch_record":
@@ -189,6 +198,14 @@ linux_rcu_read_lock(unsigned type)
 	if (RCU_SKIP())
 		return;
 
+	ts = current;
+
+	/* assert valid refcount */
+	MPASS(ts->rcu_recurse[type] != INT_MAX);
+
+	if (++(ts->rcu_recurse[type]) != 1)
+		return;
+
 	/*
 	 * Pin thread to current CPU so that the unlock code gets the
 	 * same per-CPU epoch record:
@@ -196,17 +213,15 @@ linux_rcu_read_lock(unsigned type)
 	sched_pin();
 
 	record = &DPCPU_GET(linux_epoch_record[type]);
-	ts = current;
 
 	/*
 	 * Use a critical section to prevent recursion inside
 	 * ck_epoch_begin(). Else this function supports recursion.
 	 */
 	critical_enter();
-	ck_epoch_begin(&record->epoch_record, NULL);
-	ts->rcu_recurse[type]++;
-	if (ts->rcu_recurse[type] == 1)
-		TAILQ_INSERT_TAIL(&record->ts_head, ts, rcu_entry[type]);
+	ck_epoch_begin(&record->epoch_record,
+	    (ck_epoch_section_t *)&ts->rcu_section[type]);
+	TAILQ_INSERT_TAIL(&record->ts_head, ts, rcu_entry[type]);
 	critical_exit();
 }
 
@@ -221,18 +236,24 @@ linux_rcu_read_unlock(unsigned type)
 	if (RCU_SKIP())
 		return;
 
-	record = &DPCPU_GET(linux_epoch_record[type]);
 	ts = current;
 
+	/* assert valid refcount */
+	MPASS(ts->rcu_recurse[type] > 0);
+	
+	if (--(ts->rcu_recurse[type]) != 0)
+		return;
+
+	record = &DPCPU_GET(linux_epoch_record[type]);
+
 	/*
 	 * Use a critical section to prevent recursion inside
 	 * ck_epoch_end(). Else this function supports recursion.
 	 */
 	critical_enter();
-	ck_epoch_end(&record->epoch_record, NULL);
-	ts->rcu_recurse[type]--;
-	if (ts->rcu_recurse[type] == 0)
-		TAILQ_REMOVE(&record->ts_head, ts, rcu_entry[type]);
+	ck_epoch_end(&record->epoch_record,
+	    (ck_epoch_section_t *)&ts->rcu_section[type]);
+	TAILQ_REMOVE(&record->ts_head, ts, rcu_entry[type]);
 	critical_exit();
 
 	sched_unpin();