git: db6b56441e3a - main - ktls: Avoid wakeups and locking for synchronous callbacks

[Mark Johnston]

The branch main has been updated by markj:

URL: https://cgit.FreeBSD.org/src/commit/?id=db6b56441e3a142e7fe58a5a85d9071e772a943e

commit db6b56441e3a142e7fe58a5a85d9071e772a943e
Author:     Mark Johnston <markj at FreeBSD.org>
AuthorDate: 2021-02-08 14:19:27 +0000
Commit:     Mark Johnston <markj at FreeBSD.org>
CommitDate: 2021-02-08 14:19:27 +0000

    ktls: Avoid wakeups and locking for synchronous callbacks
    
    When performing encryption in software, the KTLS crypto callback always
    locks the session to deliver a wakeup.  But, if we're handling the
    operation synchronously this is wasted effort and can result in
    sleepqueue lock contention on large systems.
    
    Use CRYPTO_SESS_SYNC() to determine whether the operation will be
    completed asynchronously or not, and select a callback appropriately.
    Avoid locking the session to check for completion if the session handles
    requests synchronously.
    
    Reviewed by:    jhb
    Sponsored by:   Ampere Computing
    Submitted by:   Klara, Inc.
    MFC after:      2 weeks
    Differential Revision:  https://reviews.freebsd.org/D28195
---
 sys/opencrypto/ktls_ocf.c | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/sys/opencrypto/ktls_ocf.c b/sys/opencrypto/ktls_ocf.c
index 7f9ece99ccb1..28c55257b047 100644
--- a/sys/opencrypto/ktls_ocf.c
+++ b/sys/opencrypto/ktls_ocf.c
@@ -108,7 +108,13 @@ SYSCTL_COUNTER_U64(_kern_ipc_tls_stats_ocf, OID_AUTO, retries, CTLFLAG_RD,
     "Number of OCF encryption operation retries");
 
 static int
-ktls_ocf_callback(struct cryptop *crp)
+ktls_ocf_callback_sync(struct cryptop *crp __unused)
+{
+	return (0);
+}
+
+static int
+ktls_ocf_callback_async(struct cryptop *crp)
 {
 	struct ocf_operation *oo;
 
@@ -125,21 +131,26 @@ ktls_ocf_dispatch(struct ocf_session *os, struct cryptop *crp)
 {
 	struct ocf_operation oo;
 	int error;
+	bool async;
 
 	oo.os = os;
 	oo.done = false;
 
 	crp->crp_opaque = &oo;
-	crp->crp_callback = ktls_ocf_callback;
 	for (;;) {
+		async = !CRYPTO_SESS_SYNC(crp->crp_session);
+		crp->crp_callback = async ? ktls_ocf_callback_async :
+		    ktls_ocf_callback_sync;
+
 		error = crypto_dispatch(crp);
 		if (error)
 			break;
-
-		mtx_lock(&os->lock);
-		while (!oo.done)
-			mtx_sleep(&oo, &os->lock, 0, "ocfktls", 0);
-		mtx_unlock(&os->lock);
+		if (async) {
+			mtx_lock(&os->lock);
+			while (!oo.done)
+				mtx_sleep(&oo, &os->lock, 0, "ocfktls", 0);
+			mtx_unlock(&os->lock);
+		}
 
 		if (crp->crp_etype != EAGAIN) {
 			error = crp->crp_etype;