git: 2d5a6bc2a4c5 - main - mlx5: add per-flow-group hash table for FTE lookup

From: Konstantin Belousov <kib_at_FreeBSD.org>
Date: Wed, 11 Nov 2026 01:45:23 UTC
The branch main has been updated by kib:

URL: https://cgit.FreeBSD.org/src/commit/?id=2d5a6bc2a4c5eb061f8d63b7aaff7dc04e96f1ea

commit 2d5a6bc2a4c5eb061f8d63b7aaff7dc04e96f1ea
Author:     Slava Shwartsman <slavash@nvidia.com>
AuthorDate: 2026-10-26 19:27:56 +0000
Commit:     Konstantin Belousov <kib@FreeBSD.org>
CommitDate: 2026-11-11 01:40:53 +0000

    mlx5: add per-flow-group hash table for FTE lookup
    
    lookup_fte_locked() scanned every FTE in a flow group with xa_next()
    and memcmp() on each one.  A new RX TLS session always misses, so the
    whole group was walked under the group lock for every insertion.  With
    tens of thousands of RX-offloaded sessions in the same IPv6 TCP group
    this costs milliseconds per call and grows linearly with the number of
    FTEs.
    
    Keep a hash table of the FTEs in each flow group, keyed by a jhash of
    the match value.  The table is sized from the group size, capped at
    16K buckets.  It is updated in insert_fte() and del_sw_fte(), both of
    which hold the group write lock, and read in lookup_fte_locked() under
    the group read or write lock.  The xarray is kept for index
    allocation.  No lockless reads are introduced.
    
    Reported by:    NetApp
    Reviewed by:    kib
    Sponsored by:   NVidia networking
    MFC after:      1 week
---
 sys/dev/mlx5/mlx5_core/fs_core.h      |  7 +++++++
 sys/dev/mlx5/mlx5_core/mlx5_fs_core.c | 31 +++++++++++++++++++++++++++----
 2 files changed, 34 insertions(+), 4 deletions(-)

diff --git a/sys/dev/mlx5/mlx5_core/fs_core.h b/sys/dev/mlx5/mlx5_core/fs_core.h
index 1d7339e6b7d5..2eafc95eb0c6 100644
--- a/sys/dev/mlx5/mlx5_core/fs_core.h
+++ b/sys/dev/mlx5/mlx5_core/fs_core.h
@@ -216,6 +216,8 @@ struct fs_fte {
 	u32				dests_size;
 	u32				fwd_dests;
 	u32				index;
+	u32				hash;
+	struct hlist_node		hnode;
 	struct mlx5_flow_context	flow_context;
 	struct mlx5_flow_act		action;
 	enum fs_fte_status		status;
@@ -244,6 +246,8 @@ struct mlx5_flow_group_mask {
 	u32	match_criteria[MLX5_ST_SZ_DW_MATCH_PARAM];
 };
 
+#define	FS_FTE_HASH_MAX_BUCKETS	(1U << 14)
+
 /* Type of children is fs_fte */
 struct mlx5_flow_group {
 	struct fs_node			node;
@@ -253,6 +257,9 @@ struct mlx5_flow_group {
 	struct ida			fte_allocator;
 	u32				id;
 	struct xarray			ftes_xa;
+	/* match value -> FTE hash, protected by the group node lock */
+	struct hlist_head		*fte_hash;
+	u32				fte_hash_mask;
 };
 
 struct mlx5_flow_root_namespace {
diff --git a/sys/dev/mlx5/mlx5_core/mlx5_fs_core.c b/sys/dev/mlx5/mlx5_core/mlx5_fs_core.c
index 8d93d4740462..6678d4e64b91 100644
--- a/sys/dev/mlx5/mlx5_core/mlx5_fs_core.c
+++ b/sys/dev/mlx5/mlx5_core/mlx5_fs_core.c
@@ -36,6 +36,8 @@
 #include <dev/mlx5/mlx5_core/fs_core.h>
 #include <linux/string.h>
 #include <linux/compiler.h>
+#include <linux/jhash.h>
+#include <linux/list.h>
 
 #include "eswitch.h"
 #include "fs_ft_pool.h"
@@ -658,6 +660,7 @@ static void del_sw_fte(struct fs_node *node)
 
 	deleted_fte = xa_erase(&fg->ftes_xa, fte->index);
 	WARN_ON(deleted_fte != fte);
+	hlist_del_init(&fte->hnode);
 	ida_free(&fg->fte_allocator, fte->index - fg->start_index);
 	kmem_cache_free(steering->ftes_cache, fte);
 }
@@ -691,6 +694,7 @@ static void del_sw_flow_group(struct fs_node *node)
 
 	xa_destroy(&fg->ftes_xa);
 	ida_destroy(&fg->fte_allocator);
+	kvfree(fg->fte_hash);
 	if (ft->autogroup.active &&
 	    fg->max_ftes == ft->autogroup.group_size &&
 	    fg->start_index < ft->autogroup.max_fte)
@@ -714,6 +718,7 @@ static int insert_fte(struct mlx5_flow_group *fg, struct fs_fte *fte)
 	if (ret)
 		goto err_ida_remove;
 
+	hlist_add_head(&fte->hnode, &fg->fte_hash[fte->hash & fg->fte_hash_mask]);
 	tree_add_node(&fte->node, &fg->node);
 	list_add_tail(&fte->node.list, &fg->node.children);
 	return 0;
@@ -735,6 +740,8 @@ static struct fs_fte *alloc_fte(struct mlx5_flow_table *ft,
 		return ERR_PTR(-ENOMEM);
 
 	memcpy(fte->val, &spec->match_value, sizeof(fte->val));
+	fte->hash = jhash(fte->val, sizeof(fte->val), 0);
+	INIT_HLIST_NODE(&fte->hnode);
 	fte->node.type =  FS_TYPE_FLOW_ENTRY;
 	fte->action = *flow_act;
 	fte->flow_context = spec->flow_context;
@@ -748,6 +755,7 @@ static void dealloc_flow_group(struct mlx5_flow_steering *steering,
 			       struct mlx5_flow_group *fg)
 {
 	xa_destroy(&fg->ftes_xa);
+	kvfree(fg->fte_hash);
 	kmem_cache_free(steering->fgs_cache, fg);
 }
 
@@ -758,11 +766,24 @@ static struct mlx5_flow_group *alloc_flow_group(struct mlx5_flow_steering *steer
 						int end_index)
 {
 	struct mlx5_flow_group *fg;
+	u32 nbuckets;
 
 	fg = kmem_cache_zalloc(steering->fgs_cache, GFP_KERNEL);
 	if (!fg)
 		return ERR_PTR(-ENOMEM);
 
+	/* Aim for short chains, but do not waste memory on small groups. */
+	nbuckets = 1;
+	while (nbuckets < (u32)(end_index - start_index + 1) / 2 &&
+	    nbuckets < FS_FTE_HASH_MAX_BUCKETS)
+		nbuckets <<= 1;
+	fg->fte_hash = kvcalloc(nbuckets, sizeof(*fg->fte_hash), GFP_KERNEL);
+	if (!fg->fte_hash) {
+		kmem_cache_free(steering->fgs_cache, fg);
+		return ERR_PTR(-ENOMEM);
+	}
+	fg->fte_hash_mask = nbuckets - 1;
+
 	xa_init(&fg->ftes_xa);
 
 	ida_init(&fg->fte_allocator);
@@ -1807,17 +1828,18 @@ static u64 matched_fgs_get_version(struct list_head *match_head)
 static struct fs_fte *
 lookup_fte_locked(struct mlx5_flow_group *g,
 		  const u32 *match_value,
+		  u32 hash,
 		  bool take_write)
 {
 	struct fs_fte *fte_tmp;
-	unsigned long index;
 
 	if (take_write)
 		nested_down_write_ref_node(&g->node, FS_LOCK_PARENT);
 	else
 		nested_down_read_ref_node(&g->node, FS_LOCK_PARENT);
-	xa_for_each_rcu(&g->ftes_xa, index, fte_tmp) {
-		if (!memcmp(match_value, fte_tmp->val, sizeof_field(struct fs_fte, val)))
+	hlist_for_each_entry(fte_tmp, &g->fte_hash[hash & g->fte_hash_mask], hnode) {
+		if (fte_tmp->hash == hash &&
+		    !memcmp(match_value, fte_tmp->val, sizeof_field(struct fs_fte, val)))
 			break;
 	}
 	if (!fte_tmp || !tree_get_node(&fte_tmp->node)) {
@@ -1872,7 +1894,8 @@ search_again_locked:
 		struct fs_fte *fte_tmp;
 
 		g = iter->g;
-		fte_tmp = lookup_fte_locked(g, spec->match_value, take_write);
+		fte_tmp = lookup_fte_locked(g, spec->match_value, fte->hash,
+		    take_write);
 		if (!fte_tmp)
 			continue;
 		rule = add_rule_fg(g, spec, flow_act, dest, dest_num, fte_tmp);