Re: git: 0d469d23715d - main - net: attach IPv4 and IPv6 stacks to an interface with EVENTHANDLER(9)
Date: Mon, 12 Jan 2026 16:15:30 UTC
In addition to the other reports there is also the following:
panic: mtx_lock() of spin mutex (null) @
/var/jenkins/workspace/pfSense-Plus-snapshots-master-main/sources/FreeBSD-src-plus-devel-main/sys/netinet/igmp.c:627
cpuid = 1
time = 1
KDB: enter: panic
[ thread pid 0 tid 100000 ]
Stopped at kdb_enter+0x33: movq $0,0x1b23e72(%rip)
db> bt
Tracing pid 0 tid 100000 td 0xffffffff828665e0
kdb_enter() at kdb_enter+0x33/frame 0xffffffff83963cb0
panic() at panic+0x43/frame 0xffffffff83963d10
__mtx_lock_flags() at __mtx_lock_flags+0x12f/frame 0xffffffff83963d60
igmp_domifattach() at igmp_domifattach+0x24/frame 0xffffffff83963d80
in_ifattach() at in_ifattach+0xb2/frame 0xffffffff83963db0
if_attach_internal() at if_attach_internal+0x374/frame 0xffffffff83963e00
enc_clone_create() at enc_clone_create+0x7e/frame 0xffffffff83963e30
if_clone_createif_nl() at if_clone_createif_nl+0x166/frame 0xffffffff83963ea0
ifc_create_ifp() at ifc_create_ifp+0x102/frame 0xffffffff83963f10
vnet_enc_init() at vnet_enc_init+0x97/frame 0xffffffff83963f70
vnet_register_sysinit() at vnet_register_sysinit+0x125/frame 0xffffffff83963fa0
mi_startup() at mi_startup+0x1f4/frame 0xffffffff83963ff0
db>
the igmp mutex is not yet initialized
On Thu, Dec 18, 2025 at 8:16 PM Gleb Smirnoff <glebius@freebsd.org> wrote:
>
> The branch main has been updated by glebius:
>
> URL: https://cgit.FreeBSD.org/src/commit/?id=0d469d23715d690b863787ebfa51529e1f6a9092
>
> commit 0d469d23715d690b863787ebfa51529e1f6a9092
> Author: Gleb Smirnoff <glebius@FreeBSD.org>
> AuthorDate: 2025-12-18 16:47:39 +0000
> Commit: Gleb Smirnoff <glebius@FreeBSD.org>
> CommitDate: 2025-12-18 19:15:53 +0000
>
> net: attach IPv4 and IPv6 stacks to an interface with EVENTHANDLER(9)
>
> This change retires two historic relics: the if_afdata[] array and the
> dom_ifattach/dom_ifdetach methods.
>
> The if_afdata[] array is a relic of the era, when there was expectation
> that many transport protocols will coexist with IP, e.g. IPX or NetAtalk.
> The array hasn't had any members except AF_INET and AF_INET6 for over a
> decade already. This change removes the array and just leaves two pointer
> fields: if_inet and if_inet6.
>
> The dom_ifattach/dom_ifdetach predates the EVENTHANDLER(9) framework and
> was a good enough method to initialize protocol contexts back then. Today
> there is no good reason to treat IPv4 and IPv6 stacks differently to other
> protocols/features that attach and detach from an interface.
>
> The locking of if_afdata[] is a relic of SMPng times, when the system
> startup and the interface attach was even more convoluted than before this
> change, and we also had unloadable protocols that used a field in
> if_afdata[]. Note that IPv4 and IPv6 are not unloadable.
>
> Note that this change removes NET_EPOCH_WAIT() from the interface detach
> sequence. This may surface several new races associated with interface
> removal. I failed to hit any with consecutive test suite runs, though.
> The expected general race scenario is that while struct ifnet is freed
> with proper epoch_call(9) itself, some structures hanging off ifnet are
> freed with direct free(9). The proper fix is either make if_foo point at
> some static "dead" structure providing SMP visibility of this store, or
> free those structure with epoch_call(9). All of these cases are planned
> to be found and resolved during 16.0-CURRENT lifetime.
>
> Reviewed by: zlei, gallatin, melifaro
> Differential Revision: https://reviews.freebsd.org/D54089
> ---
> sys/kern/uipc_debug.c | 4 --
> sys/net/if.c | 122 ++++----------------------------------------
> sys/net/if_loop.c | 2 +-
> sys/net/if_private.h | 19 +------
> sys/net/if_var.h | 3 +-
> sys/net/vnet.h | 3 ++
> sys/netinet/icmp6.h | 2 +-
> sys/netinet/if_ether.c | 4 +-
> sys/netinet/igmp.c | 10 ++--
> sys/netinet/in.c | 98 +++++++++++++++++++----------------
> sys/netinet/in.h | 1 -
> sys/netinet/in_mcast.c | 2 +-
> sys/netinet/in_proto.c | 2 -
> sys/netinet/in_var.h | 7 ++-
> sys/netinet/ip_input.c | 9 ++++
> sys/netinet6/in6.c | 49 +++++-------------
> sys/netinet6/in6_ifattach.c | 39 +++++++++++++-
> sys/netinet6/in6_mcast.c | 2 +-
> sys/netinet6/in6_proto.c | 2 -
> sys/netinet6/in6_var.h | 11 ++--
> sys/netinet6/ip6_input.c | 7 +++
> sys/netinet6/ip6_output.c | 2 +-
> sys/netinet6/mld6_var.h | 3 +-
> sys/netinet6/nd6.c | 11 ++--
> sys/netinet6/nd6.h | 3 +-
> sys/netinet6/scope6.c | 10 ++--
> sys/sys/domain.h | 2 -
> sys/sys/kernel.h | 2 +-
> 28 files changed, 173 insertions(+), 258 deletions(-)
>
> diff --git a/sys/kern/uipc_debug.c b/sys/kern/uipc_debug.c
> index df36ec75cb5f..f6b7b1899d02 100644
> --- a/sys/kern/uipc_debug.c
> +++ b/sys/kern/uipc_debug.c
> @@ -242,10 +242,6 @@ db_print_domain(struct domain *d, const char *domain_name, int indent)
>
> db_print_indent(indent);
> db_printf("dom_rtattach: %p ", d->dom_rtattach);
> -
> - db_print_indent(indent);
> - db_printf("dom_ifattach: %p ", d->dom_ifattach);
> - db_printf("dom_ifdetach: %p\n", d->dom_ifdetach);
> }
>
> static void
> diff --git a/sys/net/if.c b/sys/net/if.c
> index b603c72bd106..3c16246e8b62 100644
> --- a/sys/net/if.c
> +++ b/sys/net/if.c
> @@ -102,7 +102,6 @@
> #endif /* INET */
> #ifdef INET6
> #include <netinet6/in6_var.h>
> -#include <netinet6/in6_ifattach.h>
> #endif /* INET6 */
> #endif /* INET || INET6 */
>
> @@ -270,8 +269,6 @@ struct mbuf *(*tbr_dequeue_ptr)(struct ifaltq *, int) = NULL;
> * static functions should be prototyped. Currently they are sorted by
> * declaration order.
> */
> -static void if_attachdomain(void *);
> -static void if_attachdomain1(struct ifnet *);
> static int ifconf(u_long, caddr_t);
> static void if_input_default(struct ifnet *, struct mbuf *);
> static int if_requestencap_default(struct ifnet *, struct if_encap_req *);
> @@ -348,11 +345,6 @@ struct sx ifnet_detach_sxlock;
> SX_SYSINIT_FLAGS(ifnet_detach, &ifnet_detach_sxlock, "ifnet_detach_sx",
> SX_RECURSE);
>
> -#ifdef VIMAGE
> -#define VNET_IS_SHUTTING_DOWN(_vnet) \
> - ((_vnet)->vnet_shutdown && (_vnet)->vnet_state < SI_SUB_VNET_DONE)
> -#endif
> -
> static if_com_alloc_t *if_com_alloc[256];
> static if_com_free_t *if_com_free[256];
>
> @@ -553,8 +545,6 @@ if_alloc_domain(u_char type, int numa_domain)
> IF_ADDR_LOCK_INIT(ifp);
> TASK_INIT(&ifp->if_linktask, 0, do_link_state_change, ifp);
> TASK_INIT(&ifp->if_addmultitask, 0, if_siocaddmulti, ifp);
> - ifp->if_afdata_initialized = 0;
> - IF_AFDATA_LOCK_INIT(ifp);
> CK_STAILQ_INIT(&ifp->if_addrhead);
> CK_STAILQ_INIT(&ifp->if_multiaddrs);
> CK_STAILQ_INIT(&ifp->if_groups);
> @@ -641,7 +631,6 @@ if_free_deferred(epoch_context_t ctx)
> #ifdef MAC
> mac_ifnet_destroy(ifp);
> #endif /* MAC */
> - IF_AFDATA_DESTROY(ifp);
> IF_ADDR_LOCK_DESTROY(ifp);
> ifq_delete(&ifp->if_snd);
>
> @@ -930,8 +919,6 @@ if_attach_internal(struct ifnet *ifp, bool vmove)
> #endif
> }
>
> - if (domain_init_status >= 2)
> - if_attachdomain1(ifp);
> EVENTHANDLER_INVOKE(ifnet_arrival_event, ifp);
> if_link_ifnet(ifp);
> EVENTHANDLER_INVOKE(ifnet_attached_event, ifp);
> @@ -947,45 +934,6 @@ if_epochalloc(void *dummy __unused)
> }
> SYSINIT(ifepochalloc, SI_SUB_EPOCH, SI_ORDER_ANY, if_epochalloc, NULL);
>
> -static void
> -if_attachdomain(void *dummy)
> -{
> - struct ifnet *ifp;
> -
> - CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link)
> - if_attachdomain1(ifp);
> -}
> -SYSINIT(domainifattach, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_SECOND,
> - if_attachdomain, NULL);
> -
> -static void
> -if_attachdomain1(struct ifnet *ifp)
> -{
> - struct domain *dp;
> -
> - /*
> - * Since dp->dom_ifattach calls malloc() with M_WAITOK, we
> - * cannot lock ifp->if_afdata initialization, entirely.
> - */
> - IF_AFDATA_LOCK(ifp);
> - if (ifp->if_afdata_initialized >= domain_init_status) {
> - IF_AFDATA_UNLOCK(ifp);
> - log(LOG_WARNING, "%s called more than once on %s\n",
> - __func__, ifp->if_xname);
> - return;
> - }
> - ifp->if_afdata_initialized = domain_init_status;
> - IF_AFDATA_UNLOCK(ifp);
> -
> - /* address family dependent data region */
> - bzero(ifp->if_afdata, sizeof(ifp->if_afdata));
> - SLIST_FOREACH(dp, &domains, dom_next) {
> - if (dp->dom_ifattach)
> - ifp->if_afdata[dp->dom_family] =
> - (*dp->dom_ifattach)(ifp);
> - }
> -}
> -
> /*
> * Remove any unicast or broadcast network addresses from an interface.
> */
> @@ -1098,9 +1046,6 @@ static void
> if_detach_internal(struct ifnet *ifp, bool vmove)
> {
> struct ifaddr *ifa;
> - int i;
> - struct domain *dp;
> - void *if_afdata[AF_MAX];
> #ifdef VIMAGE
> bool shutdown;
>
> @@ -1151,7 +1096,7 @@ if_detach_internal(struct ifnet *ifp, bool vmove)
> * if_detach() calls us in void context and does not care
> * about an early abort notification, so life is splendid :)
> */
> - goto finish_vnet_shutdown;
> + return;
> }
> #endif
>
> @@ -1172,20 +1117,6 @@ if_detach_internal(struct ifnet *ifp, bool vmove)
> #endif
>
> if_purgeaddrs(ifp);
> -
> -#ifdef INET
> - in_ifdetach(ifp);
> -#endif
> -
> -#ifdef INET6
> - /*
> - * Remove all IPv6 kernel structs related to ifp. This should be done
> - * before removing routing entries below, since IPv6 interface direct
> - * routes are expected to be removed by the IPv6-specific kernel API.
> - * Otherwise, the kernel will detect some inconsistency and bark it.
> - */
> - in6_ifdetach(ifp);
> -#endif
> if_purgemaddrs(ifp);
>
> EVENTHANDLER_INVOKE(ifnet_departure_event, ifp);
> @@ -1212,43 +1143,6 @@ if_detach_internal(struct ifnet *ifp, bool vmove)
> }
>
> rt_flushifroutes(ifp);
> -
> -#ifdef VIMAGE
> -finish_vnet_shutdown:
> -#endif
> - /*
> - * We cannot hold the lock over dom_ifdetach calls as they might
> - * sleep, for example trying to drain a callout, thus open up the
> - * theoretical race with re-attaching.
> - */
> - IF_AFDATA_LOCK(ifp);
> - i = ifp->if_afdata_initialized;
> - ifp->if_afdata_initialized = 0;
> - if (i != 0) {
> - /*
> - * Defer the dom_ifdetach call.
> - */
> - _Static_assert(sizeof(if_afdata) == sizeof(ifp->if_afdata),
> - "array size mismatch");
> - memcpy(if_afdata, ifp->if_afdata, sizeof(if_afdata));
> - memset(ifp->if_afdata, 0, sizeof(ifp->if_afdata));
> - }
> - IF_AFDATA_UNLOCK(ifp);
> - if (i == 0)
> - return;
> - /*
> - * XXXZL: This net epoch wait is not necessary if we have done right.
> - * But if we do not, at least we can make a guarantee that threads those
> - * enter net epoch will see NULL address family dependent data,
> - * e.g. if_afdata[AF_INET6]. A clear NULL pointer derefence is much
> - * better than writing to freed memory.
> - */
> - NET_EPOCH_WAIT();
> - SLIST_FOREACH(dp, &domains, dom_next) {
> - if (dp->dom_ifdetach != NULL &&
> - if_afdata[dp->dom_family] != NULL)
> - (*dp->dom_ifdetach)(ifp, if_afdata[dp->dom_family]);
> - }
> }
>
> #ifdef VIMAGE
> @@ -5121,10 +5015,16 @@ if_getvnet(if_t ifp)
> return (ifp->if_vnet);
> }
>
> -void *
> -if_getafdata(if_t ifp, int af)
> +struct in_ifinfo *
> +if_getinet(if_t ifp)
> +{
> + return (ifp->if_inet);
> +}
> +
> +struct in6_ifextra *
> +if_getinet6(if_t ifp)
> {
> - return (ifp->if_afdata[af]);
> + return (ifp->if_inet6);
> }
>
> u_int
> @@ -5189,8 +5089,6 @@ if_show_ifnet(struct ifnet *ifp)
> IF_DB_PRINTF("%d", if_amcount);
> IF_DB_PRINTF("%p", if_addr);
> IF_DB_PRINTF("%p", if_broadcastaddr);
> - IF_DB_PRINTF("%p", if_afdata);
> - IF_DB_PRINTF("%d", if_afdata_initialized);
> IF_DB_PRINTF("%u", if_fib);
> IF_DB_PRINTF("%p", if_vnet);
> IF_DB_PRINTF("%p", if_home_vnet);
> diff --git a/sys/net/if_loop.c b/sys/net/if_loop.c
> index ec0ff0e77aa6..37309260a0d3 100644
> --- a/sys/net/if_loop.c
> +++ b/sys/net/if_loop.c
> @@ -151,7 +151,7 @@ vnet_loif_init(const void *unused __unused)
> struct ifc_data ifd = { .unit = 0 };
> ifc_create_ifp(loname, &ifd, &V_loif);
> }
> -VNET_SYSINIT(vnet_loif_init, SI_SUB_PSEUDO, SI_ORDER_ANY,
> +VNET_SYSINIT(vnet_loif_init, SI_SUB_PROTO_IF, SI_ORDER_ANY,
> vnet_loif_init, NULL);
>
> #ifdef VIMAGE
> diff --git a/sys/net/if_private.h b/sys/net/if_private.h
> index b8cd0722eba6..c4ade7308df4 100644
> --- a/sys/net/if_private.h
> +++ b/sys/net/if_private.h
> @@ -101,9 +101,8 @@ struct ifnet {
> struct ifaddr *if_addr; /* pointer to link-level address */
> void *if_hw_addr; /* hardware link-level address */
> const u_int8_t *if_broadcastaddr; /* linklevel broadcast bytestring */
> - struct mtx if_afdata_lock;
> - void *if_afdata[AF_MAX];
> - int if_afdata_initialized;
> + struct in_ifinfo *if_inet;
> + struct in6_ifextra *if_inet6;
>
> /* Additional features hung off the interface. */
> u_int if_fib; /* interface FIB */
> @@ -193,20 +192,6 @@ struct ifnet {
> int if_ispare[4]; /* general use */
> };
>
> -#define IF_AFDATA_LOCK_INIT(ifp) \
> - mtx_init(&(ifp)->if_afdata_lock, "if_afdata", NULL, MTX_DEF)
> -
> -#define IF_AFDATA_WLOCK(ifp) mtx_lock(&(ifp)->if_afdata_lock)
> -#define IF_AFDATA_WUNLOCK(ifp) mtx_unlock(&(ifp)->if_afdata_lock)
> -#define IF_AFDATA_LOCK(ifp) IF_AFDATA_WLOCK(ifp)
> -#define IF_AFDATA_UNLOCK(ifp) IF_AFDATA_WUNLOCK(ifp)
> -#define IF_AFDATA_TRYLOCK(ifp) mtx_trylock(&(ifp)->if_afdata_lock)
> -#define IF_AFDATA_DESTROY(ifp) mtx_destroy(&(ifp)->if_afdata_lock)
> -
> -#define IF_AFDATA_LOCK_ASSERT(ifp) MPASS(in_epoch(net_epoch_preempt) || mtx_owned(&(ifp)->if_afdata_lock))
> -#define IF_AFDATA_WLOCK_ASSERT(ifp) mtx_assert(&(ifp)->if_afdata_lock, MA_OWNED)
> -#define IF_AFDATA_UNLOCK_ASSERT(ifp) mtx_assert(&(ifp)->if_afdata_lock, MA_NOTOWNED)
> -
> #define IF_LLADDR(ifp) \
> LLADDR((struct sockaddr_dl *)((ifp)->if_addr->ifa_addr))
>
> diff --git a/sys/net/if_var.h b/sys/net/if_var.h
> index cbc4969918ba..4fb51aca0cd7 100644
> --- a/sys/net/if_var.h
> +++ b/sys/net/if_var.h
> @@ -673,7 +673,8 @@ void *if_getl2com(if_t ifp);
> struct ifvlantrunk *if_getvlantrunk(if_t ifp);
> bool if_altq_is_enabled(if_t ifp);
>
> -void *if_getafdata(if_t ifp, int);
> +struct in_ifinfo *if_getinet(if_t ifp);
> +struct in6_ifextra *if_getinet6(if_t ifp);
>
> int if_snd_tag_alloc(if_t ifp, union if_snd_tag_alloc_params *params,
> struct m_snd_tag **mstp);
> diff --git a/sys/net/vnet.h b/sys/net/vnet.h
> index 670e99a455ae..f8ffcb4e2b76 100644
> --- a/sys/net/vnet.h
> +++ b/sys/net/vnet.h
> @@ -395,6 +395,9 @@ do { \
> } \
> } while(0)
>
> +#define VNET_IS_SHUTTING_DOWN(_vnet) \
> + ((_vnet)->vnet_shutdown && (_vnet)->vnet_state < SI_SUB_VNET_DONE)
> +
> #else /* !VIMAGE */
>
> /*
> diff --git a/sys/netinet/icmp6.h b/sys/netinet/icmp6.h
> index 2ca5b3433e47..082ef5d29ce9 100644
> --- a/sys/netinet/icmp6.h
> +++ b/sys/netinet/icmp6.h
> @@ -718,7 +718,7 @@ int icmp6_ratelimit(const struct in6_addr *, const int, const int);
> do { \
> if (ifp) \
> counter_u64_add(((struct in6_ifextra *) \
> - ((ifp)->if_afdata[AF_INET6]))->icmp6_ifstat[\
> + ((ifp)->if_inet6))->icmp6_ifstat[ \
> offsetof(struct icmp6_ifstat, tag) / sizeof(uint64_t)], 1);\
> } while (/*CONSTCOND*/ 0)
>
> diff --git a/sys/netinet/if_ether.c b/sys/netinet/if_ether.c
> index 7b223f1f2f11..8d65b88913b8 100644
> --- a/sys/netinet/if_ether.c
> +++ b/sys/netinet/if_ether.c
> @@ -1485,8 +1485,8 @@ static void
> arp_iflladdr(void *arg __unused, struct ifnet *ifp)
> {
> /* if_bridge can update its lladdr during if_vmove(), after we've done
> - * if_detach_internal()/dom_ifdetach(). */
> - if (ifp->if_afdata[AF_INET] == NULL)
> + * with in_ifdetach(). XXXGL: needs to be fixed. */
> + if (ifp->if_inet == NULL)
> return;
>
> lltable_update_ifaddr(LLTABLE(ifp));
> diff --git a/sys/netinet/igmp.c b/sys/netinet/igmp.c
> index ad22efb16495..de9d38c7fb9d 100644
> --- a/sys/netinet/igmp.c
> +++ b/sys/netinet/igmp.c
> @@ -690,7 +690,7 @@ igmp_ifdetach(struct ifnet *ifp)
> SLIST_INIT(&inm_free_tmp);
> IGMP_LOCK();
>
> - igi = ((struct in_ifinfo *)ifp->if_afdata[AF_INET])->ii_igmp;
> + igi = ((struct in_ifinfo *)ifp->if_inet)->ii_igmp;
> if (igi->igi_version == IGMP_VERSION_3) {
> IF_ADDR_WLOCK(ifp);
> NET_EPOCH_ENTER(et);
> @@ -781,7 +781,7 @@ igmp_input_v1_query(struct ifnet *ifp, const struct ip *ip,
> IN_MULTI_LIST_LOCK();
> IGMP_LOCK();
>
> - igi = ((struct in_ifinfo *)ifp->if_afdata[AF_INET])->ii_igmp;
> + igi = ((struct in_ifinfo *)ifp->if_inet)->ii_igmp;
> KASSERT(igi != NULL, ("%s: no igmp_ifsoftc for ifp %p", __func__, ifp));
>
> if (igi->igi_flags & IGIF_LOOPBACK) {
> @@ -874,7 +874,7 @@ igmp_input_v2_query(struct ifnet *ifp, const struct ip *ip,
> IN_MULTI_LIST_LOCK();
> IGMP_LOCK();
>
> - igi = ((struct in_ifinfo *)ifp->if_afdata[AF_INET])->ii_igmp;
> + igi = ((struct in_ifinfo *)ifp->if_inet)->ii_igmp;
> KASSERT(igi != NULL, ("%s: no igmp_ifsoftc for ifp %p", __func__, ifp));
>
> if (igi->igi_flags & IGIF_LOOPBACK) {
> @@ -1066,7 +1066,7 @@ igmp_input_v3_query(struct ifnet *ifp, const struct ip *ip,
> IN_MULTI_LIST_LOCK();
> IGMP_LOCK();
>
> - igi = ((struct in_ifinfo *)ifp->if_afdata[AF_INET])->ii_igmp;
> + igi = ((struct in_ifinfo *)ifp->if_inet)->ii_igmp;
> KASSERT(igi != NULL, ("%s: no igmp_ifsoftc for ifp %p", __func__, ifp));
>
> if (igi->igi_flags & IGIF_LOOPBACK) {
> @@ -2347,7 +2347,7 @@ igmp_change_state(struct in_multi *inm)
>
> IGMP_LOCK();
>
> - igi = ((struct in_ifinfo *)ifp->if_afdata[AF_INET])->ii_igmp;
> + igi = ((struct in_ifinfo *)ifp->if_inet)->ii_igmp;
> KASSERT(igi != NULL, ("%s: no igmp_ifsoftc for ifp %p", __func__, ifp));
>
> /*
> diff --git a/sys/netinet/in.c b/sys/netinet/in.c
> index e824c937af8e..510cfa79d54b 100644
> --- a/sys/netinet/in.c
> +++ b/sys/netinet/in.c
> @@ -669,7 +669,7 @@ in_aifaddr_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp, struct ucred *cred
> struct in_addr allhosts_addr;
> struct in_ifinfo *ii;
>
> - ii = ((struct in_ifinfo *)ifp->if_afdata[AF_INET]);
> + ii = (struct in_ifinfo *)ifp->if_inet;
> allhosts_addr.s_addr = htonl(INADDR_ALLHOSTS_GROUP);
>
> error = in_joingroup(ifp, &allhosts_addr, NULL,
> @@ -789,7 +789,7 @@ in_difaddr_ioctl(u_long cmd, caddr_t data, struct ifnet *ifp, struct ucred *cred
> if (iaIsLast && (ifp->if_flags & IFF_MULTICAST)) {
> struct in_ifinfo *ii;
>
> - ii = ((struct in_ifinfo *)ifp->if_afdata[AF_INET]);
> + ii = (struct in_ifinfo *)ifp->if_inet;
> if (ii->ii_allhosts) {
> (void)in_leavegroup(ii->ii_allhosts, NULL);
> ii->ii_allhosts = NULL;
> @@ -1329,26 +1329,62 @@ in_ifnet_broadcast(struct in_addr in, struct ifnet *ifp)
> return (false);
> }
>
> +
> +static struct lltable *in_lltattach(struct ifnet *);
> +void
> +in_ifattach(void *arg __unused, struct ifnet *ifp)
> +{
> + struct in_ifinfo *ii;
> +
> + ii = malloc(sizeof(struct in_ifinfo), M_IFADDR, M_WAITOK|M_ZERO);
> + ii->ii_llt = in_lltattach(ifp);
> + ii->ii_igmp = igmp_domifattach(ifp);
> + ifp->if_inet = ii;
> +}
> +EVENTHANDLER_DEFINE(ifnet_arrival_event, in_ifattach, NULL,
> + EVENTHANDLER_PRI_ANY);
> +
> /*
> * On interface removal, clean up IPv4 data structures hung off of the ifnet.
> */
> -void
> -in_ifdetach(struct ifnet *ifp)
> +static void
> +in_ifdetach(void *arg __unused, struct ifnet *ifp)
> {
> - IN_MULTI_LOCK();
> - in_pcbpurgeif0(&V_ripcbinfo, ifp);
> - in_pcbpurgeif0(&V_udbinfo, ifp);
> - in_pcbpurgeif0(&V_ulitecbinfo, ifp);
> - in_purgemaddrs(ifp);
> - IN_MULTI_UNLOCK();
> + struct in_ifinfo *ii = ifp->if_inet;
>
> +#ifdef VIMAGE
> /*
> - * Make sure all multicast deletions invoking if_ioctl() are
> - * completed before returning. Else we risk accessing a freed
> - * ifnet structure pointer.
> + * On VNET shutdown abort here as the stack teardown will do all
> + * the work top-down for us. XXXGL: this logic is copied from
> + * if_detach() before dom_ifattach removal. Ideally we'd like to have
> + * same logic for VNET shutdown and normal detach. This means that
> + * interfaces should be detach before protocols destroyed during VNET
> + * shutdown.
> */
> - inm_release_wait(NULL);
> + if (!VNET_IS_SHUTTING_DOWN(ifp->if_vnet))
> +#endif
> + {
> + IN_MULTI_LOCK();
> + in_pcbpurgeif0(&V_ripcbinfo, ifp);
> + in_pcbpurgeif0(&V_udbinfo, ifp);
> + in_pcbpurgeif0(&V_ulitecbinfo, ifp);
> + in_purgemaddrs(ifp);
> + IN_MULTI_UNLOCK();
> +
> + /*
> + * Make sure all multicast deletions invoking if_ioctl() are
> + * completed before returning. Else we risk accessing a freed
> + * ifnet structure pointer.
> + */
> + inm_release_wait(NULL);
> + }
> +
> + igmp_domifdetach(ifp);
> + lltable_free(ii->ii_llt);
> + free(ii, M_IFADDR);
> }
> +EVENTHANDLER_DEFINE(ifnet_departure_event, in_ifdetach, NULL,
> + EVENTHANDLER_PRI_ANY);
>
> static void
> in_ifnet_event(void *arg __unused, struct ifnet *ifp, int event)
> @@ -1862,35 +1898,9 @@ in_lltattach(struct ifnet *ifp)
> struct lltable *
> in_lltable_get(struct ifnet *ifp)
> {
> - struct lltable *llt = NULL;
> -
> - void *afdata_ptr = ifp->if_afdata[AF_INET];
> - if (afdata_ptr != NULL)
> - llt = ((struct in_ifinfo *)afdata_ptr)->ii_llt;
> - return (llt);
> -}
> -
> -void *
> -in_domifattach(struct ifnet *ifp)
> -{
> - struct in_ifinfo *ii;
> -
> - ii = malloc(sizeof(struct in_ifinfo), M_IFADDR, M_WAITOK|M_ZERO);
> -
> - ii->ii_llt = in_lltattach(ifp);
> - ii->ii_igmp = igmp_domifattach(ifp);
> -
> - return (ii);
> -}
> -
> -void
> -in_domifdetach(struct ifnet *ifp, void *aux)
> -{
> - struct in_ifinfo *ii = (struct in_ifinfo *)aux;
> -
> - MPASS(ifp->if_afdata[AF_INET] == NULL);
> + /* XXXGL: ??? */
> + if (ifp->if_inet == NULL)
> + return (NULL);
>
> - igmp_domifdetach(ifp);
> - lltable_free(ii->ii_llt);
> - free(ii, M_IFADDR);
> + return (((struct in_ifinfo *)ifp->if_inet)->ii_llt);
> }
> diff --git a/sys/netinet/in.h b/sys/netinet/in.h
> index 3f2c388548ec..0cd55bd1e409 100644
> --- a/sys/netinet/in.h
> +++ b/sys/netinet/in.h
> @@ -684,7 +684,6 @@ int inet_aton(const char *, struct in_addr *); /* in libkern */
> char *inet_ntoa_r(struct in_addr ina, char *buf); /* in libkern */
> char *inet_ntop(int, const void *, char *, socklen_t); /* in libkern */
> int inet_pton(int af, const char *, void *); /* in libkern */
> -void in_ifdetach(struct ifnet *);
>
> static inline bool
> in_broadcast(struct in_addr in)
> diff --git a/sys/netinet/in_mcast.c b/sys/netinet/in_mcast.c
> index ba112afbf002..131e72780ebe 100644
> --- a/sys/netinet/in_mcast.c
> +++ b/sys/netinet/in_mcast.c
> @@ -500,7 +500,7 @@ in_getmulti(struct ifnet *ifp, const struct in_addr *group,
>
> IN_MULTI_LOCK_ASSERT();
>
> - ii = (struct in_ifinfo *)ifp->if_afdata[AF_INET];
> + ii = (struct in_ifinfo *)ifp->if_inet;
> IN_MULTI_LIST_LOCK();
> inm = inm_lookup(ifp, *group);
> if (inm != NULL) {
> diff --git a/sys/netinet/in_proto.c b/sys/netinet/in_proto.c
> index 3ec515d216fa..ff3040bfc7b1 100644
> --- a/sys/netinet/in_proto.c
> +++ b/sys/netinet/in_proto.c
> @@ -76,8 +76,6 @@ struct domain inetdomain = {
> #ifdef VIMAGE
> .dom_rtdetach = in_detachhead,
> #endif
> - .dom_ifattach = in_domifattach,
> - .dom_ifdetach = in_domifdetach,
> .dom_nprotosw = 14,
> .dom_protosw = {
> &tcp_protosw,
> diff --git a/sys/netinet/in_var.h b/sys/netinet/in_var.h
> index 1f6f6edb9219..311d82da4605 100644
> --- a/sys/netinet/in_var.h
> +++ b/sys/netinet/in_var.h
> @@ -102,8 +102,7 @@ struct in_ifaddr {
> ((((d).s_addr ^ (a).s_addr) & (m).s_addr)) == 0 )
> #endif
>
> -#define LLTABLE(ifp) \
> - ((struct in_ifinfo *)(ifp)->if_afdata[AF_INET])->ii_llt
> +#define LLTABLE(ifp) ((struct in_ifinfo *)(ifp)->if_inet)->ii_llt
> /*
> * Hash table for IP addresses.
> */
> @@ -471,9 +470,9 @@ void ip_input(struct mbuf *);
> void ip_direct_input(struct mbuf *);
> void in_ifadown(struct ifaddr *ifa, int);
> struct mbuf *ip_tryforward(struct mbuf *);
> -void *in_domifattach(struct ifnet *);
> -void in_domifdetach(struct ifnet *, void *);
> struct rib_head *in_inithead(uint32_t fibnum);
> +void in_ifattach(void *, struct ifnet *);
> +
> #ifdef VIMAGE
> void in_detachhead(struct rib_head *rh);
> #endif
> diff --git a/sys/netinet/ip_input.c b/sys/netinet/ip_input.c
> index 6ddec6726428..2e0635f8e482 100644
> --- a/sys/netinet/ip_input.c
> +++ b/sys/netinet/ip_input.c
> @@ -352,6 +352,7 @@ VNET_SYSINIT(ip_vnet_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_FOURTH,
> static void
> ip_init(const void *unused __unused)
> {
> + struct ifnet *ifp;
>
> ipreass_init();
>
> @@ -376,6 +377,14 @@ ip_init(const void *unused __unused)
> #ifdef RSS
> netisr_register(&ip_direct_nh);
> #endif
> + /*
> + * XXXGL: we use SYSINIT() here, but go over V_ifnet. It was the same
> + * way before dom_ifattach removal. This worked because when any
> + * non-default vnet is created, there are no interfaces inside.
> + * Eventually this needs to be fixed.
> + */
> + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link)
> + in_ifattach(NULL, ifp);
> }
> SYSINIT(ip_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_THIRD, ip_init, NULL);
>
> diff --git a/sys/netinet6/in6.c b/sys/netinet6/in6.c
> index 158129258587..a953bb5546d6 100644
> --- a/sys/netinet6/in6.c
> +++ b/sys/netinet6/in6.c
> @@ -478,7 +478,7 @@ in6_control_ioctl(u_long cmd, void *data,
> /* FALLTHROUGH */
> case SIOCGIFSTAT_IN6:
> case SIOCGIFSTAT_ICMP6:
> - if (ifp->if_afdata[AF_INET6] == NULL) {
> + if (ifp->if_inet6 == NULL) {
> error = EPFNOSUPPORT;
> goto out;
> }
> @@ -525,15 +525,13 @@ in6_control_ioctl(u_long cmd, void *data,
> break;
>
> case SIOCGIFSTAT_IN6:
> - COUNTER_ARRAY_COPY(((struct in6_ifextra *)
> - ifp->if_afdata[AF_INET6])->in6_ifstat,
> + COUNTER_ARRAY_COPY(ifp->if_inet6->in6_ifstat,
> &ifr->ifr_ifru.ifru_stat,
> sizeof(struct in6_ifstat) / sizeof(uint64_t));
> break;
>
> case SIOCGIFSTAT_ICMP6:
> - COUNTER_ARRAY_COPY(((struct in6_ifextra *)
> - ifp->if_afdata[AF_INET6])->icmp6_ifstat,
> + COUNTER_ARRAY_COPY(ifp->if_inet6->icmp6_ifstat,
> &ifr->ifr_ifru.ifru_icmp6stat,
> sizeof(struct icmp6_ifstat) / sizeof(uint64_t));
> break;
> @@ -2567,16 +2565,14 @@ in6_lltattach(struct ifnet *ifp)
> struct lltable *
> in6_lltable_get(struct ifnet *ifp)
> {
> - struct lltable *llt = NULL;
> + if (ifp->if_inet6 == NULL)
> + return (NULL);
>
> - void *afdata_ptr = ifp->if_afdata[AF_INET6];
> - if (afdata_ptr != NULL)
> - llt = ((struct in6_ifextra *)afdata_ptr)->lltable;
> - return (llt);
> + return (ifp->if_inet6->lltable);
> }
>
> -void *
> -in6_domifattach(struct ifnet *ifp)
> +void
> +in6_ifarrival(void *arg __unused, struct ifnet *ifp)
> {
> struct in6_ifextra *ext;
>
> @@ -2584,7 +2580,8 @@ in6_domifattach(struct ifnet *ifp)
> switch (ifp->if_type) {
> case IFT_PFLOG:
> case IFT_PFSYNC:
> - return (NULL);
> + ifp->if_inet6 = NULL;
> + return;
> }
> ext = (struct in6_ifextra *)malloc(sizeof(*ext), M_IFADDR, M_WAITOK);
> bzero(ext, sizeof(*ext));
> @@ -2606,8 +2603,10 @@ in6_domifattach(struct ifnet *ifp)
>
> ext->mld_ifinfo = mld_domifattach(ifp);
>
> - return ext;
> + ifp->if_inet6 = ext;
> }
> +EVENTHANDLER_DEFINE(ifnet_arrival_event, in6_ifarrival, NULL,
> + EVENTHANDLER_PRI_ANY);
>
> uint32_t
> in6_ifmtu(struct ifnet *ifp)
> @@ -2615,26 +2614,6 @@ in6_ifmtu(struct ifnet *ifp)
> return (IN6_LINKMTU(ifp));
> }
>
> -void
> -in6_domifdetach(struct ifnet *ifp, void *aux)
> -{
> - struct in6_ifextra *ext = (struct in6_ifextra *)aux;
> -
> - MPASS(ifp->if_afdata[AF_INET6] == NULL);
> -
> - mld_domifdetach(ifp);
> - scope6_ifdetach(ext->scope6_id);
> - nd6_ifdetach(ifp, ext->nd_ifinfo);
> - lltable_free(ext->lltable);
> - COUNTER_ARRAY_FREE(ext->in6_ifstat,
> - sizeof(struct in6_ifstat) / sizeof(uint64_t));
> - free(ext->in6_ifstat, M_IFADDR);
> - COUNTER_ARRAY_FREE(ext->icmp6_ifstat,
> - sizeof(struct icmp6_ifstat) / sizeof(uint64_t));
> - free(ext->icmp6_ifstat, M_IFADDR);
> - free(ext, M_IFADDR);
> -}
> -
> /*
> * Convert sockaddr_in6 to sockaddr_in. Original sockaddr_in6 must be
> * v4 mapped addr or v4 compat addr
> @@ -2736,7 +2715,7 @@ in6_purge_proxy_ndp(struct ifnet *ifp)
> struct lltable *llt;
> bool need_purge;
>
> - if (ifp->if_afdata[AF_INET6] == NULL)
> + if (ifp->if_inet6 == NULL)
> return;
>
> llt = LLTABLE6(ifp);
> diff --git a/sys/netinet6/in6_ifattach.c b/sys/netinet6/in6_ifattach.c
> index 090ba610460b..a2d236cccc41 100644
> --- a/sys/netinet6/in6_ifattach.c
> +++ b/sys/netinet6/in6_ifattach.c
> @@ -50,6 +50,7 @@
> #include <net/if_dl.h>
> #include <net/if_private.h>
> #include <net/if_types.h>
> +#include <net/if_llatbl.h>
> #include <net/route.h>
> #include <net/vnet.h>
>
> @@ -783,7 +784,8 @@ in6_ifattach(struct ifnet *ifp, struct ifnet *altifp)
> {
> struct in6_ifaddr *ia;
>
> - if (ifp->if_afdata[AF_INET6] == NULL)
> + /* XXXGL: can this happen after IFT_PFLOG and IFT_PFSYNC are gone? */
> + if (ifp->if_inet6 == NULL)
> return;
> /*
> * quirks based on interface type
> @@ -857,7 +859,8 @@ _in6_ifdetach(struct ifnet *ifp, int purgeulp)
> {
> struct ifaddr *ifa, *next;
>
> - if (ifp->if_afdata[AF_INET6] == NULL)
> + /* XXXGL: can this happen after IFT_PFLOG and IFT_PFSYNC are gone? */
> + if (ifp->if_inet6 == NULL)
> return;
>
> /*
> @@ -895,6 +898,38 @@ in6_ifdetach(struct ifnet *ifp)
> _in6_ifdetach(ifp, 1);
> }
>
> +static void
> +in6_ifdeparture(void *arg __unused, struct ifnet *ifp)
> +{
> + struct in6_ifextra *ext = ifp->if_inet6;
> +
> + /* XXXGL: can this happen after IFT_PFLOG and IFT_PFSYNC are gone? */
> + if (ifp->if_inet6 == NULL)
> + return;
> +
> +#ifdef VIMAGE
> + /*
> + * On VNET shutdown abort here as the stack teardown will do all
> + * the work top-down for us. XXXGL: see comment in in.c:in_ifdetach().
> + */
> + if (!VNET_IS_SHUTTING_DOWN(ifp->if_vnet))
> +#endif
> + _in6_ifdetach(ifp, 1);
> + mld_domifdetach(ifp);
> + scope6_ifdetach(ext->scope6_id);
> + nd6_ifdetach(ifp, ext->nd_ifinfo);
> + lltable_free(ext->lltable);
> + COUNTER_ARRAY_FREE(ext->in6_ifstat,
> + sizeof(struct in6_ifstat) / sizeof(uint64_t));
> + free(ext->in6_ifstat, M_IFADDR);
> + COUNTER_ARRAY_FREE(ext->icmp6_ifstat,
> + sizeof(struct icmp6_ifstat) / sizeof(uint64_t));
> + free(ext->icmp6_ifstat, M_IFADDR);
> + free(ext, M_IFADDR);
> +}
> +EVENTHANDLER_DEFINE(ifnet_departure_event, in6_ifdeparture, NULL,
> + EVENTHANDLER_PRI_ANY);
> +
> void
> in6_ifdetach_destroy(struct ifnet *ifp)
> {
> diff --git a/sys/netinet6/in6_mcast.c b/sys/netinet6/in6_mcast.c
> index 613bd14428bb..4075e75eac85 100644
> --- a/sys/netinet6/in6_mcast.c
> +++ b/sys/netinet6/in6_mcast.c
> @@ -400,7 +400,7 @@ in6_getmulti(struct ifnet *ifp, const struct in6_addr *group,
> /*
> * Does ifp support IPv6 multicasts?
> */
> - if (ifp->if_afdata[AF_INET6] == NULL)
> + if (ifp->if_inet6 == NULL)
> error = ENODEV;
> else
> inm = in6m_lookup_locked(ifp, group);
> diff --git a/sys/netinet6/in6_proto.c b/sys/netinet6/in6_proto.c
> index f3ed72719abf..dbbe7c4b3eca 100644
> --- a/sys/netinet6/in6_proto.c
> +++ b/sys/netinet6/in6_proto.c
> @@ -111,8 +111,6 @@ struct domain inet6domain = {
> #ifdef VIMAGE
> .dom_rtdetach = in6_detachhead,
> #endif
> - .dom_ifattach = in6_domifattach,
> - .dom_ifdetach = in6_domifdetach,
> .dom_nprotosw = 14,
> .dom_protosw = {
> &tcp6_protosw,
> diff --git a/sys/netinet6/in6_var.h b/sys/netinet6/in6_var.h
> index 1210937a5dc4..0cfdde652c0a 100644
> --- a/sys/netinet6/in6_var.h
> +++ b/sys/netinet6/in6_var.h
> @@ -109,8 +109,8 @@ struct in6_ifextra {
> u_int dad_failures; /* DAD failures when using RFC 7217 stable addresses */
> };
>
> -#define LLTABLE6(ifp) (((struct in6_ifextra *)(ifp)->if_afdata[AF_INET6])->lltable)
> -#define DAD_FAILURES(ifp) (((struct in6_ifextra *)(ifp)->if_afdata[AF_INET6])->dad_failures)
> +#define LLTABLE6(ifp) ((ifp)->if_inet6->lltable)
> +#define DAD_FAILURES(ifp) ((ifp)->if_inet6->dad_failures)
>
> #ifdef _KERNEL
>
> @@ -545,8 +545,7 @@ extern struct rmlock in6_ifaddr_lock;
> #define in6_ifstat_inc(ifp, tag) \
> do { \
> if (ifp) \
> - counter_u64_add(((struct in6_ifextra *) \
> - ((ifp)->if_afdata[AF_INET6]))->in6_ifstat[ \
> + counter_u64_add((ifp)->if_inet6->in6_ifstat[ \
> offsetof(struct in6_ifstat, tag) / sizeof(uint64_t)], 1);\
> } while (/*CONSTCOND*/ 0)
> #endif /* _KERNEL */
> @@ -867,8 +866,6 @@ void in6_purgeaddr(struct ifaddr *);
> void in6_purgeifaddr(struct in6_ifaddr *);
> int in6if_do_dad(struct ifnet *);
> void in6_savemkludge(struct in6_ifaddr *);
> -void *in6_domifattach(struct ifnet *);
> -void in6_domifdetach(struct ifnet *, void *);
> uint32_t in6_ifmtu(struct ifnet *);
> struct rib_head *in6_inithead(uint32_t fibnum);
> void in6_detachhead(struct rib_head *rh);
> @@ -893,6 +890,8 @@ int in6_src_ioctl(u_long, caddr_t);
> void in6_newaddrmsg(struct in6_ifaddr *, int);
>
> void in6_purge_proxy_ndp(struct ifnet *);
> +void in6_ifarrival(void *, struct ifnet *);
> +
> /*
> * Extended API for IPv6 FIB support.
> */
> diff --git a/sys/netinet6/ip6_input.c b/sys/netinet6/ip6_input.c
> index 380391fd71f4..29fa4741a509 100644
> --- a/sys/netinet6/ip6_input.c
> +++ b/sys/netinet6/ip6_input.c
> @@ -286,6 +286,7 @@ VNET_SYSINIT(ip6_vnet_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_FOURTH,
> static void
> ip6_init(void *arg __unused)
> {
> + struct ifnet *ifp;
>
> /*
> * Register statically those protocols that are unlikely to ever go
> @@ -312,6 +313,12 @@ ip6_init(void *arg __unused)
> #ifdef RSS
> netisr_register(&ip6_direct_nh);
> #endif
> + /*
> + * XXXGL: we use SYSINIT() here, but go over V_ifnet. See comment
> + * in sys/netinet/ip_input.c:ip_init().
> + */
> + CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link)
> + in6_ifarrival(NULL, ifp);
> }
> SYSINIT(ip6_init, SI_SUB_PROTO_DOMAIN, SI_ORDER_THIRD, ip6_init, NULL);
>
> diff --git a/sys/netinet6/ip6_output.c b/sys/netinet6/ip6_output.c
> index 8ade50b13568..eb5a3a971ea0 100644
> --- a/sys/netinet6/ip6_output.c
> +++ b/sys/netinet6/ip6_output.c
> @@ -2920,7 +2920,7 @@ ip6_setpktopt(int optname, u_char *buf, int len, struct ip6_pktopts *opt,
> if (ifp == NULL)
> return (ENXIO);
> }
> - if (ifp != NULL && (ifp->if_afdata[AF_INET6] == NULL ||
> + if (ifp != NULL && (ifp->if_inet6 == NULL ||
> (ND_IFINFO(ifp)->flags & ND6_IFF_IFDISABLED) != 0))
> return (ENETDOWN);
>
> diff --git a/sys/netinet6/mld6_var.h b/sys/netinet6/mld6_var.h
> index 96b841f1564b..d75ac2450c10 100644
> --- a/sys/netinet6/mld6_var.h
> +++ b/sys/netinet6/mld6_var.h
> @@ -155,8 +155,7 @@ struct mld_ifsoftc {
> /*
> * Per-link MLD context.
> */
> *** 133 LINES SKIPPED ***
>