svn commit: r353292 - in head/sys: contrib/ipfilter/netinet dev/firewire dev/iicbus dev/usb/net kern net netgraph netinet netinet6 netipsec netpfil/ipfw netpfil/pf ofed/drivers/infiniband/ulp/ipoib
Gleb Smirnoff
glebius at FreeBSD.org
Mon Oct 7 22:40:10 UTC 2019
Author: glebius
Date: Mon Oct 7 22:40:05 2019
New Revision: 353292
URL: https://svnweb.freebsd.org/changeset/base/353292
Log:
Widen NET_EPOCH coverage.
When epoch(9) was introduced to network stack, it was basically
dropped in place of existing locking, which was mutexes and
rwlocks. For the sake of performance mutex covered areas were
as small as possible, so became epoch covered areas.
However, epoch doesn't introduce any contention, it just delays
memory reclaim. So, there is no point to minimise epoch covered
areas in sense of performance. Meanwhile entering/exiting epoch
also has non-zero CPU usage, so doing this less often is a win.
Not the least is also code maintainability. In the new paradigm
we can assume that at any stage of processing a packet, we are
inside network epoch. This makes coding both input and output
path way easier.
On output path we already enter epoch quite early - in the
ip_output(), in the ip6_output().
This patch does the same for the input path. All ISR processing,
network related callouts, other ways of packet injection to the
network stack shall be performed in net_epoch. Any leaf function
that walks network configuration now asserts epoch.
Tricky part is configuration code paths - ioctls, sysctls. They
also call into leaf functions, so some need to be changed.
This patch would introduce more epoch recursions (see EPOCH_TRACE)
than we had before. They will be cleaned up separately, as several
of them aren't trivial. Note, that unlike a lock recursion the
epoch recursion is safe and just wastes a bit of resources.
Reviewed by: gallatin, hselasky, cy, adrian, kristof
Differential Revision: https://reviews.freebsd.org/D19111
Modified:
head/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c
head/sys/dev/firewire/if_fwip.c
head/sys/dev/iicbus/if_ic.c
head/sys/dev/usb/net/if_usie.c
head/sys/dev/usb/net/uhso.c
head/sys/dev/usb/net/usb_ethernet.c
head/sys/kern/uipc_socket.c
head/sys/net/if.c
head/sys/net/if_ethersubr.c
head/sys/net/if_gif.c
head/sys/net/if_me.c
head/sys/net/if_stf.c
head/sys/net/if_tuntap.c
head/sys/net/if_vlan.c
head/sys/net/netisr.c
head/sys/net/route.c
head/sys/net/rtsock.c
head/sys/netgraph/ng_ether.c
head/sys/netgraph/ng_iface.c
head/sys/netgraph/ng_ip_input.c
head/sys/netinet/if_ether.c
head/sys/netinet/igmp.c
head/sys/netinet/in.c
head/sys/netinet/in_mcast.c
head/sys/netinet/in_rmx.c
head/sys/netinet/in_var.h
head/sys/netinet/ip_carp.c
head/sys/netinet/ip_encap.c
head/sys/netinet/ip_icmp.c
head/sys/netinet/ip_input.c
head/sys/netinet/ip_mroute.c
head/sys/netinet/ip_options.c
head/sys/netinet/ip_output.c
head/sys/netinet/sctp_bsd_addr.c
head/sys/netinet6/icmp6.c
head/sys/netinet6/in6.c
head/sys/netinet6/in6_ifattach.c
head/sys/netinet6/in6_mcast.c
head/sys/netinet6/in6_var.h
head/sys/netinet6/ip6_output.c
head/sys/netinet6/mld6.c
head/sys/netinet6/nd6.c
head/sys/netinet6/nd6_nbr.c
head/sys/netinet6/nd6_rtr.c
head/sys/netinet6/raw_ip6.c
head/sys/netipsec/xform_ipcomp.c
head/sys/netpfil/ipfw/ip_dn_io.c
head/sys/netpfil/pf/pf_if.c
head/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c
Modified: head/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c
==============================================================================
--- head/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c Mon Oct 7 20:41:55 2019 (r353291)
+++ head/sys/contrib/ipfilter/netinet/ip_fil_freebsd.c Mon Oct 7 22:40:05 2019 (r353292)
@@ -1312,8 +1312,10 @@ ipf_inject(fin, m)
fr_info_t *fin;
mb_t *m;
{
+ struct epoch_tracker et;
int error = 0;
+ NET_EPOCH_ENTER(et);
if (fin->fin_out == 0) {
netisr_dispatch(NETISR_IP, m);
} else {
@@ -1321,6 +1323,7 @@ ipf_inject(fin, m)
fin->fin_ip->ip_off = ntohs(fin->fin_ip->ip_off);
error = ip_output(m, NULL, NULL, IP_FORWARDING, NULL, NULL);
}
+ NET_EPOCH_EXIT(et);
return error;
}
Modified: head/sys/dev/firewire/if_fwip.c
==============================================================================
--- head/sys/dev/firewire/if_fwip.c Mon Oct 7 20:41:55 2019 (r353291)
+++ head/sys/dev/firewire/if_fwip.c Mon Oct 7 22:40:05 2019 (r353292)
@@ -708,6 +708,7 @@ fwip_start_send (void *arg, int count)
static void
fwip_stream_input(struct fw_xferq *xferq)
{
+ struct epoch_tracker et;
struct mbuf *m, *m0;
struct m_tag *mtag;
struct ifnet *ifp;
@@ -720,6 +721,7 @@ fwip_stream_input(struct fw_xferq *xferq)
fwip = (struct fwip_softc *)xferq->sc;
ifp = fwip->fw_softc.fwip_ifp;
+ NET_EPOCH_ENTER(et);
while ((sxfer = STAILQ_FIRST(&xferq->stvalid)) != NULL) {
STAILQ_REMOVE_HEAD(&xferq->stvalid, link);
fp = mtod(sxfer->mbuf, struct fw_pkt *);
@@ -808,6 +810,7 @@ fwip_stream_input(struct fw_xferq *xferq)
firewire_input(ifp, m, src);
if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
}
+ NET_EPOCH_EXIT(et);
if (STAILQ_FIRST(&xferq->stfree) != NULL)
fwip->fd.fc->irx_enable(fwip->fd.fc, fwip->dma_ch);
}
Modified: head/sys/dev/iicbus/if_ic.c
==============================================================================
--- head/sys/dev/iicbus/if_ic.c Mon Oct 7 20:41:55 2019 (r353291)
+++ head/sys/dev/iicbus/if_ic.c Mon Oct 7 22:40:05 2019 (r353292)
@@ -309,9 +309,13 @@ icintr(device_t dev, int event, char *ptr)
BPF_TAP(sc->ic_ifp, sc->ic_ifbuf, len + ICHDRLEN);
top = m_devget(sc->ic_ifbuf + ICHDRLEN, len, 0, sc->ic_ifp, 0);
if (top) {
+ struct epoch_tracker et;
+
mtx_unlock(&sc->ic_lock);
M_SETFIB(top, sc->ic_ifp->if_fib);
+ NET_EPOCH_ENTER(et);
netisr_dispatch(NETISR_IP, top);
+ NET_EPOCH_EXIT(et);
mtx_lock(&sc->ic_lock);
}
break;
Modified: head/sys/dev/usb/net/if_usie.c
==============================================================================
--- head/sys/dev/usb/net/if_usie.c Mon Oct 7 20:41:55 2019 (r353291)
+++ head/sys/dev/usb/net/if_usie.c Mon Oct 7 22:40:05 2019 (r353292)
@@ -773,6 +773,7 @@ tr_setup:
static void
usie_if_rx_callback(struct usb_xfer *xfer, usb_error_t error)
{
+ struct epoch_tracker et;
struct usie_softc *sc = usbd_xfer_softc(xfer);
struct ifnet *ifp = sc->sc_ifp;
struct mbuf *m0;
@@ -852,6 +853,7 @@ tr_setup:
err = pkt = 0;
/* HW can aggregate multiple frames in a single USB xfer */
+ NET_EPOCH_ENTER(et);
for (;;) {
rxd = mtod(m, struct usie_desc *);
@@ -918,6 +920,7 @@ tr_setup:
m->m_data += diff;
m->m_pkthdr.len = (m->m_len -= diff);
}
+ NET_EPOCH_EXIT(et);
mtx_lock(&sc->sc_mtx);
Modified: head/sys/dev/usb/net/uhso.c
==============================================================================
--- head/sys/dev/usb/net/uhso.c Mon Oct 7 20:41:55 2019 (r353291)
+++ head/sys/dev/usb/net/uhso.c Mon Oct 7 22:40:05 2019 (r353292)
@@ -1664,6 +1664,7 @@ tr_setup:
static void
uhso_if_rxflush(void *arg)
{
+ struct epoch_tracker et;
struct uhso_softc *sc = arg;
struct ifnet *ifp = sc->sc_ifp;
uint8_t *cp;
@@ -1677,6 +1678,7 @@ uhso_if_rxflush(void *arg)
m = NULL;
mwait = sc->sc_mwait;
+ NET_EPOCH_ENTER(et);
for (;;) {
if (m == NULL) {
if ((m = mbufq_dequeue(&sc->sc_rxq)) == NULL)
@@ -1787,6 +1789,7 @@ uhso_if_rxflush(void *arg)
m = m0 != NULL ? m0 : NULL;
mtx_lock(&sc->sc_mtx);
}
+ NET_EPOCH_EXIT(et);
sc->sc_mwait = mwait;
}
Modified: head/sys/dev/usb/net/usb_ethernet.c
==============================================================================
--- head/sys/dev/usb/net/usb_ethernet.c Mon Oct 7 20:41:55 2019 (r353291)
+++ head/sys/dev/usb/net/usb_ethernet.c Mon Oct 7 22:40:05 2019 (r353292)
@@ -645,22 +645,21 @@ void
uether_rxflush(struct usb_ether *ue)
{
struct ifnet *ifp = ue->ue_ifp;
- struct mbuf *m;
+ struct epoch_tracker et;
+ struct mbuf *m, *n;
UE_LOCK_ASSERT(ue, MA_OWNED);
- for (;;) {
- m = mbufq_dequeue(&ue->ue_rxq);
- if (m == NULL)
- break;
-
- /*
- * The USB xfer has been resubmitted so its safe to unlock now.
- */
- UE_UNLOCK(ue);
+ n = mbufq_flush(&ue->ue_rxq);
+ UE_UNLOCK(ue);
+ NET_EPOCH_ENTER(et);
+ while ((m = n) != NULL) {
+ n = STAILQ_NEXT(m, m_stailqpkt);
+ m->m_nextpkt = NULL;
ifp->if_input(ifp, m);
- UE_LOCK(ue);
}
+ NET_EPOCH_EXIT(et);
+ UE_LOCK(ue);
}
/*
Modified: head/sys/kern/uipc_socket.c
==============================================================================
--- head/sys/kern/uipc_socket.c Mon Oct 7 20:41:55 2019 (r353291)
+++ head/sys/kern/uipc_socket.c Mon Oct 7 22:40:05 2019 (r353292)
@@ -146,6 +146,8 @@ __FBSDID("$FreeBSD$");
#include <netinet/tcp.h>
#include <net/vnet.h>
+#include <net/if.h> /* XXXGL: net_epoch should move out there */
+#include <net/if_var.h> /* XXXGL: net_epoch should move out there */
#include <security/mac/mac_framework.h>
Modified: head/sys/net/if.c
==============================================================================
--- head/sys/net/if.c Mon Oct 7 20:41:55 2019 (r353291)
+++ head/sys/net/if.c Mon Oct 7 22:40:05 2019 (r353292)
@@ -351,17 +351,14 @@ ifnet_byindex(u_short idx)
struct ifnet *
ifnet_byindex_ref(u_short idx)
{
- struct epoch_tracker et;
struct ifnet *ifp;
- NET_EPOCH_ENTER(et);
+ NET_EPOCH_ASSERT();
+
ifp = ifnet_byindex_locked(idx);
- if (ifp == NULL || (ifp->if_flags & IFF_DYING)) {
- NET_EPOCH_EXIT(et);
+ if (ifp == NULL || (ifp->if_flags & IFF_DYING))
return (NULL);
- }
if_ref(ifp);
- NET_EPOCH_EXIT(et);
return (ifp);
}
@@ -425,15 +422,14 @@ ifnet_setbyindex(u_short idx, struct ifnet *ifp)
struct ifaddr *
ifaddr_byindex(u_short idx)
{
- struct epoch_tracker et;
struct ifnet *ifp;
struct ifaddr *ifa = NULL;
- NET_EPOCH_ENTER(et);
+ NET_EPOCH_ASSERT();
+
ifp = ifnet_byindex_locked(idx);
if (ifp != NULL && (ifa = ifp->if_addr) != NULL)
ifa_ref(ifa);
- NET_EPOCH_EXIT(et);
return (ifa);
}
@@ -1640,39 +1636,32 @@ ifgr_groups_get(void *ifgrp)
static int
if_getgroup(struct ifgroupreq *ifgr, struct ifnet *ifp)
{
- struct epoch_tracker et;
int len, error;
struct ifg_list *ifgl;
struct ifg_req ifgrq, *ifgp;
+ NET_EPOCH_ASSERT();
+
if (ifgr->ifgr_len == 0) {
- NET_EPOCH_ENTER(et);
CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next)
ifgr->ifgr_len += sizeof(struct ifg_req);
- NET_EPOCH_EXIT(et);
return (0);
}
len = ifgr->ifgr_len;
ifgp = ifgr_groups_get(ifgr);
/* XXX: wire */
- NET_EPOCH_ENTER(et);
CK_STAILQ_FOREACH(ifgl, &ifp->if_groups, ifgl_next) {
- if (len < sizeof(ifgrq)) {
- NET_EPOCH_EXIT(et);
+ if (len < sizeof(ifgrq))
return (EINVAL);
- }
bzero(&ifgrq, sizeof ifgrq);
strlcpy(ifgrq.ifgrq_group, ifgl->ifgl_group->ifg_group,
sizeof(ifgrq.ifgrq_group));
- if ((error = copyout(&ifgrq, ifgp, sizeof(struct ifg_req)))) {
- NET_EPOCH_EXIT(et);
+ if ((error = copyout(&ifgrq, ifgp, sizeof(struct ifg_req))))
return (error);
- }
len -= sizeof(ifgrq);
ifgp++;
}
- NET_EPOCH_EXIT(et);
return (0);
}
@@ -1972,7 +1961,8 @@ ifa_ifwithaddr(const struct sockaddr *addr)
struct ifnet *ifp;
struct ifaddr *ifa;
- MPASS(in_epoch(net_epoch_preempt));
+ NET_EPOCH_ASSERT();
+
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
CK_STAILQ_FOREACH(ifa, &ifp->if_addrhead, ifa_link) {
if (ifa->ifa_addr->sa_family != addr->sa_family)
@@ -2324,17 +2314,22 @@ if_link_state_change(struct ifnet *ifp, int link_state
ifp->if_link_state = link_state;
+ /* XXXGL: reference ifp? */
taskqueue_enqueue(taskqueue_swi, &ifp->if_linktask);
}
static void
do_link_state_change(void *arg, int pending)
{
- struct ifnet *ifp = (struct ifnet *)arg;
- int link_state = ifp->if_link_state;
- CURVNET_SET(ifp->if_vnet);
+ struct epoch_tracker et;
+ struct ifnet *ifp;
+ int link_state;
- /* Notify that the link state has changed. */
+ NET_EPOCH_ENTER(et);
+ ifp = arg;
+ link_state = ifp->if_link_state;
+
+ CURVNET_SET(ifp->if_vnet);
rt_ifmsg(ifp);
if (ifp->if_vlantrunk != NULL)
(*vlan_link_state_p)(ifp);
@@ -2360,6 +2355,7 @@ do_link_state_change(void *arg, int pending)
(link_state == LINK_STATE_UP) ? "UP" : "DOWN" );
EVENTHANDLER_INVOKE(ifnet_link_event, ifp, link_state);
CURVNET_RESTORE();
+ NET_EPOCH_EXIT(et);
}
/*
@@ -2912,9 +2908,14 @@ ifhwioctl(u_long cmd, struct ifnet *ifp, caddr_t data,
break;
case CASE_IOC_IFGROUPREQ(SIOCGIFGROUP):
- if ((error = if_getgroup((struct ifgroupreq *)data, ifp)))
- return (error);
+ {
+ struct epoch_tracker et;
+
+ NET_EPOCH_ENTER(et);
+ error = if_getgroup((struct ifgroupreq *)data, ifp);
+ NET_EPOCH_EXIT(et);
break;
+ }
case CASE_IOC_IFGROUPREQ(SIOCDIFGROUP):
error = priv_check(td, PRIV_NET_DELIFGROUP);
@@ -2983,7 +2984,7 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, s
#ifdef COMPAT_FREEBSD32
caddr_t saved_data = NULL;
struct ifmediareq ifmr;
- struct ifmediareq *ifmrp;
+ struct ifmediareq *ifmrp = NULL;
#endif
struct ifnet *ifp;
struct ifreq *ifr;
@@ -2999,12 +3000,10 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, s
}
#endif
-
switch (cmd) {
case SIOCGIFCONF:
error = ifconf(cmd, data);
- CURVNET_RESTORE();
- return (error);
+ goto out_noref;
#ifdef COMPAT_FREEBSD32
case SIOCGIFCONF32:
@@ -3017,16 +3016,14 @@ ifioctl(struct socket *so, u_long cmd, caddr_t data, s
ifc.ifc_buf = PTRIN(ifc32->ifc_buf);
error = ifconf(SIOCGIFCONF, (void *)&ifc);
- CURVNET_RESTORE();
if (error == 0)
ifc32->ifc_len = ifc.ifc_len;
- return (error);
+ goto out_noref;
}
#endif
}
#ifdef COMPAT_FREEBSD32
- ifmrp = NULL;
switch (cmd) {
case SIOCGIFMEDIA32:
case SIOCGIFXMEDIA32:
@@ -3618,16 +3615,15 @@ if_delmulti(struct ifnet *ifp, struct sockaddr *sa)
struct ifmultiaddr *ifma;
int lastref;
#ifdef INVARIANTS
- struct epoch_tracker et;
struct ifnet *oifp;
- NET_EPOCH_ENTER(et);
+ NET_EPOCH_ASSERT();
+
CK_STAILQ_FOREACH(oifp, &V_ifnet, if_link)
if (ifp == oifp)
break;
if (ifp != oifp)
ifp = NULL;
- NET_EPOCH_EXIT(et);
KASSERT(ifp != NULL, ("%s: ifnet went away", __func__));
#endif
@@ -3693,16 +3689,15 @@ if_delmulti_ifma_flags(struct ifmultiaddr *ifma, int f
if (ifp == NULL) {
printf("%s: ifma_ifp seems to be detached\n", __func__);
} else {
- struct epoch_tracker et;
struct ifnet *oifp;
- NET_EPOCH_ENTER(et);
+ NET_EPOCH_ASSERT();
+
CK_STAILQ_FOREACH(oifp, &V_ifnet, if_link)
if (ifp == oifp)
break;
if (ifp != oifp)
ifp = NULL;
- NET_EPOCH_EXIT(et);
}
#endif
/*
@@ -3826,11 +3821,11 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr,
struct sockaddr_dl *sdl;
struct ifaddr *ifa;
struct ifreq ifr;
- struct epoch_tracker et;
int rc;
+ NET_EPOCH_ASSERT();
+
rc = 0;
- NET_EPOCH_ENTER(et);
ifa = ifp->if_addr;
if (ifa == NULL) {
rc = EINVAL;
@@ -3864,7 +3859,6 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr,
* to re-init it in order to reprogram its
* address filter.
*/
- NET_EPOCH_EXIT(et);
if ((ifp->if_flags & IFF_UP) != 0) {
if (ifp->if_ioctl) {
ifp->if_flags &= ~IFF_UP;
@@ -3879,8 +3873,7 @@ if_setlladdr(struct ifnet *ifp, const u_char *lladdr,
}
EVENTHANDLER_INVOKE(iflladdr_event, ifp);
return (0);
- out:
- NET_EPOCH_EXIT(et);
+out:
return (rc);
}
Modified: head/sys/net/if_ethersubr.c
==============================================================================
--- head/sys/net/if_ethersubr.c Mon Oct 7 20:41:55 2019 (r353291)
+++ head/sys/net/if_ethersubr.c Mon Oct 7 22:40:05 2019 (r353292)
@@ -800,6 +800,7 @@ VNET_SYSUNINIT(vnet_ether_uninit, SI_SUB_PROTO_IF, SI_
static void
ether_input(struct ifnet *ifp, struct mbuf *m)
{
+ struct epoch_tracker et;
struct mbuf *mn;
/*
@@ -807,22 +808,24 @@ ether_input(struct ifnet *ifp, struct mbuf *m)
* m_nextpkt. We split them up into separate packets here and pass
* them up. This allows the drivers to amortize the receive lock.
*/
+ CURVNET_SET_QUIET(ifp->if_vnet);
+ NET_EPOCH_ENTER(et);
while (m) {
mn = m->m_nextpkt;
m->m_nextpkt = NULL;
/*
- * We will rely on rcvif being set properly in the deferred context,
- * so assert it is correct here.
+ * We will rely on rcvif being set properly in the deferred
+ * context, so assert it is correct here.
*/
MPASS((m->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0);
KASSERT(m->m_pkthdr.rcvif == ifp, ("%s: ifnet mismatch m %p "
"rcvif %p ifp %p", __func__, m, m->m_pkthdr.rcvif, ifp));
- CURVNET_SET_QUIET(ifp->if_vnet);
netisr_dispatch(NETISR_ETHER, m);
- CURVNET_RESTORE();
m = mn;
}
+ NET_EPOCH_EXIT(et);
+ CURVNET_RESTORE();
}
/*
@@ -835,6 +838,7 @@ ether_demux(struct ifnet *ifp, struct mbuf *m)
int i, isr;
u_short ether_type;
+ NET_EPOCH_ASSERT();
KASSERT(ifp != NULL, ("%s: NULL interface pointer", __func__));
/* Do not grab PROMISC frames in case we are re-entered. */
Modified: head/sys/net/if_gif.c
==============================================================================
--- head/sys/net/if_gif.c Mon Oct 7 20:41:55 2019 (r353291)
+++ head/sys/net/if_gif.c Mon Oct 7 22:40:05 2019 (r353292)
@@ -415,6 +415,8 @@ gif_input(struct mbuf *m, struct ifnet *ifp, int proto
struct ifnet *oldifp;
int isr, n, af;
+ NET_EPOCH_ASSERT();
+
if (ifp == NULL) {
/* just in case */
m_freem(m);
Modified: head/sys/net/if_me.c
==============================================================================
--- head/sys/net/if_me.c Mon Oct 7 20:41:55 2019 (r353291)
+++ head/sys/net/if_me.c Mon Oct 7 22:40:05 2019 (r353292)
@@ -451,6 +451,8 @@ me_input(struct mbuf *m, int off, int proto, void *arg
struct ip *ip;
int hlen;
+ NET_EPOCH_ASSERT();
+
ifp = ME2IFP(sc);
/* checks for short packets */
hlen = sizeof(struct mobhdr);
Modified: head/sys/net/if_stf.c
==============================================================================
--- head/sys/net/if_stf.c Mon Oct 7 20:41:55 2019 (r353291)
+++ head/sys/net/if_stf.c Mon Oct 7 22:40:05 2019 (r353292)
@@ -613,6 +613,8 @@ in_stf_input(struct mbuf *m, int off, int proto, void
u_int8_t otos, itos;
struct ifnet *ifp;
+ NET_EPOCH_ASSERT();
+
if (proto != IPPROTO_IPV6) {
m_freem(m);
return (IPPROTO_DONE);
Modified: head/sys/net/if_tuntap.c
==============================================================================
--- head/sys/net/if_tuntap.c Mon Oct 7 20:41:55 2019 (r353291)
+++ head/sys/net/if_tuntap.c Mon Oct 7 22:40:05 2019 (r353292)
@@ -1662,6 +1662,7 @@ tunwrite_l2(struct tuntap_softc *tp, struct mbuf *m)
static int
tunwrite_l3(struct tuntap_softc *tp, struct mbuf *m)
{
+ struct epoch_tracker et;
struct ifnet *ifp;
int family, isr;
@@ -1702,7 +1703,9 @@ tunwrite_l3(struct tuntap_softc *tp, struct mbuf *m)
if_inc_counter(ifp, IFCOUNTER_IPACKETS, 1);
CURVNET_SET(ifp->if_vnet);
M_SETFIB(m, ifp->if_fib);
+ NET_EPOCH_ENTER(et);
netisr_dispatch(isr, m);
+ NET_EPOCH_EXIT(et);
CURVNET_RESTORE();
return (0);
}
Modified: head/sys/net/if_vlan.c
==============================================================================
--- head/sys/net/if_vlan.c Mon Oct 7 20:41:55 2019 (r353291)
+++ head/sys/net/if_vlan.c Mon Oct 7 22:40:05 2019 (r353292)
@@ -255,7 +255,6 @@ static struct sx _VLAN_SX_ID;
#define TRUNK_LOCK_DESTROY(trunk) mtx_destroy(&(trunk)->lock)
#define TRUNK_WLOCK(trunk) mtx_lock(&(trunk)->lock)
#define TRUNK_WUNLOCK(trunk) mtx_unlock(&(trunk)->lock)
-#define TRUNK_LOCK_ASSERT(trunk) MPASS(in_epoch(net_epoch_preempt) || mtx_owned(&(trunk)->lock))
#define TRUNK_WLOCK_ASSERT(trunk) mtx_assert(&(trunk)->lock, MA_OWNED);
/*
@@ -704,18 +703,17 @@ vlan_ifdetach(void *arg __unused, struct ifnet *ifp)
static struct ifnet *
vlan_trunkdev(struct ifnet *ifp)
{
- struct epoch_tracker et;
struct ifvlan *ifv;
+ NET_EPOCH_ASSERT();
+
if (ifp->if_type != IFT_L2VLAN)
return (NULL);
- NET_EPOCH_ENTER(et);
ifv = ifp->if_softc;
ifp = NULL;
if (ifv->ifv_trunk)
ifp = PARENT(ifv);
- NET_EPOCH_EXIT(et);
return (ifp);
}
@@ -787,21 +785,18 @@ vlan_setcookie(struct ifnet *ifp, void *cookie)
static struct ifnet *
vlan_devat(struct ifnet *ifp, uint16_t vid)
{
- struct epoch_tracker et;
struct ifvlantrunk *trunk;
struct ifvlan *ifv;
- NET_EPOCH_ENTER(et);
+ NET_EPOCH_ASSERT();
+
trunk = ifp->if_vlantrunk;
- if (trunk == NULL) {
- NET_EPOCH_EXIT(et);
+ if (trunk == NULL)
return (NULL);
- }
ifp = NULL;
ifv = vlan_gethash(trunk, vid);
if (ifv)
ifp = ifv->ifv_ifp;
- NET_EPOCH_EXIT(et);
return (ifp);
}
@@ -1140,16 +1135,15 @@ vlan_init(void *foo __unused)
static int
vlan_transmit(struct ifnet *ifp, struct mbuf *m)
{
- struct epoch_tracker et;
struct ifvlan *ifv;
struct ifnet *p;
int error, len, mcast;
- NET_EPOCH_ENTER(et);
+ NET_EPOCH_ASSERT();
+
ifv = ifp->if_softc;
if (TRUNK(ifv) == NULL) {
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
- NET_EPOCH_EXIT(et);
m_freem(m);
return (ENETDOWN);
}
@@ -1169,7 +1163,6 @@ vlan_transmit(struct ifnet *ifp, struct mbuf *m)
vst = mst_to_vst(mst);
if (vst->tag->ifp != p) {
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
- NET_EPOCH_EXIT(et);
m_freem(m);
return (EAGAIN);
}
@@ -1185,14 +1178,12 @@ vlan_transmit(struct ifnet *ifp, struct mbuf *m)
*/
if (!UP_AND_RUNNING(p)) {
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
- NET_EPOCH_EXIT(et);
m_freem(m);
return (ENETDOWN);
}
if (!ether_8021q_frame(&m, ifp, p, ifv->ifv_vid, ifv->ifv_pcp)) {
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
- NET_EPOCH_EXIT(et);
return (0);
}
@@ -1206,7 +1197,6 @@ vlan_transmit(struct ifnet *ifp, struct mbuf *m)
if_inc_counter(ifp, IFCOUNTER_OMCASTS, mcast);
} else
if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
- NET_EPOCH_EXIT(et);
return (error);
}
@@ -1214,19 +1204,17 @@ static int
vlan_output(struct ifnet *ifp, struct mbuf *m, const struct sockaddr *dst,
struct route *ro)
{
- struct epoch_tracker et;
struct ifvlan *ifv;
struct ifnet *p;
- NET_EPOCH_ENTER(et);
+ NET_EPOCH_ASSERT();
+
ifv = ifp->if_softc;
if (TRUNK(ifv) == NULL) {
- NET_EPOCH_EXIT(et);
m_freem(m);
return (ENETDOWN);
}
p = PARENT(ifv);
- NET_EPOCH_EXIT(et);
return p->if_output(ifp, m, dst, ro);
}
@@ -1242,16 +1230,15 @@ vlan_qflush(struct ifnet *ifp __unused)
static void
vlan_input(struct ifnet *ifp, struct mbuf *m)
{
- struct epoch_tracker et;
struct ifvlantrunk *trunk;
struct ifvlan *ifv;
struct m_tag *mtag;
uint16_t vid, tag;
- NET_EPOCH_ENTER(et);
+ NET_EPOCH_ASSERT();
+
trunk = ifp->if_vlantrunk;
if (trunk == NULL) {
- NET_EPOCH_EXIT(et);
m_freem(m);
return;
}
@@ -1274,7 +1261,6 @@ vlan_input(struct ifnet *ifp, struct mbuf *m)
if (m->m_len < sizeof(*evl) &&
(m = m_pullup(m, sizeof(*evl))) == NULL) {
if_printf(ifp, "cannot pullup VLAN header\n");
- NET_EPOCH_EXIT(et);
return;
}
evl = mtod(m, struct ether_vlan_header *);
@@ -1297,7 +1283,6 @@ vlan_input(struct ifnet *ifp, struct mbuf *m)
__func__, ifp->if_xname, ifp->if_type);
#endif
if_inc_counter(ifp, IFCOUNTER_NOPROTO, 1);
- NET_EPOCH_EXIT(et);
m_freem(m);
return;
}
@@ -1307,7 +1292,6 @@ vlan_input(struct ifnet *ifp, struct mbuf *m)
ifv = vlan_gethash(trunk, vid);
if (ifv == NULL || !UP_AND_RUNNING(ifv->ifv_ifp)) {
- NET_EPOCH_EXIT(et);
if_inc_counter(ifp, IFCOUNTER_NOPROTO, 1);
m_freem(m);
return;
@@ -1327,7 +1311,6 @@ vlan_input(struct ifnet *ifp, struct mbuf *m)
sizeof(uint8_t), M_NOWAIT);
if (mtag == NULL) {
if_inc_counter(ifp, IFCOUNTER_IERRORS, 1);
- NET_EPOCH_EXIT(et);
m_freem(m);
return;
}
@@ -1338,7 +1321,6 @@ vlan_input(struct ifnet *ifp, struct mbuf *m)
m->m_pkthdr.rcvif = ifv->ifv_ifp;
if_inc_counter(ifv->ifv_ifp, IFCOUNTER_IPACKETS, 1);
- NET_EPOCH_EXIT(et);
/* Pass it back through the parent's input routine. */
(*ifv->ifv_ifp->if_input)(ifv->ifv_ifp, m);
@@ -1364,11 +1346,12 @@ vlan_lladdr_fn(void *arg, int pending __unused)
static int
vlan_config(struct ifvlan *ifv, struct ifnet *p, uint16_t vid)
{
- struct epoch_tracker et;
struct ifvlantrunk *trunk;
struct ifnet *ifp;
int error = 0;
+ NET_EPOCH_ASSERT();
+
/*
* We can handle non-ethernet hardware types as long as
* they handle the tagging and headers themselves.
@@ -1469,9 +1452,7 @@ vlan_config(struct ifvlan *ifv, struct ifnet *p, uint1
ifp->if_link_state = p->if_link_state;
- NET_EPOCH_ENTER(et);
vlan_capabilities(ifv);
- NET_EPOCH_EXIT(et);
/*
* Set up our interface address to reflect the underlying
@@ -1643,17 +1624,14 @@ vlan_setflags(struct ifnet *ifp, int status)
static void
vlan_link_state(struct ifnet *ifp)
{
- struct epoch_tracker et;
struct ifvlantrunk *trunk;
struct ifvlan *ifv;
- /* Called from a taskqueue_swi task, so we cannot sleep. */
- NET_EPOCH_ENTER(et);
+ NET_EPOCH_ASSERT();
+
trunk = ifp->if_vlantrunk;
- if (trunk == NULL) {
- NET_EPOCH_EXIT(et);
+ if (trunk == NULL)
return;
- }
TRUNK_WLOCK(trunk);
VLAN_FOREACH(ifv, trunk) {
@@ -1662,7 +1640,6 @@ vlan_link_state(struct ifnet *ifp)
trunk->parent->if_link_state);
}
TRUNK_WUNLOCK(trunk);
- NET_EPOCH_EXIT(et);
}
static void
@@ -1674,8 +1651,9 @@ vlan_capabilities(struct ifvlan *ifv)
int cap = 0, ena = 0, mena;
u_long hwa = 0;
- VLAN_SXLOCK_ASSERT();
NET_EPOCH_ASSERT();
+ VLAN_SXLOCK_ASSERT();
+
p = PARENT(ifv);
ifp = ifv->ifv_ifp;
@@ -1791,7 +1769,6 @@ vlan_capabilities(struct ifvlan *ifv)
static void
vlan_trunk_capabilities(struct ifnet *ifp)
{
- struct epoch_tracker et;
struct ifvlantrunk *trunk;
struct ifvlan *ifv;
@@ -1801,11 +1778,8 @@ vlan_trunk_capabilities(struct ifnet *ifp)
VLAN_SUNLOCK();
return;
}
- NET_EPOCH_ENTER(et);
- VLAN_FOREACH(ifv, trunk) {
+ VLAN_FOREACH(ifv, trunk)
vlan_capabilities(ifv);
- }
- NET_EPOCH_EXIT(et);
VLAN_SUNLOCK();
}
@@ -1820,6 +1794,8 @@ vlan_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data
struct vlanreq vlr;
int error = 0;
+ NET_EPOCH_ASSERT();
+
ifr = (struct ifreq *)data;
ifa = (struct ifaddr *) data;
ifv = ifp->if_softc;
@@ -1996,13 +1972,8 @@ vlan_ioctl(struct ifnet *ifp, u_long cmd, caddr_t data
VLAN_SLOCK();
ifv->ifv_capenable = ifr->ifr_reqcap;
trunk = TRUNK(ifv);
- if (trunk != NULL) {
- struct epoch_tracker et;
-
- NET_EPOCH_ENTER(et);
+ if (trunk != NULL)
vlan_capabilities(ifv);
- NET_EPOCH_EXIT(et);
- }
VLAN_SUNLOCK();
break;
Modified: head/sys/net/netisr.c
==============================================================================
--- head/sys/net/netisr.c Mon Oct 7 20:41:55 2019 (r353291)
+++ head/sys/net/netisr.c Mon Oct 7 22:40:05 2019 (r353292)
@@ -861,6 +861,7 @@ static u_int
netisr_process_workstream_proto(struct netisr_workstream *nwsp, u_int proto)
{
struct netisr_work local_npw, *npwp;
+ struct epoch_tracker et;
u_int handled;
struct mbuf *m;
@@ -890,6 +891,7 @@ netisr_process_workstream_proto(struct netisr_workstre
npwp->nw_len = 0;
nwsp->nws_pendingbits &= ~(1 << proto);
NWS_UNLOCK(nwsp);
+ NET_EPOCH_ENTER(et);
while ((m = local_npw.nw_head) != NULL) {
local_npw.nw_head = m->m_nextpkt;
m->m_nextpkt = NULL;
@@ -902,6 +904,7 @@ netisr_process_workstream_proto(struct netisr_workstre
netisr_proto[proto].np_handler(m);
CURVNET_RESTORE();
}
+ NET_EPOCH_EXIT(et);
KASSERT(local_npw.nw_len == 0,
("%s(%u): len %u", __func__, proto, local_npw.nw_len));
if (netisr_proto[proto].np_drainedcpu)
@@ -1088,6 +1091,7 @@ netisr_dispatch_src(u_int proto, uintptr_t source, str
int dosignal, error;
u_int cpuid, dispatch_policy;
+ NET_EPOCH_ASSERT();
KASSERT(proto < NETISR_MAXPROT,
("%s: invalid proto %u", __func__, proto));
#ifdef NETISR_LOCKING
Modified: head/sys/net/route.c
==============================================================================
--- head/sys/net/route.c Mon Oct 7 20:41:55 2019 (r353291)
+++ head/sys/net/route.c Mon Oct 7 22:40:05 2019 (r353292)
@@ -593,12 +593,12 @@ rtredirect_fib(struct sockaddr *dst,
int error = 0;
short *stat = NULL;
struct rt_addrinfo info;
- struct epoch_tracker et;
struct ifaddr *ifa;
struct rib_head *rnh;
+ NET_EPOCH_ASSERT();
+
ifa = NULL;
- NET_EPOCH_ENTER(et);
rnh = rt_tables_get_rnh(fibnum, dst->sa_family);
if (rnh == NULL) {
error = EAFNOSUPPORT;
@@ -693,7 +693,6 @@ done:
if (rt)
RTFREE_LOCKED(rt);
out:
- NET_EPOCH_EXIT(et);
if (error)
V_rtstat.rts_badredirect++;
else if (stat != NULL)
Modified: head/sys/net/rtsock.c
==============================================================================
--- head/sys/net/rtsock.c Mon Oct 7 20:41:55 2019 (r353291)
+++ head/sys/net/rtsock.c Mon Oct 7 22:40:05 2019 (r353292)
@@ -560,6 +560,7 @@ route_output(struct mbuf *m, struct socket *so, ...)
struct rib_head *rnh;
struct rt_addrinfo info;
struct sockaddr_storage ss;
+ struct epoch_tracker et;
#ifdef INET6
struct sockaddr_in6 *sin6;
int i, rti_need_deembed = 0;
@@ -579,6 +580,7 @@ route_output(struct mbuf *m, struct socket *so, ...)
return (ENOBUFS);
if ((m->m_flags & M_PKTHDR) == 0)
panic("route_output");
+ NET_EPOCH_ENTER(et);
len = m->m_pkthdr.len;
if (len < sizeof(*rtm) ||
len != mtod(m, struct rt_msghdr *)->rtm_msglen)
@@ -803,11 +805,11 @@ route_output(struct mbuf *m, struct socket *so, ...)
NET_EPOCH_ENTER(et);
ifa = ifa_ifwithnet(info.rti_info[RTAX_DST], 1,
RT_ALL_FIBS);
+ NET_EPOCH_EXIT(et);
if (ifa != NULL)
rt_maskedcopy(ifa->ifa_addr,
&laddr,
ifa->ifa_netmask);
- NET_EPOCH_EXIT(et);
} else
rt_maskedcopy(rt->rt_ifa->ifa_addr,
&laddr,
@@ -898,6 +900,7 @@ report:
}
flush:
+ NET_EPOCH_EXIT(et);
if (rt != NULL)
RTFREE(rt);
/*
@@ -1761,11 +1764,9 @@ sysctl_iflist(int af, struct walkarg *w)
struct rt_addrinfo info;
int len, error = 0;
struct sockaddr_storage ss;
- struct epoch_tracker et;
bzero((caddr_t)&info, sizeof(info));
bzero(&ifd, sizeof(ifd));
- NET_EPOCH_ENTER(et);
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
if (w->w_arg && w->w_arg != ifp->if_index)
continue;
@@ -1815,7 +1816,6 @@ sysctl_iflist(int af, struct walkarg *w)
info.rti_info[RTAX_BRD] = NULL;
}
done:
- NET_EPOCH_EXIT(et);
return (error);
}
@@ -1823,16 +1823,16 @@ static int
sysctl_ifmalist(int af, struct walkarg *w)
{
struct rt_addrinfo info;
- struct epoch_tracker et;
struct ifaddr *ifa;
struct ifmultiaddr *ifma;
struct ifnet *ifp;
int error, len;
+ NET_EPOCH_ASSERT();
+
error = 0;
bzero((caddr_t)&info, sizeof(info));
- NET_EPOCH_ENTER(et);
CK_STAILQ_FOREACH(ifp, &V_ifnet, if_link) {
if (w->w_arg && w->w_arg != ifp->if_index)
continue;
@@ -1867,7 +1867,6 @@ sysctl_ifmalist(int af, struct walkarg *w)
if (error != 0)
break;
}
- NET_EPOCH_EXIT(et);
return (error);
}
@@ -1875,6 +1874,7 @@ static int
sysctl_rtsock(SYSCTL_HANDLER_ARGS)
{
RIB_RLOCK_TRACKER;
+ struct epoch_tracker et;
int *name = (int *)arg1;
u_int namelen = arg2;
struct rib_head *rnh = NULL; /* silence compiler. */
@@ -1918,8 +1918,8 @@ sysctl_rtsock(SYSCTL_HANDLER_ARGS)
w.w_tmemsize = 65536;
w.w_tmem = malloc(w.w_tmemsize, M_TEMP, M_WAITOK);
+ NET_EPOCH_ENTER(et);
switch (w.w_op) {
-
case NET_RT_DUMP:
case NET_RT_FLAGS:
if (af == 0) { /* dump all tables */
@@ -1946,13 +1946,9 @@ sysctl_rtsock(SYSCTL_HANDLER_ARGS)
for (error = 0; error == 0 && i <= lim; i++) {
rnh = rt_tables_get_rnh(fib, i);
if (rnh != NULL) {
- struct epoch_tracker et;
-
RIB_RLOCK(rnh);
- NET_EPOCH_ENTER(et);
error = rnh->rnh_walktree(&rnh->head,
sysctl_dumpentry, &w);
- NET_EPOCH_EXIT(et);
RIB_RUNLOCK(rnh);
} else if (af != 0)
error = EAFNOSUPPORT;
@@ -1968,6 +1964,7 @@ sysctl_rtsock(SYSCTL_HANDLER_ARGS)
error = sysctl_ifmalist(af, &w);
break;
}
+ NET_EPOCH_EXIT(et);
free(w.w_tmem, M_TEMP);
return (error);
*** DIFF OUTPUT TRUNCATED AT 1000 LINES ***
More information about the svn-src-head
mailing list