PERFORCE change 124550 for review
Marko Zec
zec at FreeBSD.org
Thu Aug 2 16:14:37 PDT 2007
http://perforce.freebsd.org/chv.cgi?CH=124550
Change 124550 by zec at zec_tpx32 on 2007/08/02 23:14:03
Experimentation with enforcing initialization ordering
of vnet modules by specifiying module interdepencencies.
The vnet module registration process was contemplated
so that it should closely resemble the initialization
ordering of networking subsystems in the standard kernel.
However, it turns out that in some circumstances we might
need more strict ordering enforcement on a virtualized
system, given that for some subsystems (e.g. netgraph)
the initialization sequence can vary significantly
depending on whether the code is statically compiled in
the kernel, or is kldloaded at run time. The ordering
problem is perhaps more pronounced during cleanups of
vnet instances, for which our networking code was
never really designed.
Hence, each vnet module can now specify another module
for which it expects to be handled earlier during the
vnet instatiation process than itself. The module
registration handler will check whether the prerequisite
module is already registered, and if not, postpone the
registration of the requested module until the
prerequisite shows up. Only the time can tell whether
such a mechanism makes real sense...
While here, in effect revert change 124477 to ng_base.c,
given that now netgraph initialization point will be
determined through the new dependency enforcement
mechanisms.
Affected files ...
.. //depot/projects/vimage/src/sys/kern/kern_vimage.c#31 edit
.. //depot/projects/vimage/src/sys/kern/uipc_domain.c#9 edit
.. //depot/projects/vimage/src/sys/net/if.c#18 edit
.. //depot/projects/vimage/src/sys/net/if_gif.c#8 edit
.. //depot/projects/vimage/src/sys/net/if_loop.c#16 edit
.. //depot/projects/vimage/src/sys/net/route.c#10 edit
.. //depot/projects/vimage/src/sys/netgraph/ng_base.c#17 edit
.. //depot/projects/vimage/src/sys/netinet/if_ether.c#13 edit
.. //depot/projects/vimage/src/sys/netinet/ip_fw2.c#20 edit
.. //depot/projects/vimage/src/sys/netinet/ip_input.c#20 edit
.. //depot/projects/vimage/src/sys/netinet6/ip6_input.c#17 edit
.. //depot/projects/vimage/src/sys/netipsec/ipsec.c#13 edit
.. //depot/projects/vimage/src/sys/netipsec/xform_ah.c#9 edit
.. //depot/projects/vimage/src/sys/netipsec/xform_esp.c#8 edit
.. //depot/projects/vimage/src/sys/netipsec/xform_ipcomp.c#7 edit
.. //depot/projects/vimage/src/sys/netipsec/xform_ipip.c#10 edit
.. //depot/projects/vimage/src/sys/sys/vimage.h#29 edit
Differences ...
==== //depot/projects/vimage/src/sys/kern/kern_vimage.c#31 (text+ko) ====
@@ -69,6 +69,7 @@
MALLOC_DEFINE(M_VCPU, "vcpu", "cpu resource control block");
static void vi_destroy(struct vimage *);
+static void vnet_mod_complete_registration(struct vnet_modlink *);
static int vnet_mod_constructor(struct vnet_modlink *);
static int vnet_mod_destructor(struct vnet_modlink *);
@@ -125,6 +126,7 @@
static int last_vi_id = 0;
static TAILQ_HEAD(vnet_modlink_head, vnet_modlink) vnet_modlink_head;
+static TAILQ_HEAD(vnet_modpending_head, vnet_modlink) vnet_modpending_head;
void vnet_mod_register(vmi)
const struct vnet_modinfo *vmi;
@@ -137,13 +139,13 @@
const void *iarg;
const char *iname;
{
- struct vnet_modlink *vml;
+ struct vnet_modlink *vml, *vml_iter;
/* Do not register the same module instance more than once */
- TAILQ_FOREACH(vml, &vnet_modlink_head, vml_mod_le)
- if (vml->vml_modinfo == vmi && vml->vml_iarg == iarg)
+ TAILQ_FOREACH(vml_iter, &vnet_modlink_head, vml_mod_le)
+ if (vml_iter->vml_modinfo == vmi && vml_iter->vml_iarg == iarg)
break;
- if (vml != NULL)
+ if (vml_iter != NULL)
panic("attempt to register an already registered vnet module");
vml = vi_malloc(sizeof(struct vnet_modlink), M_VIMAGE, M_NOWAIT);
@@ -158,11 +160,52 @@
vml->vml_modinfo = vmi;
vml->vml_iarg = iarg;
vml->vml_iname = iname;
+
+ /* Check whether the module we depend on is already registered */
+ if (vmi->vmi_dependson != VNET_MOD_NONE) {
+ TAILQ_FOREACH(vml_iter, &vnet_modlink_head, vml_mod_le)
+ if (vml_iter->vml_modinfo->vmi_id ==
+ vmi->vmi_dependson)
+ break; /* Depencency found, we are done */
+ if (vml_iter == NULL) {
+#ifdef DEBUG_ORDERING
+ printf("dependency %d missing for vnet mod %s, postponing registration\n", vmi->vmi_dependson, vmi->vmi_name);
+#endif /* DEBUG_ORDERING */
+ TAILQ_INSERT_TAIL(&vnet_modpending_head, vml,
+ vml_mod_le);
+ return;
+ }
+ }
+
+ vnet_mod_complete_registration(vml);
+}
+
+void vnet_mod_complete_registration(vml)
+struct vnet_modlink *vml;
+{
+ struct vnet_modlink *vml_iter;
+
TAILQ_INSERT_TAIL(&vnet_modlink_head, vml, vml_mod_le);
VNET_ITERLOOP_BEGIN_QUIET();
vnet_mod_constructor(vml);
VNET_ITERLOOP_END();
+
+ /* Check for pending modules depending on us */
+ do {
+ TAILQ_FOREACH(vml_iter, &vnet_modpending_head, vml_mod_le)
+ if (vml_iter->vml_modinfo->vmi_dependson ==
+ vml->vml_modinfo->vmi_id)
+ break;
+ if (vml_iter != NULL) {
+#ifdef DEBUG_ORDERING
+ printf("vnet mod %s now registering, dependency %d loaded\n", vml_iter->vml_modinfo->vmi_name, vml->vml_modinfo->vmi_id);
+#endif /* DEBUG_ORDERING */
+ TAILQ_REMOVE(&vnet_modpending_head, vml_iter,
+ vml_mod_le);
+ vnet_mod_complete_registration(vml_iter);
+ }
+ } while (vml_iter != NULL);
}
void vnet_mod_deregister(vmi)
@@ -675,6 +718,7 @@
/* vnet module list is both forward and reverse traversable */
TAILQ_INIT(&vnet_modlink_head);
+ TAILQ_INIT(&vnet_modpending_head);
LIST_INIT(&vimage_head);
LIST_INIT(&vnet_head);
@@ -700,7 +744,19 @@
static void
vi_init_done(void *unused)
{
+ struct vnet_modlink *vml_iter;
+
curvnet = NULL;
+
+ if (TAILQ_EMPTY(&vnet_modpending_head))
+ return;
+
+ printf("vnet modules with unresolved dependencies:\n");
+ TAILQ_FOREACH(vml_iter, &vnet_modpending_head, vml_mod_le)
+ printf(" %s depending on %d:\n",
+ vml_iter->vml_modinfo->vmi_name,
+ vml_iter->vml_modinfo->vmi_dependson);
+ panic("going nowhere without my vnet modules!");
}
SYSINIT(vimage, SI_SUB_VIMAGE, SI_ORDER_FIRST, vi_init, NULL)
==== //depot/projects/vimage/src/sys/kern/uipc_domain.c#9 (text+ko) ====
@@ -108,7 +108,8 @@
.pru_sopoll = pru_sopoll_notsupp,
};
-VNET_MOD_DECLARE_STATELESS(DOMAIN, domain, net_init_domain, net_detach_domain)
+VNET_MOD_DECLARE_STATELESS(DOMAIN, domain, net_init_domain, net_detach_domain,
+ NET)
static void
protosw_init(struct protosw *pr)
==== //depot/projects/vimage/src/sys/net/if.c#18 (text+ko) ====
@@ -170,9 +170,10 @@
VNET_SYMMAP(net, rttrash),
VNET_SYMMAP_END
};
+#endif
-VNET_MOD_DECLARE(NET, net, vnet_net_symmap, vnet_net_iattach, vnet_net_idetach)
-#endif
+VNET_MOD_DECLARE(NET, net, vnet_net_iattach, vnet_net_idetach,
+ NONE, vnet_net_symmap)
/*
* System initialization
==== //depot/projects/vimage/src/sys/net/if_gif.c#8 (text+ko) ====
@@ -144,7 +144,7 @@
SYSCTL_V_INT(V_NET, vnet_gif, _net_link_gif, OID_AUTO, parallel_tunnels,
CTLFLAG_RW, parallel_tunnels, 0, "Allow parallel tunnels?");
-VNET_MOD_DECLARE(GIF, gif, NULL, vnet_gif_iattach, NULL)
+VNET_MOD_DECLARE(GIF, gif, NULL, vnet_gif_iattach, NET, NULL)
static int
gif_clone_create(ifc, unit, params)
==== //depot/projects/vimage/src/sys/net/if_loop.c#16 (text+ko) ====
@@ -180,7 +180,8 @@
return (0);
}
-VNET_MOD_DECLARE_STATELESS(LOIF, loif, vnet_loif_iattach, vnet_loif_idetach)
+VNET_MOD_DECLARE_STATELESS(LOIF, loif, vnet_loif_iattach, vnet_loif_idetach,
+ NET)
static int vnet_loif_iattach(unused)
const void *unused;
==== //depot/projects/vimage/src/sys/net/route.c#10 (text+ko) ====
@@ -80,7 +80,8 @@
*/
#define RNTORT(p) ((struct rtentry *)(p))
-VNET_MOD_DECLARE_STATELESS(RTABLE, rtable, rtable_init, rtable_idetach)
+VNET_MOD_DECLARE_STATELESS(RTABLE, rtable, rtable_init, rtable_idetach,
+ NET)
static int
rtable_init(unused)
==== //depot/projects/vimage/src/sys/netgraph/ng_base.c#17 (text+ko) ====
@@ -251,12 +251,10 @@
static vnet_attach_fn vnet_netgraph_iattach;
#ifdef VIMAGE
static vnet_detach_fn vnet_netgraph_idetach;
-static void netgraph_register(void *);
-SYSINIT(netgraph, SI_SUB_PROTO_END, SI_ORDER_FIRST, netgraph_register, NULL)
#endif /* VIMAGE */
-VNET_MOD_DECLARE(NETGRAPH, netgraph, NULL, vnet_netgraph_iattach,
- vnet_netgraph_idetach)
+VNET_MOD_DECLARE(NETGRAPH, netgraph, vnet_netgraph_iattach,
+ vnet_netgraph_idetach, LOIF, NULL)
#ifdef NETGRAPH_DEBUG /*----------------------------------------------*/
/*
@@ -3191,7 +3189,9 @@
uma_zone_set_max(ng_qzone, maxalloc);
netisr_register(NETISR_NETGRAPH, (netisr_t *)ngintr, NULL,
NETISR_MPSAFE);
-#ifndef VIMAGE
+#ifdef VIMAGE
+ vnet_mod_register(&vnet_netgraph_modinfo);
+#else
vnet_netgraph_iattach(NULL);
#endif /* !VIMAGE */
break;
@@ -3239,11 +3239,6 @@
return 0;
}
-
-void netgraph_register(void *unused)
-{
- vnet_mod_register(&vnet_netgraph_modinfo);
-}
#endif /* VIMAGE */
static moduledata_t netgraph_mod = {
==== //depot/projects/vimage/src/sys/netinet/if_ether.c#13 (text+ko) ====
@@ -131,7 +131,7 @@
static void in_arpinput(struct mbuf *);
#endif
-VNET_MOD_DECLARE_STATELESS(ARP, arp, arp_iattach, NULL)
+VNET_MOD_DECLARE_STATELESS(ARP, arp, arp_iattach, NULL, INET)
/*
* Timeout routine.
==== //depot/projects/vimage/src/sys/netinet/ip_fw2.c#20 (text+ko) ====
@@ -115,7 +115,7 @@
static int vnet_ipfw_iattach(const void *);
static int vnet_ipfw_idetach(const void *);
-VNET_MOD_DECLARE(IPFW, ipfw, NULL, vnet_ipfw_iattach, vnet_ipfw_idetach)
+VNET_MOD_DECLARE(IPFW, ipfw, vnet_ipfw_iattach, vnet_ipfw_idetach, INET, NULL)
/*
* set_disable contains one bit per set value (0..31).
==== //depot/projects/vimage/src/sys/netinet/ip_input.c#20 (text+ko) ====
@@ -215,7 +215,7 @@
#ifdef VIMAGE
static void vnet_inet_register(void);
-VNET_MOD_DECLARE(INET, inet, NULL, NULL, NULL)
+VNET_MOD_DECLARE(INET, inet, NULL, NULL, NET, NULL)
static void vnet_inet_register()
{
==== //depot/projects/vimage/src/sys/netinet6/ip6_input.c#17 (text+ko) ====
@@ -148,7 +148,7 @@
#ifdef VIMAGE
static void vnet_inet6_register(void);
-VNET_MOD_DECLARE(INET6, inet6, NULL, NULL, NULL)
+VNET_MOD_DECLARE(INET6, inet6, NULL, NULL, INET, NULL)
static void vnet_inet6_register()
{
==== //depot/projects/vimage/src/sys/netipsec/ipsec.c#13 (text+ko) ====
@@ -223,9 +223,8 @@
ipsecstats, CTLFLAG_RD, ipsec6stat, ipsecstat, "");
#endif /* INET6 */
-#ifdef VIMAGE
-VNET_MOD_DECLARE(IPSEC, ipsec, NULL, vnet_ipsec_iattach, vnet_ipsec_idetach)
-#endif
+VNET_MOD_DECLARE(IPSEC, ipsec, vnet_ipsec_iattach, vnet_ipsec_idetach,
+ INET, NULL)
static int ipsec4_setspidx_inpcb __P((struct mbuf *, struct inpcb *pcb));
#ifdef INET6
==== //depot/projects/vimage/src/sys/netipsec/xform_ah.c#9 (text+ko) ====
@@ -106,7 +106,7 @@
static int ah_iattach(const void *);
-VNET_MOD_DECLARE_STATELESS(AH, ah, ah_iattach, NULL)
+VNET_MOD_DECLARE_STATELESS(AH, ah, ah_iattach, NULL, IPSEC)
static unsigned char ipseczeroes[256]; /* larger than an ip6 extension hdr */
==== //depot/projects/vimage/src/sys/netipsec/xform_esp.c#8 (text+ko) ====
@@ -96,7 +96,7 @@
static int esp_iattach(const void *);
-VNET_MOD_DECLARE_STATELESS(ESP, esp, esp_iattach, NULL)
+VNET_MOD_DECLARE_STATELESS(ESP, esp, esp_iattach, NULL, IPSEC)
/*
* NB: this is public for use by the PF_KEY support.
==== //depot/projects/vimage/src/sys/netipsec/xform_ipcomp.c#7 (text+ko) ====
@@ -82,7 +82,7 @@
static int ipcomp_iattach(const void *);
-VNET_MOD_DECLARE_STATELESS(IPCOMP, ipcomp, ipcomp_iattach, NULL)
+VNET_MOD_DECLARE_STATELESS(IPCOMP, ipcomp, ipcomp_iattach, NULL, IPSEC)
static int ipcomp_input_cb(struct cryptop *crp);
static int ipcomp_output_cb(struct cryptop *crp);
==== //depot/projects/vimage/src/sys/netipsec/xform_ipip.c#10 (text+ko) ====
@@ -113,7 +113,7 @@
static int ipip_iattach(const void *);
-VNET_MOD_DECLARE_STATELESS(IPIP, ipip, ipip_iattach, NULL)
+VNET_MOD_DECLARE_STATELESS(IPIP, ipip, ipip_iattach, NULL, IPSEC)
#ifdef INET6
/*
==== //depot/projects/vimage/src/sys/sys/vimage.h#29 (text+ko) ====
@@ -67,6 +67,7 @@
#define V_PROCG 2
#define V_CPU 3
+#define VNET_MOD_NONE -1
#define VNET_MOD_NET 0
#define VNET_MOD_NETGRAPH 1
#define VNET_MOD_INET 2
@@ -78,6 +79,8 @@
#define VNET_MOD_ALTQ 8
#define VNET_MOD_IPX 9
#define VNET_MOD_ATALK 10
+#define VNET_MOD_NG_IFACE 21
+#define VNET_MOD_NG_EIFACE 22
#define VNET_MOD_ESP 23
#define VNET_MOD_IPIP 24
#define VNET_MOD_AH 25
@@ -121,6 +124,7 @@
struct vnet_modinfo {
int vmi_id;
+ int vmi_dependson;
char *vmi_name;
vnet_attach_fn *vmi_iattach;
vnet_detach_fn *vmi_idetach;
@@ -135,9 +139,11 @@
const char *vml_iname;
};
-#define VNET_MOD_DECLARE(m_name_uc, m_name_lc, m_symmap, m_iattach, m_idetach) \
+#define VNET_MOD_DECLARE(m_name_uc, m_name_lc, m_iattach, m_idetach, \
+ m_dependson, m_symmap) \
static const struct vnet_modinfo vnet_##m_name_lc##_modinfo = { \
.vmi_id = VNET_MOD_##m_name_uc, \
+ .vmi_dependson = VNET_MOD_##m_dependson, \
.vmi_name = #m_name_lc, \
.vmi_iattach = m_iattach, \
.vmi_idetach = m_idetach, \
@@ -146,9 +152,11 @@
.vmi_symmap = m_symmap \
};
-#define VNET_MOD_DECLARE_STATELESS(m_name_uc, m_name_lc, m_iattach, m_idetach) \
+#define VNET_MOD_DECLARE_STATELESS(m_name_uc, m_name_lc, m_iattach, m_idetach, \
+ m_dependson) \
static const struct vnet_modinfo vnet_##m_name_lc##_modinfo = { \
.vmi_id = VNET_MOD_##m_name_uc, \
+ .vmi_dependson = VNET_MOD_##m_dependson, \
.vmi_name = #m_name_lc, \
.vmi_iattach = m_iattach, \
.vmi_idetach = m_idetach \
More information about the p4-projects
mailing list