PERFORCE change 128682 for review

Steve Wise swise at FreeBSD.org
Mon Nov 5 07:28:04 PST 2007


http://perforce.freebsd.org/chv.cgi?CH=128682

Change 128682 by swise at swise:vic10:iwarp on 2007/11/05 15:27:49

	Get rdma_add.c to compile and load.

Affected files ...

.. //depot/projects/iwarp/sys/contrib/rdma/ib_addr.h#3 edit
.. //depot/projects/iwarp/sys/contrib/rdma/rdma_addr.c#2 edit

Differences ...

==== //depot/projects/iwarp/sys/contrib/rdma/ib_addr.h#3 (text+ko) ====

@@ -31,10 +31,15 @@
 #define IB_ADDR_H
 
 #include <contrib/rdma/ib_verbs.h>
+#include <net/if.h>
+#include <net/ethernet.h>
+
+#define MAX_ADDR_LEN ETHER_ADDR_LEN	/* XXX doesn't support IB! */
 
 struct rdma_addr_client {
 	atomic_t refcount;
 	struct cv comp;
+	struct mtx lock;
 };
 
 /**
@@ -86,7 +91,7 @@
 
 void rdma_addr_cancel(struct rdma_dev_addr *addr);
 
-int rdma_copy_addr(struct rdma_dev_addr *dev_addr, struct net_device *dev,
+int rdma_copy_addr(struct rdma_dev_addr *dev_addr, struct ifnet *dev,
 	      const unsigned char *dst_dev_addr);
 
 static inline int ip_addr_size(struct sockaddr *addr)

==== //depot/projects/iwarp/sys/contrib/rdma/rdma_addr.c#2 (text+ko) ====

@@ -31,9 +31,11 @@
 #include <sys/cdefs.h>
 
 #include <sys/param.h>
+#include <sys/condvar.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/socket.h>
+#include <sys/module.h>
 
 #include <sys/linux_compat.h>
 #include <sys/lock.h>
@@ -41,10 +43,18 @@
 #include <sys/mutex.h>
 #include <sys/rwlock.h>
 #include <sys/queue.h>
+#include <sys/taskqueue.h>
+
+#include <net/if.h>
+#include <net/if_dl.h>
+#include <net/if_var.h>
+#include <net/if_arp.h>
+#include <net/route.h>
 
+#include <net80211/ieee80211_freebsd.h>
 
-#include <net/if_arp.h>
 #include <netinet/in.h>
+#include <netinet/if_ether.h>
 
 #include <contrib/rdma/ib_addr.h>
 
@@ -67,50 +77,51 @@
 	int status;
 };
 
-static void process_req(struct work_struct *work);
+static void process_req(void *ctx, int pending);
 
 static struct mtx lock;
 
-static TAILQ_HEAD(, addr_req) req_list;
-static DECLARE_DELAYED_WORK(work, process_req);
-static struct taskqueue *addr_wq;
+static TAILQ_HEAD(addr_req_list, addr_req) req_list;
+static struct task addr_task;
+static struct taskqueue *addr_taskq;
+static struct callout addr_ch;
+static eventhandler_tag route_event_tag;
+
+static void addr_timeout(void *arg)
+{
+	taskqueue_enqueue(addr_taskq, &addr_task);
+}
 
 void rdma_addr_register_client(struct rdma_addr_client *client)
 {
 	atomic_set(&client->refcount, 1);
-	init_completion(&client->comp);
+	mtx_init(&client->lock, "rdma_addr client lock", NULL, MTX_DEF);
+	cv_init(&client->comp, "rdma_addr cv");
 }
 EXPORT_SYMBOL(rdma_addr_register_client);
 
 static inline void put_client(struct rdma_addr_client *client)
 {
+	mtx_lock(&client->lock);
 	if (atomic_dec_and_test(&client->refcount))
-		complete(&client->comp);
+		cv_signal(&client->comp);
+	mtx_unlock(&client->lock);
 }
 
 void rdma_addr_unregister_client(struct rdma_addr_client *client)
 {
 	put_client(client);
-	wait_for_completion(&client->comp);
+	mtx_lock(&client->lock);
+	cv_wait(&client->comp, &client->lock);
 }
 EXPORT_SYMBOL(rdma_addr_unregister_client);
 
-int rdma_copy_addr(struct rdma_dev_addr *dev_addr, struct net_device *dev,
+int rdma_copy_addr(struct rdma_dev_addr *dev_addr, struct ifnet *dev,
 		     const unsigned char *dst_dev_addr)
 {
-	switch (dev->type) {
-	case ARPHRD_INFINIBAND:
-		dev_addr->dev_type = RDMA_NODE_IB_CA;
-		break;
-	case ARPHRD_ETHER:
-		dev_addr->dev_type = RDMA_NODE_RNIC;
-		break;
-	default:
-		return (EADDRNOTAVAIL);
-	}
-
-	memcpy(dev_addr->src_dev_addr, dev->dev_addr, MAX_ADDR_LEN);
-	memcpy(dev_addr->broadcast, dev->broadcast, MAX_ADDR_LEN);
+	dev_addr->dev_type = RDMA_NODE_RNIC;
+	memcpy(dev_addr->src_dev_addr, IF_LLADDR(dev), MAX_ADDR_LEN);
+	memcpy(dev_addr->broadcast, dev->if_broadcastaddr, MAX_ADDR_LEN);
 	if (dst_dev_addr)
 		memcpy(dev_addr->dst_dev_addr, dst_dev_addr, MAX_ADDR_LEN);
 	return 0;
@@ -119,167 +130,138 @@
 
 int rdma_translate_ip(struct sockaddr *addr, struct rdma_dev_addr *dev_addr)
 {
-	struct net_device *dev;
-	__be32 ip = ((struct sockaddr_in *) addr)->sin_addr.s_addr;
 	int ret;
+	struct route iproute;
+	struct sockaddr_in *dst = (struct sockaddr_in *)&iproute.ro_dst;
 
-	dev = ip_dev_find(ip);
-	if (!dev)
+	bzero(&iproute, sizeof iproute);
+	dst->sin_family = AF_INET;
+	dst->sin_len = sizeof dst;
+	dst->sin_addr.s_addr = ((struct sockaddr_in *)addr)->sin_addr.s_addr;
+
+	rtalloc(&iproute);
+	if (iproute.ro_rt == NULL);
 		return (EADDRNOTAVAIL);
 
-	ret = rdma_copy_addr(dev_addr, dev, NULL);
-	dev_put(dev);
+	ret = rdma_copy_addr(dev_addr, iproute.ro_rt->rt_ifp, NULL);
+	rtfree(iproute.ro_rt);
 	return ret;
 }
 EXPORT_SYMBOL(rdma_translate_ip);
 
 static void queue_req(struct addr_req *req)
 {
-	struct addr_req *temp_req;
+	struct addr_req *tmp_req;
 	
 	mtx_lock(&lock);
-	TAILQ_FOREACH_REVERSE(tmp_req, &req_list, req_list, entry)
-	    if (time_after_eq(req->timeout, temp_req->timeout))
+	TAILQ_FOREACH_REVERSE(tmp_req, &req_list, addr_req_list, entry)
+	    if (time_after_eq(req->timeout, tmp_req->timeout))
 		    break;
 	
 	TAILQ_INSERT_AFTER(&req_list, tmp_req, req, entry);
 	
 	if (TAILQ_FIRST(&req_list) == req)	
-		callout_reset(&work, req->timeout - ticks);
+		callout_reset(&addr_ch, req->timeout - ticks, addr_timeout, NULL);
 	mtx_unlock(&lock);
 }
 
 static void addr_send_arp(struct sockaddr_in *dst_in)
 {
-	struct rtenry *rt;
-	u32 dst_ip = dst_in->sin_addr.s_addr;
+	struct route iproute;
+	struct sockaddr_in *dst = (struct sockaddr_in *)&iproute.ro_dst;
+	char dmac[ETHER_ADDR_LEN];
+
+	bzero(&iproute, sizeof iproute);
+	*dst = *dst_in;
 
-	memset(&fl, 0, sizeof fl);
-	fl.nl_u.ip4_u.daddr = dst_ip;
-	if (ip_route_output_key(&rt, &fl))
+	rtalloc(&iproute);
+	if (iproute.ro_rt == NULL);
 		return;
 
-	arp_send(ARPOP_REQUEST, ETH_P_ARP, rt->rt_gateway, rt->idev->dev,
-		 rt->rt_src, NULL, rt->idev->dev->dev_addr, NULL);
-	ip_rt_put(rt);
+	arpresolve(iproute.ro_rt->rt_ifp, iproute.ro_rt, NULL, 
+		   rt_key(iproute.ro_rt), dmac);
+
+	rtfree(iproute.ro_rt);
 }
 
 static int addr_resolve_remote(struct sockaddr_in *src_in,
 			       struct sockaddr_in *dst_in,
 			       struct rdma_dev_addr *addr)
 {
-	u32 src_ip = src_in->sin_addr.s_addr;
-	u32 dst_ip = dst_in->sin_addr.s_addr;
-	struct flowi fl;
-	struct rtable *rt;
-	struct neighbour *neigh;
-	int ret;
+	int ret = 0;
+	struct route iproute;
+	struct sockaddr_in *dst = (struct sockaddr_in *)&iproute.ro_dst;
+	char dmac[ETHER_ADDR_LEN];
+
+	bzero(&iproute, sizeof iproute);
+	*dst = *dst_in;
 
-	memset(&fl, 0, sizeof fl);
-	fl.nl_u.ip4_u.daddr = dst_ip;
-	fl.nl_u.ip4_u.saddr = src_ip;
-	ret = ip_route_output_key(&rt, &fl);
-	if (ret)
+	rtalloc(&iproute);
+	if (iproute.ro_rt == NULL);
 		goto out;
 
 	/* If the device does ARP internally, return 'done' */
-	if (rt->idev->dev->flags & IFF_NOARP) {
-		rdma_copy_addr(addr, rt->idev->dev, NULL);
+	if (iproute.ro_rt->rt_ifp->if_flags & IFF_NOARP) {
+		rdma_copy_addr(addr, iproute.ro_rt->rt_ifp, NULL);
 		goto put;
 	}
-
-	neigh = neigh_lookup(&arp_tbl, &rt->rt_gateway, rt->idev->dev);
-	if (!neigh) {
-		ret = ENODATA;
+ 	ret = arpresolve(iproute.ro_rt->rt_ifp, iproute.ro_rt, NULL, 
+		rt_key(iproute.ro_rt), dmac);
+	if (ret) {
 		goto put;
 	}
 
-	if (!(neigh->nud_state & NUD_VALID)) {
-		ret = ENODATA;
-		goto release;
-	}
-
-	if (!src_ip) {
+	if (!src_in->sin_addr.s_addr) {
+		src_in->sin_len = sizeof *src_in;
 		src_in->sin_family = dst_in->sin_family;
-		src_in->sin_addr.s_addr = rt->rt_src;
+		src_in->sin_addr.s_addr = ((struct sockaddr_in *)iproute.ro_rt->rt_ifa->ifa_addr)->sin_addr.s_addr;
 	}
 
-	ret = rdma_copy_addr(addr, neigh->dev, neigh->ha);
-release:
-	neigh_release(neigh);
+	ret = rdma_copy_addr(addr, iproute.ro_rt->rt_ifp, dmac);
 put:
-	ip_rt_put(rt);
+	rtfree(iproute.ro_rt);
 out:
 	return ret;
 }
 
-static void process_req(struct work_struct *work)
+static void process_req(void *ctx, int pending)
 {
-	struct addr_req *req, *temp_req;
+	struct addr_req *req, *tmp_req;
 	struct sockaddr_in *src_in, *dst_in;
-	struct list_head done_list;
+	TAILQ_HEAD(, addr_req) done_list;
 
 	TAILQ_INIT(&done_list);
 
 	mtx_lock(&lock);
-	list_for_each_entry_safe(req, temp_req, &req_list, list) {
-		if (req->status == ENODATA) {
+	TAILQ_FOREACH_SAFE(req, &req_list, entry, tmp_req) {
+		if (req->status == EWOULDBLOCK) {
 			src_in = (struct sockaddr_in *) &req->src_addr;
 			dst_in = (struct sockaddr_in *) &req->dst_addr;
 			req->status = addr_resolve_remote(src_in, dst_in,
 							  req->addr);
-			if (req->status && time_after_eq(jiffies, req->timeout))
+			if (req->status && time_after_eq(ticks, req->timeout))
 				req->status = ETIMEDOUT;
-			else if (req->status == ENODATA)
+			else if (req->status == EWOULDBLOCK)
 				continue;
 		}
-		list_move_tail(&req->list, &done_list);
+		TAILQ_REMOVE(&req_list, req, entry);
+		TAILQ_INSERT_TAIL(&done_list, req, entry);
 	}
 
-	if (!list_empty(&req_list)) {
-		req = list_entry(req_list.next, struct addr_req, list);
-		callout_reset(&work, req->timeout - ticks);
+	if (!TAILQ_EMPTY(&req_list)) {
+		req = TAILQ_FIRST(&req_list);
+		callout_reset(&addr_ch, req->timeout - ticks, addr_timeout, 
+			NULL);
 	}
 	mtx_unlock(&lock);
 
-	list_for_each_entry_safe(req, temp_req, &done_list, list) {
-		list_del(&req->list);
+	TAILQ_FOREACH_SAFE(req, &done_list, entry, tmp_req) {
+		TAILQ_REMOVE(&done_list, req, entry);
 		req->callback(req->status, &req->src_addr, req->addr,
 			      req->context);
 		put_client(req->client);
-		kfree(req);
-	}
-}
-
-static int addr_resolve_local(struct sockaddr_in *src_in,
-			      struct sockaddr_in *dst_in,
-			      struct rdma_dev_addr *addr)
-{
-	struct net_device *dev;
-	u32 src_ip = src_in->sin_addr.s_addr;
-	__be32 dst_ip = dst_in->sin_addr.s_addr;
-	int ret;
-
-	dev = ip_dev_find(dst_ip);
-	if (!dev)
-		return (EADDRNOTAVAIL);
-
-	if (ZERONET(src_ip)) {
-		src_in->sin_family = dst_in->sin_family;
-		src_in->sin_addr.s_addr = dst_ip;
-		ret = rdma_copy_addr(addr, dev, dev->dev_addr);
-	} else if (LOOPBACK(src_ip)) {
-		ret = rdma_translate_ip((struct sockaddr *)dst_in, addr);
-		if (!ret)
-			memcpy(addr->dst_dev_addr, dev->dev_addr, MAX_ADDR_LEN);
-	} else {
-		ret = rdma_translate_ip((struct sockaddr *)src_in, addr);
-		if (!ret)
-			memcpy(addr->dst_dev_addr, dev->dev_addr, MAX_ADDR_LEN);
+		free(req, M_DEVBUF);
 	}
-
-	dev_put(dev);
-	return ret;
 }
 
 int rdma_resolve_ip(struct rdma_addr_client *client,
@@ -293,7 +275,7 @@
 	struct addr_req *req;
 	int ret = 0;
 
-	req = kmalloc(sizeof *req, GFP_KERNEL);
+	req = malloc(sizeof *req, M_DEVBUF, M_WAITOK);
 	if (!req)
 		return (ENOMEM);
 	memset(req, 0, sizeof *req);
@@ -310,24 +292,22 @@
 	src_in = (struct sockaddr_in *) &req->src_addr;
 	dst_in = (struct sockaddr_in *) &req->dst_addr;
 
-	req->status = addr_resolve_local(src_in, dst_in, addr);
-	if (req->status == EADDRNOTAVAIL)
-		req->status = addr_resolve_remote(src_in, dst_in, addr);
+	req->status = addr_resolve_remote(src_in, dst_in, addr);
 
 	switch (req->status) {
 	case 0:
-		req->timeout = jiffies;
+		req->timeout = ticks;
 		queue_req(req);
 		break;
-	case ENODATA:
-		req->timeout = msecs_to_jiffies(timeout_ms) + jiffies;
+	case EWOULDBLOCK:
+		req->timeout = msecs_to_ticks(timeout_ms) + ticks;
 		queue_req(req);
 		addr_send_arp(dst_in);
 		break;
 	default:
 		ret = req->status;
 		atomic_dec(&client->refcount);
-		kfree(req);
+		free(req, M_DEVBUF);
 		break;
 	}
 	return ret;
@@ -336,15 +316,16 @@
 
 void rdma_addr_cancel(struct rdma_dev_addr *addr)
 {
-	struct addr_req *req, *temp_req;
+	struct addr_req *req, *tmp_req;
 
 	mtx_lock(&lock);
-	list_for_each_entry_safe(req, temp_req, &req_list, list) {
+	TAILQ_FOREACH_SAFE(req, &req_list, entry, tmp_req) {
 		if (req->addr == addr) {
 			req->status = ECANCELED;
-			req->timeout = jiffies;
-			list_move(&req->list, &req_list);
-			callout_reset(&work, req->timeout - ticks);
+			req->timeout = ticks;
+			TAILQ_REMOVE(&req_list, req, entry);
+			TAILQ_INSERT_HEAD(&req_list, req, entry);
+			callout_reset(&addr_ch, req->timeout - ticks, addr_timeout, NULL);
 			break;
 		}
 	}
@@ -352,36 +333,77 @@
 }
 EXPORT_SYMBOL(rdma_addr_cancel);
 
-static int netevent_callback(struct notifier_block *self, unsigned long event,
-	void *ctx)
+static void
+route_event_callback(void *unused, int event, struct rtentry *rt0, struct rtentry *rt1)
 {
-	if (event == NETEVENT_NEIGH_UPDATE) {
-		struct neighbour *neigh = ctx;
-
-		if (neigh->nud_state & NUD_VALID) {
-			callout_reset(&work, 0);
-		}
+	if (event == RTEVENT_ARP_UPDATE) {
+		callout_stop(&addr_ch);
+		taskqueue_enqueue(addr_taskq, &addr_task);
 	}
-	return 0;
 }
 
-static struct notifier_block nb = {
-	.notifier_call = netevent_callback
-};
-
 static int addr_init(void)
 {
+	TAILQ_INIT(&req_list);
+	mtx_init(&lock, "rdma_addr req_list lock", NULL, MTX_DEF);
+
+	addr_taskq = taskqueue_create("rdma_addr_taskq", M_WAITOK,
+		taskqueue_thread_enqueue, &addr_taskq);
+        if (addr_taskq == NULL) {
+                printf("failed to allocate rdma_addr taskqueue\n");
+                return (ENOMEM);
+        }
+        taskqueue_start_threads(&addr_taskq, 1, PI_NET, "rdma_addr taskq");
+        TASK_INIT(&addr_task, 0, process_req, NULL);
+
 	callout_init(&addr_ch, TRUE);
 
-	register_netevent_notifier(&nb);
+	route_event_tag = EVENTHANDLER_REGISTER(route_event, 
+		route_event_callback, NULL, EVENTHANDLER_PRI_ANY);
+
 	return 0;
 }
 
 static void addr_cleanup(void)
 {
-	unregister_netevent_notifier(&nb);
-	taskqueue_free(addr_wq);
+	EVENTHANDLER_DEREGISTER(route_event_callback, route_event_tag);
+	callout_stop(&addr_ch);
+	taskqueue_drain(addr_taskq, &addr_task);
+	taskqueue_free(addr_taskq);
+}
+
+static int 
+addr_load(module_t mod, int cmd, void *arg)
+{
+        int err = 0;
+
+        switch (cmd) {
+        case MOD_LOAD:
+                printf("Loading rdma_addr.\n");
+
+                addr_init();
+                break;
+        case MOD_QUIESCE:
+                break;
+        case MOD_UNLOAD:
+                printf("Unloading rdma_addr.\n");
+		addr_cleanup();
+                break;
+        case MOD_SHUTDOWN:
+                break;
+        default:
+                err = EOPNOTSUPP;
+                break;
+        }
+
+        return (err);
 }
 
-module_init(addr_init);
-module_exit(addr_cleanup);
+static moduledata_t mod_data = {
+	"rdma_addr",
+	addr_load,
+	0
+};
+
+MODULE_VERSION(rdma_addr, 1);
+DECLARE_MODULE(rdma_addr, mod_data, SI_SUB_EXEC, SI_ORDER_ANY);


More information about the p4-projects mailing list