PERFORCE change 128682 for review
Steve Wise
swise at FreeBSD.org
Mon Nov 5 07:28:04 PST 2007
http://perforce.freebsd.org/chv.cgi?CH=128682
Change 128682 by swise at swise:vic10:iwarp on 2007/11/05 15:27:49
Get rdma_add.c to compile and load.
Affected files ...
.. //depot/projects/iwarp/sys/contrib/rdma/ib_addr.h#3 edit
.. //depot/projects/iwarp/sys/contrib/rdma/rdma_addr.c#2 edit
Differences ...
==== //depot/projects/iwarp/sys/contrib/rdma/ib_addr.h#3 (text+ko) ====
@@ -31,10 +31,15 @@
#define IB_ADDR_H
#include <contrib/rdma/ib_verbs.h>
+#include <net/if.h>
+#include <net/ethernet.h>
+
+#define MAX_ADDR_LEN ETHER_ADDR_LEN /* XXX doesn't support IB! */
struct rdma_addr_client {
atomic_t refcount;
struct cv comp;
+ struct mtx lock;
};
/**
@@ -86,7 +91,7 @@
void rdma_addr_cancel(struct rdma_dev_addr *addr);
-int rdma_copy_addr(struct rdma_dev_addr *dev_addr, struct net_device *dev,
+int rdma_copy_addr(struct rdma_dev_addr *dev_addr, struct ifnet *dev,
const unsigned char *dst_dev_addr);
static inline int ip_addr_size(struct sockaddr *addr)
==== //depot/projects/iwarp/sys/contrib/rdma/rdma_addr.c#2 (text+ko) ====
@@ -31,9 +31,11 @@
#include <sys/cdefs.h>
#include <sys/param.h>
+#include <sys/condvar.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/socket.h>
+#include <sys/module.h>
#include <sys/linux_compat.h>
#include <sys/lock.h>
@@ -41,10 +43,18 @@
#include <sys/mutex.h>
#include <sys/rwlock.h>
#include <sys/queue.h>
+#include <sys/taskqueue.h>
+
+#include <net/if.h>
+#include <net/if_dl.h>
+#include <net/if_var.h>
+#include <net/if_arp.h>
+#include <net/route.h>
+#include <net80211/ieee80211_freebsd.h>
-#include <net/if_arp.h>
#include <netinet/in.h>
+#include <netinet/if_ether.h>
#include <contrib/rdma/ib_addr.h>
@@ -67,50 +77,51 @@
int status;
};
-static void process_req(struct work_struct *work);
+static void process_req(void *ctx, int pending);
static struct mtx lock;
-static TAILQ_HEAD(, addr_req) req_list;
-static DECLARE_DELAYED_WORK(work, process_req);
-static struct taskqueue *addr_wq;
+static TAILQ_HEAD(addr_req_list, addr_req) req_list;
+static struct task addr_task;
+static struct taskqueue *addr_taskq;
+static struct callout addr_ch;
+static eventhandler_tag route_event_tag;
+
+static void addr_timeout(void *arg)
+{
+ taskqueue_enqueue(addr_taskq, &addr_task);
+}
void rdma_addr_register_client(struct rdma_addr_client *client)
{
atomic_set(&client->refcount, 1);
- init_completion(&client->comp);
+ mtx_init(&client->lock, "rdma_addr client lock", NULL, MTX_DEF);
+ cv_init(&client->comp, "rdma_addr cv");
}
EXPORT_SYMBOL(rdma_addr_register_client);
static inline void put_client(struct rdma_addr_client *client)
{
+ mtx_lock(&client->lock);
if (atomic_dec_and_test(&client->refcount))
- complete(&client->comp);
+ cv_signal(&client->comp);
+ mtx_unlock(&client->lock);
}
void rdma_addr_unregister_client(struct rdma_addr_client *client)
{
put_client(client);
- wait_for_completion(&client->comp);
+ mtx_lock(&client->lock);
+ cv_wait(&client->comp, &client->lock);
}
EXPORT_SYMBOL(rdma_addr_unregister_client);
-int rdma_copy_addr(struct rdma_dev_addr *dev_addr, struct net_device *dev,
+int rdma_copy_addr(struct rdma_dev_addr *dev_addr, struct ifnet *dev,
const unsigned char *dst_dev_addr)
{
- switch (dev->type) {
- case ARPHRD_INFINIBAND:
- dev_addr->dev_type = RDMA_NODE_IB_CA;
- break;
- case ARPHRD_ETHER:
- dev_addr->dev_type = RDMA_NODE_RNIC;
- break;
- default:
- return (EADDRNOTAVAIL);
- }
-
- memcpy(dev_addr->src_dev_addr, dev->dev_addr, MAX_ADDR_LEN);
- memcpy(dev_addr->broadcast, dev->broadcast, MAX_ADDR_LEN);
+ dev_addr->dev_type = RDMA_NODE_RNIC;
+ memcpy(dev_addr->src_dev_addr, IF_LLADDR(dev), MAX_ADDR_LEN);
+ memcpy(dev_addr->broadcast, dev->if_broadcastaddr, MAX_ADDR_LEN);
if (dst_dev_addr)
memcpy(dev_addr->dst_dev_addr, dst_dev_addr, MAX_ADDR_LEN);
return 0;
@@ -119,167 +130,138 @@
int rdma_translate_ip(struct sockaddr *addr, struct rdma_dev_addr *dev_addr)
{
- struct net_device *dev;
- __be32 ip = ((struct sockaddr_in *) addr)->sin_addr.s_addr;
int ret;
+ struct route iproute;
+ struct sockaddr_in *dst = (struct sockaddr_in *)&iproute.ro_dst;
- dev = ip_dev_find(ip);
- if (!dev)
+ bzero(&iproute, sizeof iproute);
+ dst->sin_family = AF_INET;
+ dst->sin_len = sizeof dst;
+ dst->sin_addr.s_addr = ((struct sockaddr_in *)addr)->sin_addr.s_addr;
+
+ rtalloc(&iproute);
+ if (iproute.ro_rt == NULL);
return (EADDRNOTAVAIL);
- ret = rdma_copy_addr(dev_addr, dev, NULL);
- dev_put(dev);
+ ret = rdma_copy_addr(dev_addr, iproute.ro_rt->rt_ifp, NULL);
+ rtfree(iproute.ro_rt);
return ret;
}
EXPORT_SYMBOL(rdma_translate_ip);
static void queue_req(struct addr_req *req)
{
- struct addr_req *temp_req;
+ struct addr_req *tmp_req;
mtx_lock(&lock);
- TAILQ_FOREACH_REVERSE(tmp_req, &req_list, req_list, entry)
- if (time_after_eq(req->timeout, temp_req->timeout))
+ TAILQ_FOREACH_REVERSE(tmp_req, &req_list, addr_req_list, entry)
+ if (time_after_eq(req->timeout, tmp_req->timeout))
break;
TAILQ_INSERT_AFTER(&req_list, tmp_req, req, entry);
if (TAILQ_FIRST(&req_list) == req)
- callout_reset(&work, req->timeout - ticks);
+ callout_reset(&addr_ch, req->timeout - ticks, addr_timeout, NULL);
mtx_unlock(&lock);
}
static void addr_send_arp(struct sockaddr_in *dst_in)
{
- struct rtenry *rt;
- u32 dst_ip = dst_in->sin_addr.s_addr;
+ struct route iproute;
+ struct sockaddr_in *dst = (struct sockaddr_in *)&iproute.ro_dst;
+ char dmac[ETHER_ADDR_LEN];
+
+ bzero(&iproute, sizeof iproute);
+ *dst = *dst_in;
- memset(&fl, 0, sizeof fl);
- fl.nl_u.ip4_u.daddr = dst_ip;
- if (ip_route_output_key(&rt, &fl))
+ rtalloc(&iproute);
+ if (iproute.ro_rt == NULL);
return;
- arp_send(ARPOP_REQUEST, ETH_P_ARP, rt->rt_gateway, rt->idev->dev,
- rt->rt_src, NULL, rt->idev->dev->dev_addr, NULL);
- ip_rt_put(rt);
+ arpresolve(iproute.ro_rt->rt_ifp, iproute.ro_rt, NULL,
+ rt_key(iproute.ro_rt), dmac);
+
+ rtfree(iproute.ro_rt);
}
static int addr_resolve_remote(struct sockaddr_in *src_in,
struct sockaddr_in *dst_in,
struct rdma_dev_addr *addr)
{
- u32 src_ip = src_in->sin_addr.s_addr;
- u32 dst_ip = dst_in->sin_addr.s_addr;
- struct flowi fl;
- struct rtable *rt;
- struct neighbour *neigh;
- int ret;
+ int ret = 0;
+ struct route iproute;
+ struct sockaddr_in *dst = (struct sockaddr_in *)&iproute.ro_dst;
+ char dmac[ETHER_ADDR_LEN];
+
+ bzero(&iproute, sizeof iproute);
+ *dst = *dst_in;
- memset(&fl, 0, sizeof fl);
- fl.nl_u.ip4_u.daddr = dst_ip;
- fl.nl_u.ip4_u.saddr = src_ip;
- ret = ip_route_output_key(&rt, &fl);
- if (ret)
+ rtalloc(&iproute);
+ if (iproute.ro_rt == NULL);
goto out;
/* If the device does ARP internally, return 'done' */
- if (rt->idev->dev->flags & IFF_NOARP) {
- rdma_copy_addr(addr, rt->idev->dev, NULL);
+ if (iproute.ro_rt->rt_ifp->if_flags & IFF_NOARP) {
+ rdma_copy_addr(addr, iproute.ro_rt->rt_ifp, NULL);
goto put;
}
-
- neigh = neigh_lookup(&arp_tbl, &rt->rt_gateway, rt->idev->dev);
- if (!neigh) {
- ret = ENODATA;
+ ret = arpresolve(iproute.ro_rt->rt_ifp, iproute.ro_rt, NULL,
+ rt_key(iproute.ro_rt), dmac);
+ if (ret) {
goto put;
}
- if (!(neigh->nud_state & NUD_VALID)) {
- ret = ENODATA;
- goto release;
- }
-
- if (!src_ip) {
+ if (!src_in->sin_addr.s_addr) {
+ src_in->sin_len = sizeof *src_in;
src_in->sin_family = dst_in->sin_family;
- src_in->sin_addr.s_addr = rt->rt_src;
+ src_in->sin_addr.s_addr = ((struct sockaddr_in *)iproute.ro_rt->rt_ifa->ifa_addr)->sin_addr.s_addr;
}
- ret = rdma_copy_addr(addr, neigh->dev, neigh->ha);
-release:
- neigh_release(neigh);
+ ret = rdma_copy_addr(addr, iproute.ro_rt->rt_ifp, dmac);
put:
- ip_rt_put(rt);
+ rtfree(iproute.ro_rt);
out:
return ret;
}
-static void process_req(struct work_struct *work)
+static void process_req(void *ctx, int pending)
{
- struct addr_req *req, *temp_req;
+ struct addr_req *req, *tmp_req;
struct sockaddr_in *src_in, *dst_in;
- struct list_head done_list;
+ TAILQ_HEAD(, addr_req) done_list;
TAILQ_INIT(&done_list);
mtx_lock(&lock);
- list_for_each_entry_safe(req, temp_req, &req_list, list) {
- if (req->status == ENODATA) {
+ TAILQ_FOREACH_SAFE(req, &req_list, entry, tmp_req) {
+ if (req->status == EWOULDBLOCK) {
src_in = (struct sockaddr_in *) &req->src_addr;
dst_in = (struct sockaddr_in *) &req->dst_addr;
req->status = addr_resolve_remote(src_in, dst_in,
req->addr);
- if (req->status && time_after_eq(jiffies, req->timeout))
+ if (req->status && time_after_eq(ticks, req->timeout))
req->status = ETIMEDOUT;
- else if (req->status == ENODATA)
+ else if (req->status == EWOULDBLOCK)
continue;
}
- list_move_tail(&req->list, &done_list);
+ TAILQ_REMOVE(&req_list, req, entry);
+ TAILQ_INSERT_TAIL(&done_list, req, entry);
}
- if (!list_empty(&req_list)) {
- req = list_entry(req_list.next, struct addr_req, list);
- callout_reset(&work, req->timeout - ticks);
+ if (!TAILQ_EMPTY(&req_list)) {
+ req = TAILQ_FIRST(&req_list);
+ callout_reset(&addr_ch, req->timeout - ticks, addr_timeout,
+ NULL);
}
mtx_unlock(&lock);
- list_for_each_entry_safe(req, temp_req, &done_list, list) {
- list_del(&req->list);
+ TAILQ_FOREACH_SAFE(req, &done_list, entry, tmp_req) {
+ TAILQ_REMOVE(&done_list, req, entry);
req->callback(req->status, &req->src_addr, req->addr,
req->context);
put_client(req->client);
- kfree(req);
- }
-}
-
-static int addr_resolve_local(struct sockaddr_in *src_in,
- struct sockaddr_in *dst_in,
- struct rdma_dev_addr *addr)
-{
- struct net_device *dev;
- u32 src_ip = src_in->sin_addr.s_addr;
- __be32 dst_ip = dst_in->sin_addr.s_addr;
- int ret;
-
- dev = ip_dev_find(dst_ip);
- if (!dev)
- return (EADDRNOTAVAIL);
-
- if (ZERONET(src_ip)) {
- src_in->sin_family = dst_in->sin_family;
- src_in->sin_addr.s_addr = dst_ip;
- ret = rdma_copy_addr(addr, dev, dev->dev_addr);
- } else if (LOOPBACK(src_ip)) {
- ret = rdma_translate_ip((struct sockaddr *)dst_in, addr);
- if (!ret)
- memcpy(addr->dst_dev_addr, dev->dev_addr, MAX_ADDR_LEN);
- } else {
- ret = rdma_translate_ip((struct sockaddr *)src_in, addr);
- if (!ret)
- memcpy(addr->dst_dev_addr, dev->dev_addr, MAX_ADDR_LEN);
+ free(req, M_DEVBUF);
}
-
- dev_put(dev);
- return ret;
}
int rdma_resolve_ip(struct rdma_addr_client *client,
@@ -293,7 +275,7 @@
struct addr_req *req;
int ret = 0;
- req = kmalloc(sizeof *req, GFP_KERNEL);
+ req = malloc(sizeof *req, M_DEVBUF, M_WAITOK);
if (!req)
return (ENOMEM);
memset(req, 0, sizeof *req);
@@ -310,24 +292,22 @@
src_in = (struct sockaddr_in *) &req->src_addr;
dst_in = (struct sockaddr_in *) &req->dst_addr;
- req->status = addr_resolve_local(src_in, dst_in, addr);
- if (req->status == EADDRNOTAVAIL)
- req->status = addr_resolve_remote(src_in, dst_in, addr);
+ req->status = addr_resolve_remote(src_in, dst_in, addr);
switch (req->status) {
case 0:
- req->timeout = jiffies;
+ req->timeout = ticks;
queue_req(req);
break;
- case ENODATA:
- req->timeout = msecs_to_jiffies(timeout_ms) + jiffies;
+ case EWOULDBLOCK:
+ req->timeout = msecs_to_ticks(timeout_ms) + ticks;
queue_req(req);
addr_send_arp(dst_in);
break;
default:
ret = req->status;
atomic_dec(&client->refcount);
- kfree(req);
+ free(req, M_DEVBUF);
break;
}
return ret;
@@ -336,15 +316,16 @@
void rdma_addr_cancel(struct rdma_dev_addr *addr)
{
- struct addr_req *req, *temp_req;
+ struct addr_req *req, *tmp_req;
mtx_lock(&lock);
- list_for_each_entry_safe(req, temp_req, &req_list, list) {
+ TAILQ_FOREACH_SAFE(req, &req_list, entry, tmp_req) {
if (req->addr == addr) {
req->status = ECANCELED;
- req->timeout = jiffies;
- list_move(&req->list, &req_list);
- callout_reset(&work, req->timeout - ticks);
+ req->timeout = ticks;
+ TAILQ_REMOVE(&req_list, req, entry);
+ TAILQ_INSERT_HEAD(&req_list, req, entry);
+ callout_reset(&addr_ch, req->timeout - ticks, addr_timeout, NULL);
break;
}
}
@@ -352,36 +333,77 @@
}
EXPORT_SYMBOL(rdma_addr_cancel);
-static int netevent_callback(struct notifier_block *self, unsigned long event,
- void *ctx)
+static void
+route_event_callback(void *unused, int event, struct rtentry *rt0, struct rtentry *rt1)
{
- if (event == NETEVENT_NEIGH_UPDATE) {
- struct neighbour *neigh = ctx;
-
- if (neigh->nud_state & NUD_VALID) {
- callout_reset(&work, 0);
- }
+ if (event == RTEVENT_ARP_UPDATE) {
+ callout_stop(&addr_ch);
+ taskqueue_enqueue(addr_taskq, &addr_task);
}
- return 0;
}
-static struct notifier_block nb = {
- .notifier_call = netevent_callback
-};
-
static int addr_init(void)
{
+ TAILQ_INIT(&req_list);
+ mtx_init(&lock, "rdma_addr req_list lock", NULL, MTX_DEF);
+
+ addr_taskq = taskqueue_create("rdma_addr_taskq", M_WAITOK,
+ taskqueue_thread_enqueue, &addr_taskq);
+ if (addr_taskq == NULL) {
+ printf("failed to allocate rdma_addr taskqueue\n");
+ return (ENOMEM);
+ }
+ taskqueue_start_threads(&addr_taskq, 1, PI_NET, "rdma_addr taskq");
+ TASK_INIT(&addr_task, 0, process_req, NULL);
+
callout_init(&addr_ch, TRUE);
- register_netevent_notifier(&nb);
+ route_event_tag = EVENTHANDLER_REGISTER(route_event,
+ route_event_callback, NULL, EVENTHANDLER_PRI_ANY);
+
return 0;
}
static void addr_cleanup(void)
{
- unregister_netevent_notifier(&nb);
- taskqueue_free(addr_wq);
+ EVENTHANDLER_DEREGISTER(route_event_callback, route_event_tag);
+ callout_stop(&addr_ch);
+ taskqueue_drain(addr_taskq, &addr_task);
+ taskqueue_free(addr_taskq);
+}
+
+static int
+addr_load(module_t mod, int cmd, void *arg)
+{
+ int err = 0;
+
+ switch (cmd) {
+ case MOD_LOAD:
+ printf("Loading rdma_addr.\n");
+
+ addr_init();
+ break;
+ case MOD_QUIESCE:
+ break;
+ case MOD_UNLOAD:
+ printf("Unloading rdma_addr.\n");
+ addr_cleanup();
+ break;
+ case MOD_SHUTDOWN:
+ break;
+ default:
+ err = EOPNOTSUPP;
+ break;
+ }
+
+ return (err);
}
-module_init(addr_init);
-module_exit(addr_cleanup);
+static moduledata_t mod_data = {
+ "rdma_addr",
+ addr_load,
+ 0
+};
+
+MODULE_VERSION(rdma_addr, 1);
+DECLARE_MODULE(rdma_addr, mod_data, SI_SUB_EXEC, SI_ORDER_ANY);
More information about the p4-projects
mailing list