PERFORCE change 129704 for review

[Steve Wise]

http://perforce.freebsd.org/chv.cgi?CH=129704

Change 129704 by swise at swise:vic10:iwarp on 2007/11/28 14:56:30

	Fixed cm_id refcounts to correctly work with the cv.

Affected files ...

.. //depot/projects/iwarp/sys/contrib/rdma/rdma_cma.c#6 edit

Differences ...

==== //depot/projects/iwarp/sys/contrib/rdma/rdma_cma.c#6 (text+ko) ====

@@ -89,7 +89,7 @@
 	struct ib_device	*device;
 	struct mtx		lock;
 	struct cv		comp;
-	atomic_t		refcount;
+	int			refcount;
 
 	LIST_HEAD(, rdma_id_private) id_list;
 	TAILQ_ENTRY(cma_device) list;
@@ -136,7 +136,7 @@
 	enum cma_state		state;
 	struct mtx		lock;
 	struct cv		comp;
-	atomic_t		refcount;
+	int			refcount;
 	struct cv		wait_remove;
 	atomic_t		dev_remove;
 
@@ -279,7 +279,9 @@
 static void cma_attach_to_dev(struct rdma_id_private *id_priv,
 			      struct cma_device *cma_dev)
 {
-	atomic_inc(&cma_dev->refcount);
+	mtx_lock(&cma_dev->lock);
+	cma_dev->refcount++;
+	mtx_unlock(&cma_dev->lock);
 	id_priv->cma_dev = cma_dev;
 	id_priv->id.device = cma_dev->device;
 	LIST_INSERT_HEAD(&cma_dev->id_list, id_priv, list);
@@ -288,7 +290,7 @@
 static inline void cma_deref_dev(struct cma_device *cma_dev)
 {
 	mtx_lock(&cma_dev->lock);
-	if (atomic_dec_and_test(&cma_dev->refcount))
+	if (--cma_dev->refcount == 0)
 		cv_broadcast(&cma_dev->comp);
 	mtx_unlock(&cma_dev->lock);
 }
@@ -365,8 +367,11 @@
 static void cma_deref_id(struct rdma_id_private *id_priv)
 {
 	mtx_lock(&id_priv->lock);
-	if (atomic_dec_and_test(&id_priv->refcount))
+	printf("%s id %p refcount %d\n", __FUNCTION__, id_priv, id_priv->refcount);
+	if (--id_priv->refcount == 0) {
+		printf("%s cv_bcast %p \n", __FUNCTION__, id_priv);
 		cv_broadcast(&id_priv->comp);
+	}
 	mtx_unlock(&id_priv->lock);
 }
 
@@ -414,7 +419,8 @@
 	id_priv->id.ps = ps;
 	mtx_init(&id_priv->lock, "rdma_cm_id_priv", NULL, MTX_DUPOK|MTX_DEF);
 	cv_init(&id_priv->comp, "rdma_cm_id_priv");
-	atomic_set(&id_priv->refcount, 1);
+	id_priv->refcount = 1;
+	printf("%s id %p refcount %d\n", __FUNCTION__, id_priv, id_priv->refcount);
 	cv_init(&id_priv->wait_remove, "id priv wait remove");
 	atomic_set(&id_priv->dev_remove, 0);
 	LIST_INIT(&id_priv->listen_list);
@@ -784,7 +790,8 @@
 
 	cma_deref_id(id_priv);
 	mtx_lock(&id_priv->lock);
-	cv_wait(&id_priv->comp, &id_priv->lock);
+	if (id_priv->refcount)
+		cv_wait_unlock(&id_priv->comp, &id_priv->lock);
 
 	free(id_priv, M_DEVBUF);
 }
@@ -886,12 +893,13 @@
 		cma_detach_from_dev(id_priv);
 	}
 	mtx_unlock(&lock);
-
 	cma_release_port(id_priv);
 	cma_deref_id(id_priv);
 	mtx_lock(&id_priv->lock);
-	cv_wait(&id_priv->comp, &id_priv->lock);
-
+	BUG_ON(id_priv->refcount < 0);
+	if (id_priv->refcount)
+		cv_wait(&id_priv->comp, &id_priv->lock);
+	mtx_unlock(&id_priv->lock);
 	free(id_priv->id.route.path_rec, M_DEVBUF);
 	free(id_priv, M_DEVBUF);
 }
@@ -1687,7 +1695,10 @@
 	if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ROUTE_QUERY))
 		return (EINVAL);
 
-	atomic_inc(&id_priv->refcount);
+	mtx_lock(&id_priv->lock);
+	id_priv->refcount++;
+	printf("%s id %p refcount %d\n", __FUNCTION__, id_priv, id_priv->refcount);
+	mtx_unlock(&id_priv->lock);
 #ifdef IB_SUPPORTED
 	switch (rdma_node_get_transport(id->device->node_type)) {
 	case RDMA_TRANSPORT_IB:
@@ -1866,7 +1877,10 @@
 	if (!cma_comp_exch(id_priv, CMA_ADDR_BOUND, CMA_ADDR_QUERY))
 		return (EINVAL);
 
-	atomic_inc(&id_priv->refcount);
+	mtx_lock(&id_priv->lock);
+	id_priv->refcount++;
+	printf("%s id %p refcount %d\n", __FUNCTION__, id_priv, id_priv->refcount);
+	mtx_unlock(&id_priv->lock);
 	memcpy(&id->route.addr.dst_addr, dst_addr, ip_addr_size(dst_addr));
 	if (cma_any_addr(dst_addr))
 		ret = cma_resolve_loopback(id_priv);
@@ -2783,9 +2797,8 @@
 
 	cma_dev->device = device;
 
-	mtx_init(&lock, "cma_device", NULL, MTX_DEF);
 	cv_init(&cma_dev->comp, "cma_device");
-	atomic_set(&cma_dev->refcount, 1);
+	cma_dev->refcount = 1;
 	LIST_INIT(&cma_dev->id_list);
 	ib_set_client_data(device, &cma_client, cma_dev);
 
@@ -2834,7 +2847,10 @@
 		}
 
 		LIST_REMOVE(id_priv, list);
-		atomic_inc(&id_priv->refcount);
+		mtx_lock(&id_priv->lock);
+		id_priv->refcount++;
+	printf("%s id %p refcount %d\n", __FUNCTION__, id_priv, id_priv->refcount);
+		mtx_unlock(&id_priv->lock);
 		mtx_unlock(&lock);
 
 		ret = cma_remove_id_dev(id_priv);
@@ -2872,6 +2888,7 @@
 
 	LIST_INIT(&listen_any_list);
 	TAILQ_INIT(&dev_list);
+	mtx_init(&lock, "cma_device", NULL, MTX_DEF);
 
 	arc4rand(&next_port, sizeof next_port, 0);
 	next_port = ((unsigned int) next_port %