Infiniband loopback fails to connect

James Pan jiaming.pan at yahoo.com
Tue Oct 28 09:39:38 UTC 2014


Jason,

Thank you for your reply, the HCA I am using is Mellanox MT26428, my OS is FreeBSD 10.0, below is the detail.
Please let me know if you need more information. Thanks!

root at rd1:~ # ibstat 
CA 'mlx4_0'
	CA type: MT26428
	Number of ports: 2
	Firmware version: 2.8.0
	Hardware version: b0
	Node GUID: 0x0002c9030007f6e2
	System image GUID: 0x0002c9030007f6e5
	Port 1:
		State: Down
		Physical state: Polling
		Rate: 10
		Base lid: 0
		LMC: 0
		SM lid: 0
		Capability mask: 0x02510868
		Port GUID: 0x0002c9030007f6e3
	Port 2:
		State: Down
		Physical state: Polling
		Rate: 10
		Base lid: 0
		LMC: 0
		SM lid: 0
		Capability mask: 0x02510868
		Port GUID: 0x0002c9030007f6e4
root at rd1:~ # uname -a
FreeBSD rd1 10.0-RELEASE FreeBSD 10.0-RELEASE #23 fba8f85(master)-dirty: Fri Oct 24 04:08:09 CST 2014     root at rd1:/usr/obj/usr/src/sys/GENERIC  amd64
root at rd1:~ # 

Best regards,

James Pan

> 在 2014年10月27日,下午9:58,Jason Bacon <jwbacon at tds.net> 写道:
> 
> 
> Jim,
> 
> Thanks much for your efforts on this.  I'm sure the Mellanox developers will look into it as they're actively working on an overhaul of the code right now.
> 
> In the meantime, you might want to post more info, like what HCA you're using.
> 
> Cheers,
> 
>    Jason
> 
> On 10/27/14 6:39 AM, James Pan via freebsd-infiniband wrote:
>> Hi,
>> I’ve configured Inifniband on two FreeBSD machines (rd1 and rd2), the IB interface appears as ib0 on both machines,
>> I assigned 10.9.0.1 to ib0 on rd1 and 10.9.0.2 to ib0 on rd2.
>> Then I downloaded a sample code from http://thegeekinthecorner.wordpress.com and modified them a little, the code could be found as attached.
>> 
>> The problem I met is:
>> if I run the server on rd2 and then run the client from rd1, the program works as expected.
>> if I run both the server and the client on the same host, the client will fail to connect to the server:
>> 
>> root at rd2:~/the-geek-in-the-corner/01_basic-client-server # ./server
>> listening on port 21277.
>> root at rd2:~/the-geek-in-the-corner/01_basic-client-server # ./client 10.9.0.2 21277
>> event 1, status -60
>> on_event: unknown event.
>> 
>> It looks like rdma_resolve_addr() has failed.
>> 
>> I did some debugging on the driver and the cause seems to be:
>> In the infiniband driver addr_resolve() (in file /usr/src/sys/ofed/driers/infiniband/core/addr.c)
>> depends on arpresolve() to resolve the address but unfortunately the ifp passed to
>> arpresolve() is lo0, because the ifp is got from the route table while route table on rd2 is:
>> 
>> root at rd2:~/the-geek-in-the-corner/01_basic-client-server # netstat -rn
>> Routing tables
>> 
>> Internet:
>> Destination        Gateway            Flags    Refs      Use  Netif Expire
>> default            192.168.1.1        UGS         0        0    ix0
>> 10.9.0.0/24        link#5             U           0        1    ib0
>> 10.9.0.2           link#5             UHS         0        0    lo0 <-----------------
>> 127.0.0.1          link#4             UH          0        0    lo0
>> 
>> as the destination ip address is 10.9.0.2,  lo0 is found and passed to arpresolve()
>> lo0 doesn’t have an address so it fails.
>> 
>> I made some changes to the driver and pass the correct ifp to addr_resolve(), this time addr_resolve() passed but rmda_resolve_route() failed.
>> 
>> Could anyone who has experience with infiniband programming help take a look?
>> Your help is very appreciated, thanks a lot!
>> 
>> 
>> 
>> ------------------------------ the sample code --------------------------------
>> 
>> root at rd2:~/the-geek-in-the-corner/01_basic-client-server # cat server.c
>> #include <stdio.h>
>> #include <stdlib.h>
>> #include <string.h>
>> #include <unistd.h>
>> #include <rdma/rdma_cma.h>
>> 
>> #define TEST_NZ(x) do { if ( (x)) die("error: " #x " failed (returned non-zero)." ); } while (0)
>> #define TEST_Z(x)  do { if (!(x)) die("error: " #x " failed (returned zero/null)."); } while (0)
>> 
>> const int BUFFER_SIZE = 1024;
>> 
>> struct context {
>>   struct ibv_context *ctx;
>>   struct ibv_pd *pd;
>>   struct ibv_cq *cq;
>>   struct ibv_comp_channel *comp_channel;
>> 
>>   pthread_t cq_poller_thread;
>> };
>> 
>> struct connection {
>>   struct ibv_qp *qp;
>> 
>>   struct ibv_mr *recv_mr;
>>   struct ibv_mr *send_mr;
>> 
>>   char *recv_region;
>>   char *send_region;
>> };
>> 
>> static void die(const char *reason);
>> 
>> static void build_context(struct ibv_context *verbs);
>> static void build_qp_attr(struct ibv_qp_init_attr *qp_attr);
>> static void * poll_cq(void *);
>> static void post_receives(struct connection *conn);
>> static void register_memory(struct connection *conn);
>> 
>> static void on_completion(struct ibv_wc *wc);
>> static int on_connect_request(struct rdma_cm_id *id);
>> static int on_connection(void *context);
>> static int on_disconnect(struct rdma_cm_id *id);
>> static int on_event(struct rdma_cm_event *event);
>> 
>> static struct context *s_ctx = NULL;
>> 
>> int main(int argc, char **argv)
>> {
>>   struct sockaddr_in addr;
>>   struct rdma_cm_event *event = NULL;
>>   struct rdma_cm_id *listener = NULL;
>>   struct rdma_event_channel *ec = NULL;
>>   uint16_t port = 0;
>> 
>>   memset(&addr, 0, sizeof(addr));
>>   addr.sin_family = AF_INET;
>>   addr.sin_len = sizeof addr;
>> 
>>   TEST_Z(ec = rdma_create_event_channel());
>>   TEST_NZ(rdma_create_id(ec, &listener, NULL, RDMA_PS_TCP));
>>   return 0;
>> }
>> 
>> int on_disconnect(struct rdma_cm_id *id)
>> {
>>   struct connection *conn = (struct connection *)id->context;
>> 
>>   printf("peer disconnected.\n");
>> 
>>   rdma_destroy_qp(id);
>> 
>>   ibv_dereg_mr(conn->send_mr);
>>   ibv_dereg_mr(conn->recv_mr);
>> 
>>   free(conn->send_region);
>>   free(conn->recv_region);
>> 
>>   free(conn);
>> 
>>   rdma_destroy_id(id);
>> 
>>   return 0;
>> }
>> 
>> int on_event(struct rdma_cm_event *event)
>> {
>>   int r = 0;
>> 
>>   if (event->event == RDMA_CM_EVENT_CONNECT_REQUEST)
>>     r = on_connect_request(event->id);
>>   else if (event->event == RDMA_CM_EVENT_ESTABLISHED)
>>     r = on_connection(event->id->context);
>>   else if (event->event == RDMA_CM_EVENT_DISCONNECTED)
>>     r = on_disconnect(event->id);
>>   else
>>     die("on_event: unknown event.");
>> 
>>   return r;
>> }
>> 
>> 
>> root at rd2:~/the-geek-in-the-corner/01_basic-client-server # cat client.c
>> #include <netdb.h>
>> #include <stdio.h>
>> #include <stdlib.h>
>> #include <string.h>
>> #include <unistd.h>
>> #include <rdma/rdma_cma.h>
>> 
>> #define TEST_NZ(x) do { if ( (x)) die("error: " #x " failed (returned non-zero)." ); } while (0)
>> #define TEST_Z(x)  do { if (!(x)) die("error: " #x " failed (returned zero/null)."); } while (0)
>> 
>> const int BUFFER_SIZE = 1024;
>> const int TIMEOUT_IN_MS = 500; /* ms */
>> 
>> struct context {
>>   struct ibv_context *ctx;
>>   struct ibv_pd *pd;
>>   struct ibv_cq *cq;
>>   struct ibv_comp_channel *comp_channel;
>> 
>>   pthread_t cq_poller_thread;
>> };
>> 
>> struct connection {
>>   struct rdma_cm_id *id;
>>   struct ibv_qp *qp;
>> 
>>   struct ibv_mr *recv_mr;
>>   struct ibv_mr *send_mr;
>> 
>>   char *recv_region;
>>   char *send_region;
>> 
>>   int num_completions;
>> };
>> 
>> static void die(const char *reason);
>> 
>> static void build_context(struct ibv_context *verbs);
>> static void build_qp_attr(struct ibv_qp_init_attr *qp_attr);
>> static void * poll_cq(void *);
>> static void post_receives(struct connection *conn);
>> static void register_memory(struct connection *conn);
>> 
>> static int on_addr_resolved(struct rdma_cm_id *id);
>> static void on_completion(struct ibv_wc *wc);
>> static int on_connection(void *context);
>> static int on_disconnect(struct rdma_cm_id *id);
>> static int on_event(struct rdma_cm_event *event);
>> static int on_route_resolved(struct rdma_cm_id *id);
>> 
>> static struct context *s_ctx = NULL;
>> 
>> int main(int argc, char **argv)
>> {
>>   struct addrinfo *addr;
>>   struct rdma_cm_event *event = NULL;
>>   struct rdma_cm_id *conn= NULL;
>>   struct rdma_event_channel *ec = NULL;
>> 
>>   if (argc != 3)
>>     die("usage: client <server-address> <server-port>");
>> 
>>   TEST_NZ(getaddrinfo(argv[1], argv[2], NULL, &addr));
>> 
>>   TEST_Z(ec = rdma_create_event_channel());
>>   TEST_NZ(rdma_create_id(ec, &conn, NULL, RDMA_PS_TCP));
>>   TEST_NZ(rdma_resolve_addr(conn, NULL, addr->ai_addr, TIMEOUT_IN_MS));
>> 
>>   freeaddrinfo(addr);
>> 
>>   while (rdma_get_cm_event(ec, &event) == 0) {
>>     struct rdma_cm_event event_copy;
>> 
>>     memcpy(&event_copy, event, sizeof(*event));
>>     rdma_ack_cm_event(event);
>> 
>>     if (on_event(&event_copy))
>>       break;
>>   }
>>   free(conn->recv_region);
>> 
>>   free(conn);
>> 
>>   rdma_destroy_id(id);
>> 
>>   return 1; /* exit event loop */
>> }
>> 
>> int on_event(struct rdma_cm_event *event)
>> {
>>   int r = 0;
>> 
>>   if (event->event == RDMA_CM_EVENT_ADDR_RESOLVED)
>>     r = on_addr_resolved(event->id);
>>   else if (event->event == RDMA_CM_EVENT_ROUTE_RESOLVED)
>>     r = on_route_resolved(event->id);
>>   else if (event->event == RDMA_CM_EVENT_ESTABLISHED)
>>     r = on_connection(event->id->context);
>>   else if (event->event == RDMA_CM_EVENT_DISCONNECTED)
>>     r = on_disconnect(event->id);
>>   else {
>>     printf("event %d, status %d\n", event->event, event->status);
>>     die("on_event: unknown event.");
>>   }
>> 
>>   return r;
>> }
>> 
>> int on_route_resolved(struct rdma_cm_id *id)
>> {
>>   struct rdma_conn_param cm_params;
>> 
>>   printf("route resolved.\n");
>> 
>>   memset(&cm_params, 0, sizeof(cm_params));
>>   TEST_NZ(rdma_connect(id, &cm_params));
>> 
>>   return 0;
>> }
>> 
>>  _______________________________________________
>> freebsd-infiniband at freebsd.org mailing list
>> http://lists.freebsd.org/mailman/listinfo/freebsd-infiniband
>> To unsubscribe, send any mail to "freebsd-infiniband-unsubscribe at freebsd.org"
> 
> 
> -- 
> ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
>  Jason W. Bacon
>  jwbacon at tds.net
> 
>  Circumstances don't make a man:
>  They reveal him.
> ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
> 



More information about the freebsd-infiniband mailing list