libfabric Initialization Process for Efficient Communication
Learn about the libfabric initialization process through code snippets showcasing device listing, address information setup, fabric and domain creation, event queue handling, endpoint setup, and more for optimized communication within applications.
Download Presentation
Please find below an Image/Link to download the presentation.
The content on the website is provided AS IS for your information and personal use only. It may not be sold, licensed, or shared on other websites without obtaining consent from the author. If you encounter any issues during the download, it is possible that the publisher has removed the file from their server.
You are allowed to download the files provided on this website for personal or commercial use, subject to the condition that they are used lawfully. All files are the property of their respective owners.
The content on the website is provided AS IS for your information and personal use only. It may not be sold, licensed, or shared on other websites without obtaining consent from the author.
E N D
Presentation Transcript
libfabric example rc_pingpong.c
Initialization libibverbs libfabric 547 struct ibv_device 661 dev_list = ibv_get_device_list(NULL); 139 static struct pingpong_dest *pp_client_exch_dest( ) 143 struct addrinfo hints = { 144 .ai_family = AF_UNSPEC, 145 .ai_socktype = SOCK_STREAM 146 }; 157 n = getaddrinfo(servername, service, &hints, &res); 373 struct ibv_qp_init_attr init_attr = { 382 .qp_type = IBV_QPT_RC 383 }; 384 385 ctx->qp = ibv_create_qp(ctx->pd, &init_attr); **dev_list; 421 struct 422 struct 423 struct 541 hints.type 542 hints.ep_cap 543 hints.addr_format 544 545 asprintf(&service, "%d", port); 546 if (!servername) { 547 hints.ep_cap |= FI_PASSIVE; 548 } else { 549 node = servername; 550 } 551 552 rc = fi_getinfo(node, service, flags, &hints, &prov_list); 553 if (rc) { 554 FI_ERR_LOG("fi_getinfo", rc); 555 return 1; 556 } fi_info fi_info fi_info *prov_list; *prov; hints; = FID_MSG; = FI_MSG; = FI_SOCKADDR;
Initialization Domain, MR libibverbs libfabric 336 ctx->context = ibv_open_device(ib_dev); 325 rc = fi_fabric(prov->fabric_name, 0, &ctx->fabric, NULL); 248 rc = fi_fdomain(ctx->fabric, ctx->prov, &ctx->dom, NULL); 352 ctx->pd = ibv_alloc_pd(ctx->context); 358 ctx->mr = ibv_reg_mr(ctx->pd, ctx->buf, size, IBV_ACCESS_LOCAL_WRITE); 256 rc = fi_mr_reg(ctx->dom, ctx->buf, ctx->size, FI_SEND | FI_RECV, 0, 0, 0, &ctx->mr, NULL);
Initialization Event Queue libibverbs libfabric 343 if (use_event) { 344 ctx->channel = ibv_create_comp_channel(ctx- >context); 349 } else 350 ctx->channel = NULL; 364 ctx->cq = ibv_create_cq(ctx->context, rx_depth + 1, NULL, ctx->channel, 0); 113 struct fi_eq_attr cq_attr; 120 cq_attr.format 121 if (ctx->use_event) { 122 cq_attr.wait_obj 124 } else { 125 cq_attr.wait_obj 126 } 127 cq_attr.wait_cond 128 cq_attr.size 129 130 rc = fi_eq_open(ctx->dom, &cq_attr, &ctx->cq, NULL); = FI_EQ_FORMAT_CONTEXT; = FI_WAIT_FD; = FI_WAIT_NONE; = FI_EQ_COND_NONE; = ctx->rx_depth + 1;
Initialization Endpoint libibverbs libfabric 372 struct ibv_qp_attr attr; 373 struct ibv_qp_init_attr init_attr = { 374 .send_cq = ctx->cq, 375 .recv_cq = ctx->cq, 376 .cap = { 377 .max_send_wr 378 .max_recv_wr 379 .max_send_sge = 1, 380 .max_recv_sge = 1 381 }, 382 .qp_type = IBV_QPT_RC 383 }; 384 385 ctx->qp = ibv_create_qp(ctx->pd, &init_attr); 209 rc = fi_endpoint(ctx->dom, entry.info, &ctx->ep, NULL); 277 rc = fi_bind(&ctx->ep->fid, &ctx->cq->fid, FI_SEND | FI_RECV); = 1, = rx_depth,
Initialization contd. libibverbs libfabric 139 static struct pingpong_dest *pp_client_exch_dest( ) 168 if (!connect(sockfd, t->ai_addr, t- >ai_addrlen)) 183 gid_to_wire_gid(&my_dest->gid, gid); 184 sprintf(msg, "%04x:%06x:%06x:%s", my_dest->lid, my_dest->qpn, my_dest->psn, gid); 186 if (write(sockfd, msg, sizeof msg) != sizeof msg) { 191 if (read(sockfd, msg, sizeof msg) != sizeof msg) { 385 ctx->qp = ibv_create_qp(ctx->pd, &init_attr); 107 if (ibv_modify_qp(ctx->qp, &attr, 285 rc = fi_connect(ctx->ep, ctx->prov->dest_addr, NULL, 0);
Post recv libibverbs libfabric 485 static int pp_post_recv( ) 487 struct ibv_sge list = { 488 .addr 489 .length = ctx->size, 490 .lkey 491 }; 492 struct ibv_recv_wr wr = { 493 .wr_id 494 .sg_list 495 .num_sge 496 }; 497 struct ibv_recv_wr *bad_wr; 499 501 if (ibv_post_recv(ctx->qp, &wr, &bad_wr)) 368 static int pp_post_recv( ) 375 rc = fi_recv(ctx->ep, ctx->buf, ctx->size, fi_mr_desc(ctx->mr), (void *)(uintptr_t)PINGPONG_RECV_WCID); = (uintptr_t) ctx->buf, = ctx->mr->lkey = PINGPONG_RECV_WRID, = &list, = 1,
Post send libibverbs libfabric 507 static int pp_post_send( ) 509 struct ibv_sge list = { 510 .addr 511 .length = ctx->size, 512 .lkey 513 }; 514 struct ibv_send_wr wr = { 515 .wr_id 516 .sg_list 517 .num_sge 518 .opcode 519 .send_flags = ctx->send_flags, 520 }; 521 struct ibv_send_wr *bad_wr; 522 523 return ibv_post_send(ctx->qp, &wr, &bad_wr); 386 static int pp_post_send( ) 390 rc = fi_send(ctx->ep, ctx->buf, ctx->size, fi_mr_desc(ctx->mr), (void *)(uintptr_t)PINGPONG_SEND_WCID); = (uintptr_t) ctx->buf, = ctx->mr->lkey = PINGPONG_SEND_WRID, = &list, = 1, = IBV_WR_SEND,
Reading completion events libibverbs libfabric 763 while (rcnt < iters || scnt < iters) { 764 if (use_event) { 765 struct ibv_cq *ev_cq; 766 void *ev_ctx; 767 768 if (ibv_get_cq_event(ctx->channel, &ev_cq, &ev_ctx)) { 772 780 if (ibv_req_notify_cq(ctx->cq, 0)) { 787 struct ibv_wc wc[2]; 788 int ne, i; 789 790 do { 791 ne = ibv_poll_cq(ctx->cq, 2, wc); 792 if (ne < 0) { 793 fprintf(stderr, "poll CQ failed %d\n", ne); 794 return 1; 795 } 796 797 } while (!use_event && ne < 1); switch ((int) wc[i].wr_id) { 616 while (rcnt < iters || scnt < iters) { 617 struct fi_eq_entry 621 if (use_event) { 622 /* Blocking read */ 623 rd = fi_eq_condread(ctx->cq, wc, sizeof wc, NULL); 624 } else { 625 do { 626 rd = fi_eq_read(ctx->cq, wc, sizeof wc); 627 } while (rd == 0); 628 } wc[2]; 642 switch ((int)(uintptr_t)wc[i].op_context) {
