prov/efa: Implement the rma interface

Rename efa_dgram_rma.c to efa_rma.c and move it to prov/efa/src as a common RMA interface for both rdm and dgram ep type. Update that dgram does not support rma. Implement rdma write and inject. Support inline rdma write. Signed-off-by: Jessie Yang <[email protected]>
ofiwg · Nov 23, 2024 · f12f5ea · f12f5ea
1 parent d5d8694
commit f12f5ea
Show file tree

Hide file tree

Showing 6 changed files with 421 additions and 157 deletions.
diff --git a/libfabric.vcxproj b/libfabric.vcxproj
@@ -885,9 +885,9 @@
     <ClCompile Include="prov\efa\src\efa_env.c" />
     <ClCompile Include="prov\efa\src\efa_cntr.c" />
     <ClCompile Include="prov\efa\src\efa_msg.c" />
+    <ClCompile Include="prov\efa\src\efa_rma.c" />
     <ClCompile Include="prov\efa\src\dgram\efa_dgram_ep.c" />
     <ClCompile Include="prov\efa\src\dgram\efa_dgram_cq.c" />
-    <ClCompile Include="prov\efa\src\dgram\efa_dgram_rma.c" />
     <ClCompile Include="prov\efa\src\rdm\efa_rdm_ope.c" />
     <ClCompile Include="prov\efa\src\rdm\efa_rdm_rxe_map.c" />
     <ClCompile Include="prov\efa\src\rdm\efa_rdm_atomic.c" />

diff --git a/prov/efa/Makefile.include b/prov/efa/Makefile.include
@@ -48,9 +48,9 @@ _efa_files = \
 	prov/efa/src/efa_env.c	\
 	prov/efa/src/efa_cntr.c	\
 	prov/efa/src/efa_msg.c \
+	prov/efa/src/efa_rma.c \
 	prov/efa/src/dgram/efa_dgram_ep.c \
 	prov/efa/src/dgram/efa_dgram_cq.c \
-	prov/efa/src/dgram/efa_dgram_rma.c \
 	prov/efa/src/rdm/efa_rdm_peer.c	\
 	prov/efa/src/rdm/efa_rdm_cq.c \
 	prov/efa/src/rdm/efa_rdm_ep_utils.c \

diff --git a/prov/efa/src/dgram/efa_dgram_rma.c b/prov/efa/src/dgram/efa_dgram_rma.c
diff --git a/prov/efa/src/efa_base_ep.c b/prov/efa/src/efa_base_ep.c
@@ -186,6 +186,12 @@ int efa_qp_create(struct efa_qp **qp, struct ibv_qp_init_attr_ex *init_attr_ex,
 					      init_attr_ex);
 	} else {
 		assert(init_attr_ex->qp_type == IBV_QPT_DRIVER);
+		if (efa_device_support_rdma_read())
+			init_attr_ex->send_ops_flags |= IBV_QP_EX_WITH_RDMA_READ;
+		if (efa_device_support_rdma_write()) {
+			init_attr_ex->send_ops_flags |= IBV_QP_EX_WITH_RDMA_WRITE;
+			init_attr_ex->send_ops_flags |= IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM;
+		}
 #if HAVE_CAPS_UNSOLICITED_WRITE_RECV
 		if (efa_rdm_use_unsolicited_write_recv())
 			efa_attr.flags |= EFADV_QP_FLAGS_UNSOLICITED_WRITE_RECV;
@@ -362,7 +368,9 @@ int efa_base_ep_construct(struct efa_base_ep *base_ep,
 	base_ep->max_msg_size = info->ep_attr->max_msg_size;
 	base_ep->max_rma_size = info->ep_attr->max_msg_size;
 	base_ep->inject_msg_size = info->tx_attr->inject_size;
-	base_ep->inject_rma_size = info->tx_attr->inject_size;
+	/* TODO: update inject_rma_size to inline size after firmware
+	 * supports inline rdma write */
+	base_ep->inject_rma_size = 0;
 	return 0;
 }