efa: Optimize WR memset on WR creation

Since the WQE size (64 or 128 bytes) is a runtime variable, the compiler
cannot inline memset and emits a function call instead. This adds
function call overhead and multiple instructions to the post send hot
path.

Replace with an explicit u64 zeroing loop. The compiler recognizes the
small iteration count (8 or 16) and emits inlined stores, avoiding the
function call and the overhead because of it.

Measured with perf stat on osu_bibw (2x p4d.24xlarge,
FI_EFA_USE_DEVICE_RDMA=0):
Before: 7.93B cycles (memset function call)
After:  7.42B cycles (inlined loop)

Signed-off-by: Yonatan Nachum <ynachum@amazon.com>

providers/efa/verbs.c

diff --git a/providers/efa/verbs.c b/providers/efa/verbs.c
index f7dd9bb..ef951ca 100644
--- a/providers/efa/verbs.c
+++ b/providers/efa/verbs.c
@@ -2454,7 +2454,7 @@
 	uint32_t curbatch = 0;
 	uint8_t *inline_data;
 	struct efa_ah *ah;
-	int err = 0;
+	int i, err = 0;
 
 	switch (sq->wqe_size) {
 	case EFA_IO_TX_DESC_SIZE_64:
@@ -2486,7 +2486,9 @@
 			goto ring_db;
 		}
 
-		memset(wqe_buf, 0, sq->wqe_size);
+		for (i = 0; i < sq->wqe_size / sizeof(uint64_t); i++)
+			((uint64_t *)wqe_buf)[i] = 0;
+
 		ah = to_efa_ah(wr->wr.ud.ah);
 
 		if (wr->send_flags & IBV_SEND_INLINE) {
@@ -2546,7 +2548,7 @@
 static void *efa_send_wr_alloc(struct efa_qp *qp, struct ibv_qp_ex *ibvqpx)
 {
 	struct efa_sq *sq = &qp->sq;
-	int err;
+	int err, i;
 
 	if (unlikely(qp->wr_session_err))
 		return NULL;
@@ -2558,7 +2560,8 @@
 	}
 
 	sq->curr_tx_wqe.buff = sq->local_queue + sq->num_wqe_pending * sq->wqe_size;
-	memset(sq->curr_tx_wqe.buff, 0, sq->wqe_size);
+	for (i = 0; i < sq->wqe_size / sizeof(uint64_t); i++)
+		((uint64_t *)sq->curr_tx_wqe.buff)[i] = 0;
 
 	return sq->curr_tx_wqe.buff;
 }