ionic: Add support for PHC timestamping Support IBV_WC_EX_WITH_COMPLETION_TIMESTAMP and IBV_WC_EX_WITH_COMPLETION_TIMESTAMP_WALLCLOCK. Modify the polling path to extract timestamps from the CQE. For Send Queue completions, cache the timestamp in the request metadata (`sq_meta`) which allows the driver to propagate the correct hardware timestamp to all Work Completions covered by a coalesced (MSN) completion event. Update `start_poll` to process one completion at a time and store the current timestamp in the `vcq` context. This ensures that the `read_completion_ts` and `read_completion_wallclock_ns` accessors return the correct value for the specific Work Completion currently being inspected. Signed-off-by: Allen Hubbe <allen.hubbe@amd.com> Signed-off-by: Abhijit Gangurde <abhijit.gangurde@amd.com>
diff --git a/providers/ionic/ionic.h b/providers/ionic/ionic.h index 2ac0959..cb1a8da 100644 --- a/providers/ionic/ionic.h +++ b/providers/ionic/ionic.h
@@ -54,7 +54,9 @@ IBV_WC_EX_WITH_SRC_QP | IBV_WC_EX_WITH_SLID | IBV_WC_EX_WITH_SL | - IBV_WC_EX_WITH_DLID_PATH_BITS + IBV_WC_EX_WITH_DLID_PATH_BITS | + IBV_WC_EX_WITH_COMPLETION_TIMESTAMP | + IBV_WC_EX_WITH_COMPLETION_TIMESTAMP_WALLCLOCK }; enum { @@ -143,6 +145,7 @@ struct list_head poll_sq; struct list_head poll_rq; bool flush; + bool do_timestamp; struct list_head flush_sq; struct list_head flush_rq; struct ionic_queue q; @@ -154,6 +157,16 @@ int reserve_pending; uint16_t arm_any_prod; uint16_t arm_sol_prod; + uint64_t phc_tick; +}; + +struct ionic_phc { + uint64_t mask; + uint64_t cycles; + uint64_t nsec; + uint64_t frac; + uint32_t mult; + uint32_t shift; }; struct ionic_vcq { @@ -161,11 +174,16 @@ struct ionic_cq cq[2]; uint8_t udma_mask; uint8_t poll_idx; + bool phc_update; + bool cur_wc_pending; struct ibv_wc cur_wc; /* for use with start_poll/next_poll */ + uint64_t cur_wc_timestamp; + struct ionic_phc phc; }; struct ionic_sq_meta { uint64_t wrid; + uint64_t cqe_timestamp; uint32_t len; uint16_t seq; uint8_t ibop;
diff --git a/providers/ionic/ionic_fw_types.h b/providers/ionic/ionic_fw_types.h index 1546ff8..b57f7f9 100644 --- a/providers/ionic/ionic_fw_types.h +++ b/providers/ionic/ionic_fw_types.h
@@ -74,7 +74,7 @@ struct ionic_v1_cqe { union { struct { - __le64 wqe_idx; + __le64 wqe_idx_timestamp; __be32 src_qpn_op; __u8 src_mac[6]; __be16 vlan_tag; @@ -84,16 +84,17 @@ __u8 rsvd[4]; __be32 msg_msn; __u8 rsvd2[8]; - __le64 npg_wqe_idx; + __le64 npg_wqe_idx_timestamp; } send; }; __be32 status_length; __be32 qid_type_flags; }; -/* bits for cqe wqe_idx */ -enum ionic_v1_cqe_wqe_idx_bits { +/* bits for cqe wqe_idx and timestamp */ +enum ionic_v1_cqe_wqe_idx_timestamp_bits { IONIC_V1_CQE_WQE_IDX_MASK = 0xffff, + IONIC_V1_CQE_TIMESTAMP_SHIFT = 16, }; /* bits for cqe recv */
diff --git a/providers/ionic/ionic_verbs.c b/providers/ionic/ionic_verbs.c index 53d22e9..1aa08f5 100644 --- a/providers/ionic/ionic_verbs.c +++ b/providers/ionic/ionic_verbs.c
@@ -167,6 +167,38 @@ #define IONIC_OP(version, opname) \ ((version) < 2 ? IONIC_V1_OP_##opname : IONIC_V2_OP_##opname) +static int ionic_phc_update(struct ionic_phc *phc, struct ib_uverbs_clock_info *state) +{ + uint32_t sign, *state_sign; + int retry; + + state_sign = &state->sign; + + for (retry = 10; true; --retry) { + if (!retry) + return EBUSY; + + sign = atomic_load((_Atomic(uint32_t) *)state_sign); + + /* odd sign means state is updating */ + if (sign & 1) + continue; + + phc->mask = state->mask; + phc->cycles = state->cycles; + phc->nsec = state->nsec; + phc->frac = state->frac; + phc->mult = state->mult; + phc->shift = state->shift; + + /* same sign means state did not update */ + if (sign == atomic_load((_Atomic(uint32_t) *)state_sign)) + break; + } + + return 0; +} + static int ionic_query_device_ex(struct ibv_context *ibctx, const struct ibv_query_device_ex_input *input, struct ibv_device_attr_ex *ex, @@ -174,6 +206,7 @@ { struct ibv_device_attr *dev_attr = &ex->orig_attr; struct ib_uverbs_ex_query_device_resp resp = {}; + struct ionic_ctx *ctx = to_ionic_ctx(ibctx); size_t resp_size = sizeof(resp); int rc; @@ -186,6 +219,24 @@ if (rc < 0) dev_attr->fw_ver[0] = 0; + if (ctx->phc_state) { + struct ionic_phc phc = {}; + + rc = ionic_phc_update(&phc, ctx->phc_state); + if (!rc) { + const uint64_t ns_per_ms = 1000000; + const uint32_t shift = phc.shift; + const uint32_t mult = phc.mult; + + if (ex_size > offsetof(typeof(*ex), completion_timestamp_mask)) + ex->completion_timestamp_mask = phc.mask; + + if (ex_size > offsetof(typeof(*ex), hca_core_clock)) + ex->hca_core_clock = + ((ns_per_ms << shift) + (mult >> 1)) / mult; + } + } + return 0; } @@ -480,6 +531,25 @@ ionic_queue_dbell_init(&cq->q, cq->cqid); } +static uint64_t ionic_phc_ts_to_ns(struct ionic_phc *phc, uint64_t ts) +{ + uint64_t dt, dt_sign, ns; + + dt = (ts - phc->cycles) & phc->mask; + dt_sign = phc->mask ^ (phc->mask >> 1); + + ns = phc->nsec; + + if (dt & dt_sign) { + dt = -dt & phc->mask; + ns -= ((dt * phc->mult) - phc->frac) >> phc->shift; + } else { + ns += ((dt * phc->mult) + phc->frac) >> phc->shift; + } + + return ns; +} + /* * NOTE: ionic_start_poll, ionic_next_poll and ionic_end_poll provide a * minimal implementations of the ibv_cq_ex polling mechanism, sufficient to @@ -491,14 +561,34 @@ { struct ibv_cq *ibcq = ibv_cq_ex_to_cq(ibcq_ex); struct ionic_vcq *vcq = to_ionic_vcq(ibcq); + int rc; - int rc = ionic_poll_cq(ibcq, 1, &vcq->cur_wc); + if (!vcq->cur_wc_pending) { + rc = ionic_poll_cq(ibcq, 1, &vcq->cur_wc); - if (rc != 1) /* no completions ready or poll failed */ - return (rc == 0) ? ENOENT : rc; + if (rc != 1) /* no completions ready or poll failed */ + return (rc == 0) ? ENOENT : rc; - ibcq_ex->wr_id = vcq->cur_wc.wr_id; - ibcq_ex->status = vcq->cur_wc.status; + ibcq_ex->wr_id = vcq->cur_wc.wr_id; + ibcq_ex->status = vcq->cur_wc.status; + + /* if there is an error after polling the cur_wc, + * don't replace it in the next call to start. + */ + vcq->cur_wc_pending = true; + } + + if (vcq->phc_update) { + struct ionic_ctx *ctx = to_ionic_ctx(ibcq->context); + + rc = ionic_phc_update(&vcq->phc, ctx->phc_state); + if (rc) + return rc; + } + + /* success, so the next call to start should replace cur_wc */ + vcq->cur_wc_pending = false; + return 0; } @@ -592,6 +682,22 @@ return vcq->cur_wc.dlid_path_bits; } +static uint64_t ionic_wc_read_completion_ts(struct ibv_cq_ex *ibcq_ex) +{ + struct ibv_cq *ibcq = ibv_cq_ex_to_cq(ibcq_ex); + struct ionic_vcq *vcq = to_ionic_vcq(ibcq); + + return vcq->cur_wc_timestamp; +} + +static uint64_t ionic_wc_read_completion_wallclock_ns(struct ibv_cq_ex *ibcq_ex) +{ + struct ibv_cq *ibcq = ibv_cq_ex_to_cq(ibcq_ex); + struct ionic_vcq *vcq = to_ionic_vcq(ibcq); + + return ionic_phc_ts_to_ns(&vcq->phc, vcq->cur_wc_timestamp); +} + static struct ibv_cq_ex *ionic_create_cq_ex(struct ibv_context *ibctx, struct ibv_cq_init_attr_ex *ex) { @@ -677,6 +783,23 @@ vcq->vcq.cq_ex.read_slid = ionic_wc_read_slid; if (ex->wc_flags & IBV_WC_EX_WITH_DLID_PATH_BITS) vcq->vcq.cq_ex.read_dlid_path_bits = ionic_wc_read_dlid_path_bits; + if (ex->wc_flags & IBV_WC_EX_WITH_COMPLETION_TIMESTAMP) { + if (!ctx->phc_state) { + rc = EOPNOTSUPP; + goto err_udma; + } + vcq->vcq.cq_ex.read_completion_ts = ionic_wc_read_completion_ts; + } + if (ex->wc_flags & IBV_WC_EX_WITH_COMPLETION_TIMESTAMP_WALLCLOCK) { + if (!ctx->phc_state) { + rc = EOPNOTSUPP; + goto err_udma; + } + + vcq->phc_update = true; + vcq->vcq.cq_ex.read_completion_wallclock_ns = + ionic_wc_read_completion_wallclock_ns; + } return &vcq->vcq.cq_ex; @@ -847,6 +970,7 @@ { struct ionic_qp *qp = NULL; struct ionic_rq_meta *meta; + uint64_t wqe_idx_timestamp; uint16_t vlan_tag, wqe_idx; uint32_t src_qpn, st_len; uint8_t op; @@ -875,7 +999,8 @@ return -EIO; } - wqe_idx = le64toh(cqe->recv.wqe_idx) & IONIC_V1_CQE_WQE_IDX_MASK; + wqe_idx_timestamp = le64toh(cqe->recv.wqe_idx_timestamp); + wqe_idx = wqe_idx_timestamp & IONIC_V1_CQE_WQE_IDX_MASK; /* wqe_idx must be a valid queue index */ if (unlikely(wqe_idx >> qp->rq.queue.depth_log2)) { @@ -896,6 +1021,7 @@ meta->next = qp->rq.meta_head; qp->rq.meta_head = meta; + cq->vcq->cur_wc_timestamp = wqe_idx_timestamp >> IONIC_V1_CQE_TIMESTAMP_SHIFT; memset(wc, 0, sizeof(*wc)); @@ -1022,6 +1148,8 @@ /* produce wc only if signaled or error status */ } while (!meta->signal && meta->ibsts == IBV_WC_SUCCESS); + cq->vcq->cur_wc_timestamp = meta->cqe_timestamp; + memset(wc, 0, sizeof(*wc)); wc->status = meta->ibsts; @@ -1085,6 +1213,7 @@ { struct ionic_sq_meta *meta; uint16_t cqe_seq, cqe_idx; + uint64_t timestamp; int rc; if (qp->sq.flush) @@ -1106,6 +1235,16 @@ return rc; } + timestamp = le64toh(cqe->send.npg_wqe_idx_timestamp) >> IONIC_V1_CQE_TIMESTAMP_SHIFT; + + for (cqe_idx = qp->sq.msn_cons; + cqe_idx != cqe_seq; + cqe_idx = ionic_queue_next(&qp->sq.queue, cqe_idx)) { + meta = &qp->sq.meta[cqe_idx]; + if (meta->remote) + meta->cqe_timestamp = timestamp; + } + qp->sq.msn_cons = cqe_seq; if (ionic_v1_cqe_error(cqe)) { @@ -1131,6 +1270,7 @@ struct ionic_v1_cqe *cqe) { struct ionic_sq_meta *meta; + uint64_t wqe_idx_timestamp; uint16_t wqe_idx; uint32_t st_len; @@ -1152,9 +1292,11 @@ return 0; } - wqe_idx = le64toh(cqe->send.npg_wqe_idx) & qp->sq.queue.mask; + wqe_idx_timestamp = le64toh(cqe->send.npg_wqe_idx_timestamp); + wqe_idx = wqe_idx_timestamp & qp->sq.queue.mask; meta = &qp->sq.meta[wqe_idx]; meta->local_comp = true; + meta->cqe_timestamp = wqe_idx_timestamp >> IONIC_V1_CQE_TIMESTAMP_SHIFT; if (ionic_v1_cqe_error(cqe)) { struct ionic_cq *cq =