| /* | |
| * Copyright (c) 2005 Cisco Systems. All rights reserved. | |
| * | |
| * This software is available to you under a choice of one of two | |
| * licenses. You may choose to be licensed under the terms of the GNU | |
| * General Public License (GPL) Version 2, available from the file | |
| * COPYING in the main directory of this source tree, or the | |
| * OpenIB.org BSD license below: | |
| * | |
| * Redistribution and use in source and binary forms, with or | |
| * without modification, are permitted provided that the following | |
| * conditions are met: | |
| * | |
| * - Redistributions of source code must retain the above | |
| * copyright notice, this list of conditions and the following | |
| * disclaimer. | |
| * | |
| * - Redistributions in binary form must reproduce the above | |
| * copyright notice, this list of conditions and the following | |
| * disclaimer in the documentation and/or other materials | |
| * provided with the distribution. | |
| * | |
| * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
| * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
| * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
| * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
| * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
| * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
| * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
| * SOFTWARE. | |
| */ | |
| #include <config.h> | |
| #include <endian.h> | |
| #include <stdlib.h> | |
| #include <pthread.h> | |
| #include <string.h> | |
| #include "mthca.h" | |
| #include "doorbell.h" | |
| #include "wqe.h" | |
| static void *get_wqe(struct mthca_srq *srq, int n) | |
| { | |
| return srq->buf.buf + (n << srq->wqe_shift); | |
| } | |
| /* | |
| * Return a pointer to the location within a WQE that we're using as a | |
| * link when the WQE is in the free list. We use the imm field at an | |
| * offset of 12 bytes because in the Tavor case, posting a WQE may | |
| * overwrite the next segment of the previous WQE, but a receive WQE | |
| * will never touch the imm field. This avoids corrupting our free | |
| * list if the previous WQE has already completed and been put on the | |
| * free list when we post the next WQE. | |
| */ | |
| static inline int *wqe_to_link(void *wqe) | |
| { | |
| return (int *) (wqe + 12); | |
| } | |
| void mthca_free_srq_wqe(struct mthca_srq *srq, int ind) | |
| { | |
| struct mthca_next_seg *last_free; | |
| pthread_spin_lock(&srq->lock); | |
| last_free = get_wqe(srq, srq->last_free); | |
| *wqe_to_link(last_free) = ind; | |
| last_free->nda_op = htobe32((ind << srq->wqe_shift) | 1); | |
| *wqe_to_link(get_wqe(srq, ind)) = -1; | |
| srq->last_free = ind; | |
| pthread_spin_unlock(&srq->lock); | |
| } | |
| int mthca_tavor_post_srq_recv(struct ibv_srq *ibsrq, | |
| struct ibv_recv_wr *wr, | |
| struct ibv_recv_wr **bad_wr) | |
| { | |
| struct mthca_srq *srq = to_msrq(ibsrq); | |
| uint32_t doorbell[2]; | |
| int err = 0; | |
| int first_ind; | |
| int ind; | |
| int next_ind; | |
| int nreq; | |
| int i; | |
| void *wqe; | |
| void *prev_wqe; | |
| pthread_spin_lock(&srq->lock); | |
| first_ind = srq->first_free; | |
| for (nreq = 0; wr; wr = wr->next) { | |
| ind = srq->first_free; | |
| wqe = get_wqe(srq, ind); | |
| next_ind = *wqe_to_link(wqe); | |
| if (next_ind < 0) { | |
| err = -1; | |
| *bad_wr = wr; | |
| break; | |
| } | |
| prev_wqe = srq->last; | |
| srq->last = wqe; | |
| ((struct mthca_next_seg *) wqe)->ee_nds = 0; | |
| /* flags field will always remain 0 */ | |
| wqe += sizeof (struct mthca_next_seg); | |
| if (wr->num_sge > srq->max_gs) { | |
| err = -1; | |
| *bad_wr = wr; | |
| srq->last = prev_wqe; | |
| break; | |
| } | |
| for (i = 0; i < wr->num_sge; ++i) { | |
| ((struct mthca_data_seg *) wqe)->byte_count = | |
| htobe32(wr->sg_list[i].length); | |
| ((struct mthca_data_seg *) wqe)->lkey = | |
| htobe32(wr->sg_list[i].lkey); | |
| ((struct mthca_data_seg *) wqe)->addr = | |
| htobe64(wr->sg_list[i].addr); | |
| wqe += sizeof (struct mthca_data_seg); | |
| } | |
| if (i < srq->max_gs) { | |
| ((struct mthca_data_seg *) wqe)->byte_count = 0; | |
| ((struct mthca_data_seg *) wqe)->lkey = htobe32(MTHCA_INVAL_LKEY); | |
| ((struct mthca_data_seg *) wqe)->addr = 0; | |
| } | |
| ((struct mthca_next_seg *) prev_wqe)->ee_nds = | |
| htobe32(MTHCA_NEXT_DBD); | |
| srq->wrid[ind] = wr->wr_id; | |
| srq->first_free = next_ind; | |
| if (++nreq == MTHCA_TAVOR_MAX_WQES_PER_RECV_DB) { | |
| nreq = 0; | |
| doorbell[0] = first_ind << srq->wqe_shift; | |
| doorbell[1] = srq->srqn << 8; | |
| /* | |
| * Make sure that descriptors are written | |
| * before doorbell is rung. | |
| */ | |
| udma_to_device_barrier(); | |
| mthca_write64(doorbell, to_mctx(ibsrq->context)->uar + | |
| MTHCA_RECV_DOORBELL); | |
| first_ind = srq->first_free; | |
| } | |
| } | |
| if (nreq) { | |
| doorbell[0] = first_ind << srq->wqe_shift; | |
| doorbell[1] = (srq->srqn << 8) | nreq; | |
| /* | |
| * Make sure that descriptors are written before | |
| * doorbell is rung. | |
| */ | |
| udma_to_device_barrier(); | |
| mthca_write64(doorbell, to_mctx(ibsrq->context)->uar + | |
| MTHCA_RECV_DOORBELL); | |
| } | |
| pthread_spin_unlock(&srq->lock); | |
| return err; | |
| } | |
| int mthca_arbel_post_srq_recv(struct ibv_srq *ibsrq, | |
| struct ibv_recv_wr *wr, | |
| struct ibv_recv_wr **bad_wr) | |
| { | |
| struct mthca_srq *srq = to_msrq(ibsrq); | |
| int err = 0; | |
| int ind; | |
| int next_ind; | |
| int nreq; | |
| int i; | |
| void *wqe; | |
| pthread_spin_lock(&srq->lock); | |
| for (nreq = 0; wr; ++nreq, wr = wr->next) { | |
| ind = srq->first_free; | |
| wqe = get_wqe(srq, ind); | |
| next_ind = *wqe_to_link(wqe); | |
| if (next_ind < 0) { | |
| err = -1; | |
| *bad_wr = wr; | |
| break; | |
| } | |
| ((struct mthca_next_seg *) wqe)->ee_nds = 0; | |
| /* flags field will always remain 0 */ | |
| wqe += sizeof (struct mthca_next_seg); | |
| if (wr->num_sge > srq->max_gs) { | |
| err = -1; | |
| *bad_wr = wr; | |
| break; | |
| } | |
| for (i = 0; i < wr->num_sge; ++i) { | |
| ((struct mthca_data_seg *) wqe)->byte_count = | |
| htobe32(wr->sg_list[i].length); | |
| ((struct mthca_data_seg *) wqe)->lkey = | |
| htobe32(wr->sg_list[i].lkey); | |
| ((struct mthca_data_seg *) wqe)->addr = | |
| htobe64(wr->sg_list[i].addr); | |
| wqe += sizeof (struct mthca_data_seg); | |
| } | |
| if (i < srq->max_gs) { | |
| ((struct mthca_data_seg *) wqe)->byte_count = 0; | |
| ((struct mthca_data_seg *) wqe)->lkey = htobe32(MTHCA_INVAL_LKEY); | |
| ((struct mthca_data_seg *) wqe)->addr = 0; | |
| } | |
| srq->wrid[ind] = wr->wr_id; | |
| srq->first_free = next_ind; | |
| } | |
| if (nreq) { | |
| srq->counter += nreq; | |
| /* | |
| * Make sure that descriptors are written before | |
| * we write doorbell record. | |
| */ | |
| udma_ordering_write_barrier(); | |
| *srq->db = htobe32(srq->counter); | |
| } | |
| pthread_spin_unlock(&srq->lock); | |
| return err; | |
| } | |
| int mthca_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr, | |
| struct mthca_srq *srq) | |
| { | |
| struct mthca_data_seg *scatter; | |
| void *wqe; | |
| int size; | |
| int i; | |
| srq->wrid = malloc(srq->max * sizeof (uint64_t)); | |
| if (!srq->wrid) | |
| return -1; | |
| size = sizeof (struct mthca_next_seg) + | |
| srq->max_gs * sizeof (struct mthca_data_seg); | |
| for (srq->wqe_shift = 6; 1 << srq->wqe_shift < size; ++srq->wqe_shift) | |
| ; /* nothing */ | |
| srq->buf_size = srq->max << srq->wqe_shift; | |
| if (mthca_alloc_buf(&srq->buf, | |
| align(srq->buf_size, to_mdev(pd->context->device)->page_size), | |
| to_mdev(pd->context->device)->page_size)) { | |
| free(srq->wrid); | |
| return -1; | |
| } | |
| memset(srq->buf.buf, 0, srq->buf_size); | |
| /* | |
| * Now initialize the SRQ buffer so that all of the WQEs are | |
| * linked into the list of free WQEs. In addition, set the | |
| * scatter list L_Keys to the sentry value of 0x100. | |
| */ | |
| for (i = 0; i < srq->max; ++i) { | |
| struct mthca_next_seg *next; | |
| next = wqe = get_wqe(srq, i); | |
| if (i < srq->max - 1) { | |
| *wqe_to_link(wqe) = i + 1; | |
| next->nda_op = htobe32(((i + 1) << srq->wqe_shift) | 1); | |
| } else { | |
| *wqe_to_link(wqe) = -1; | |
| next->nda_op = 0; | |
| } | |
| for (scatter = wqe + sizeof (struct mthca_next_seg); | |
| (void *) scatter < wqe + (1 << srq->wqe_shift); | |
| ++scatter) | |
| scatter->lkey = htobe32(MTHCA_INVAL_LKEY); | |
| } | |
| srq->first_free = 0; | |
| srq->last_free = srq->max - 1; | |
| srq->last = get_wqe(srq, srq->max - 1); | |
| return 0; | |
| } |