providers/qedr/qelr.h

/*
 * Copyright (c) 2015-2016  QLogic Corporation
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and /or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#ifndef __QELR_H__
#define __QELR_H__

#include <inttypes.h>
#include <stddef.h>
#include <endian.h>
#include <stdio.h>
#include <endian.h>
#include <ccan/minmax.h>

#include <infiniband/driver.h>
#include <util/udma_barrier.h>

#define writel(b, p) (*(uint32_t *)(p) = (b))
#define writeq(b, p) (*(uint64_t *)(p) = (b))

#include "qelr_abi.h"
#include "qelr_hsi.h"
#include "qelr_chain.h"

#define qelr_err(format, arg...) printf(format, ##arg)

extern uint32_t qelr_dp_level;
extern uint32_t qelr_dp_module;

enum DP_MODULE {
	QELR_MSG_CQ		= 0x10000,
	QELR_MSG_RQ		= 0x20000,
	QELR_MSG_SQ		= 0x40000,
	QELR_MSG_QP		= (QELR_MSG_SQ | QELR_MSG_RQ),
	QELR_MSG_MR		= 0x80000,
	QELR_MSG_INIT		= 0x100000,
	QELR_MSG_SRQ		= 0x200000,
	/* to be added...up to 0x8000000 */
};

enum DP_LEVEL {
	QELR_LEVEL_VERBOSE	= 0x0,
	QELR_LEVEL_INFO		= 0x1,
	QELR_LEVEL_NOTICE	= 0x2,
	QELR_LEVEL_ERR		= 0x3,
};

#define DP_ERR(fd, fmt, ...)					\
do {								\
	fprintf(fd, "[%s:%d]" fmt,				\
		__func__, __LINE__,				\
		##__VA_ARGS__);					\
	fflush(fd); \
} while (0)

#define DP_NOTICE(fd, fmt, ...)					\
do {								\
	if (qelr_dp_level <= QELR_LEVEL_NOTICE)	{\
		fprintf(fd, "[%s:%d]" fmt,			\
		      __func__, __LINE__,			\
		      ##__VA_ARGS__);				\
		      fflush(fd); }				\
} while (0)

#define DP_INFO(fd, fmt, ...)					\
do {								\
	if (qelr_dp_level <= QELR_LEVEL_INFO)	{		\
		fprintf(fd, "[%s:%d]" fmt,			\
		      __func__, __LINE__,			\
		      ##__VA_ARGS__); fflush(fd);		\
	}							\
} while (0)

#define DP_VERBOSE(fd, module, fmt, ...)			\
do {								\
	if ((qelr_dp_level <= QELR_LEVEL_VERBOSE) &&		\
		     (qelr_dp_module & (module))) {		\
		fprintf(fd, "[%s:%d]" fmt,			\
		      __func__, __LINE__,			\
		      ##__VA_ARGS__);	fflush(fd); }		\
} while (0)

struct qelr_buf {
	void		*addr;
	size_t		len;		/* a 64 uint is used as s preparation
					 * for double layer pbl.
					 */
};

#define IS_IWARP(_dev)		(_dev->node_type == IBV_NODE_RNIC)
#define IS_ROCE(_dev)		(_dev->node_type == IBV_NODE_CA)

struct qelr_device {
	struct verbs_device ibv_dev;
};

enum qelr_dpm_flags {
	QELR_DPM_FLAGS_ENHANCED  = (1 << 0),
	QELR_DPM_FLAGS_LEGACY	 = (1 << 1),
	QELR_DPM_FLAGS_EDPM_MODE = (1 << 2),
};

#define QELR_MAX_SRQ_ID         4096

struct qelr_devctx {
	struct verbs_context	ibv_ctx;
	FILE			*dbg_fp;
	void			*db_addr;
	uint64_t		db_pa;
	struct qedr_user_db_rec	db_rec_addr_dummy;
	uint32_t		db_size;
	enum qelr_dpm_flags	dpm_flags;
	uint32_t		kernel_page_size;
	uint16_t		ldpm_limit_size;
	uint16_t		edpm_limit_size;
	uint8_t			edpm_trans_size;

	uint32_t		max_send_wr;
	uint32_t		max_recv_wr;
	uint32_t		max_srq_wr;
	uint32_t		sges_per_send_wr;
	uint32_t		sges_per_recv_wr;
	uint32_t		sges_per_srq_wr;
	struct qelr_srq		**srq_table;
	int			max_cqes;
};

struct qelr_pd {
	struct ibv_pd		ibv_pd;
	uint32_t		pd_id;
};

struct qelr_mr {
	struct verbs_mr		vmr;
};

union db_prod64 {
	struct rdma_pwm_val32_data data;
	uint64_t raw;
};

struct qelr_cq {
	struct ibv_cq		ibv_cq;	/* must be first */

	struct qelr_chain	chain;

	void			*db_addr;
	union db_prod64		db;
	/* Doorbell recovery entry address */
	void			*db_rec_map;
	struct qedr_user_db_rec *db_rec_addr;

	uint8_t			chain_toggle;
	union rdma_cqe		*latest_cqe;
	union rdma_cqe		*toggle_cqe;

	uint8_t			arm_flags;
};

enum qelr_qp_state {
	QELR_QPS_RST,
	QELR_QPS_INIT,
	QELR_QPS_RTR,
	QELR_QPS_RTS,
	QELR_QPS_SQD,
	QELR_QPS_ERR,
	QELR_QPS_SQE
};

union db_prod32 {
	struct rdma_pwm_val16_data	data;
	uint32_t			raw;
};

struct qelr_qp_hwq_info {
	/* WQE */
	struct qelr_chain			chain;
	uint8_t					max_sges;

	/* WQ */
	uint16_t				prod;
	uint16_t				wqe_cons;
	uint16_t				cons;
	uint16_t				max_wr;

	/* DB */
	void					*db;      /* Doorbell address */
	void					*edpm_db;
	union db_prod32				db_data;  /* Doorbell data */
	/* Doorbell recovery entry address */
	void					*db_rec_map;
	struct qedr_user_db_rec			*db_rec_addr;
	void					*iwarp_db2;
	union db_prod32				iwarp_db2_data;

	uint16_t				icid;
};

struct qelr_rdma_ext {
	__be64	remote_va;
	__be32	remote_key;
	__be32	dma_length;
};

struct qelr_xrceth {
	__be32	xrc_srq;
};

/* rdma extension, invalidate / immediate data + padding, inline data... */
#define QELR_MAX_DPM_PAYLOAD (sizeof(struct qelr_rdma_ext) + sizeof(uint64_t) +\
			       ROCE_REQ_MAX_INLINE_DATA_SIZE)
struct qelr_dpm {
	uint8_t			is_edpm;
	uint8_t			is_ldpm;
	union {
		struct db_roce_dpm_data	data;
		uint64_t raw;
	} msg;

	uint8_t			payload[QELR_MAX_DPM_PAYLOAD];
	uint32_t		payload_size;
	uint32_t		payload_offset;

	struct qelr_rdma_ext    *rdma_ext;
};

struct qelr_srq_hwq_info {
	uint32_t max_sges;
	uint32_t max_wr;
	struct qelr_chain chain;
	uint32_t wqe_prod;     /* WQE prod index in HW ring */
	uint32_t sge_prod;     /* SGE prod index in HW ring */
	uint32_t wr_prod_cnt;  /* wr producer count */
	uint32_t wr_cons_cnt;  /* wr consumer count */
	uint32_t num_elems;

	void  *virt_prod_pair_addr;  /* producer pair virtual address */
};

struct qelr_srq {
	struct verbs_srq verbs_srq;
	struct qelr_srq_hwq_info hw_srq;
	uint16_t srq_id;
	pthread_spinlock_t lock;
	bool is_xrc;
};

enum qelr_qp_flags {
	QELR_QP_FLAG_SQ		= 1 << 0,
	QELR_QP_FLAG_RQ		= 1 << 1,
};

struct qelr_qp {
	struct verbs_qp				verbs_qp;
	struct ibv_qp				*ibv_qp;
	pthread_spinlock_t			q_lock;
	enum qelr_qp_state			state;   /*  QP state */
	uint8_t					flags;

	struct qelr_qp_hwq_info			sq;
	struct qelr_qp_hwq_info			rq;
	struct {
		uint64_t wr_id;
		enum ibv_wc_opcode opcode;
		uint32_t bytes_len;
		uint8_t wqe_size;
		uint8_t signaled;
	} *wqe_wr_id;

	struct {
		uint64_t wr_id;
		uint8_t wqe_size;
	} *rqe_wr_id;

	uint8_t					prev_wqe_size;
	uint32_t				max_inline_data;
	uint32_t				qp_id;
	int					sq_sig_all;
	int					atomic_supported;
	uint8_t					edpm_disabled;
	uint8_t					edpm_mode;
	struct qelr_srq				*srq;
};

static inline struct qelr_devctx *get_qelr_ctx(struct ibv_context *ibctx)
{
	return container_of(ibctx, struct qelr_devctx, ibv_ctx.context);
}

static inline struct qelr_device *get_qelr_dev(struct ibv_device *ibdev)
{
	return container_of(ibdev, struct qelr_device, ibv_dev.device);
}

static inline struct ibv_qp *get_ibv_qp(struct qelr_qp *qp)
{
	return &qp->verbs_qp.qp;
}

static inline struct qelr_qp *get_qelr_qp(struct ibv_qp *ibqp)
{
	struct verbs_qp *vqp = (struct verbs_qp *)ibqp;

	return container_of(vqp, struct qelr_qp, verbs_qp);
}

static inline struct qelr_pd *get_qelr_pd(struct ibv_pd *ibpd)
{
	return container_of(ibpd, struct qelr_pd, ibv_pd);
}

static inline struct qelr_cq *get_qelr_cq(struct ibv_cq *ibcq)
{
	return container_of(ibcq, struct qelr_cq, ibv_cq);
}

static inline struct qelr_srq *get_qelr_srq(struct ibv_srq *ibsrq)
{
	struct verbs_srq *vsrq = (struct verbs_srq *)ibsrq;

	return container_of(vsrq, struct qelr_srq, verbs_srq);
}

static inline struct ibv_srq *get_ibv_srq(struct qelr_srq *srq)
{
	return &srq->verbs_srq.srq;
}

#define SET_FIELD(value, name, flag)				\
	do {							\
		(value) &= ~(name ## _MASK << name ## _SHIFT);	\
		(value) |= ((flag) << (name ## _SHIFT));	\
	} while (0)

#define SET_FIELD2(value, name, flag)				\
		((value) |= ((flag) << (name ## _SHIFT)))

#define GET_FIELD(value, name) \
	(((value) >> (name ## _SHIFT)) & name ## _MASK)

#define ROCE_WQE_ELEM_SIZE	sizeof(struct rdma_sq_sge)
#define RDMA_WQE_BYTES		(16)

#define QELR_RESP_IMM (RDMA_CQE_RESPONDER_IMM_FLG_MASK <<	\
			RDMA_CQE_RESPONDER_IMM_FLG_SHIFT)
#define QELR_RESP_INV	(RDMA_CQE_RESPONDER_INV_FLG_MASK << \
			 RDMA_CQE_RESPONDER_INV_FLG_SHIFT)
#define QELR_RESP_RDMA (RDMA_CQE_RESPONDER_RDMA_FLG_MASK <<	\
			RDMA_CQE_RESPONDER_RDMA_FLG_SHIFT)
#define QELR_RESP_RDMA_IMM (QELR_RESP_IMM | QELR_RESP_RDMA)

#define TYPEPTR_ADDR_SET(type_ptr, field, vaddr)			\
	do {								\
		(type_ptr)->field.hi = htole32(U64_HI(vaddr));	\
		(type_ptr)->field.lo = htole32(U64_LO(vaddr));	\
	} while (0)

#define RQ_SGE_SET(sge, vaddr, vlength, vflags)			\
	do {							\
		TYPEPTR_ADDR_SET(sge, addr, vaddr);		\
		(sge)->length = htole32(vlength);		\
		(sge)->flags = htole32(vflags);		\
	} while (0)

#define SRQ_HDR_SET(hdr, vwr_id, num_sge)			\
	do {							\
		TYPEPTR_ADDR_SET(hdr, wr_id, vwr_id);		\
		(hdr)->num_sges = num_sge;			\
	} while (0)

#define SRQ_SGE_SET(sge, vaddr, vlength, vlkey)			\
	do {							\
		TYPEPTR_ADDR_SET(sge, addr, vaddr);		\
		(sge)->length = htole32(vlength);		\
		(sge)->l_key = htole32(vlkey);		\
	} while (0)

#define U64_HI(val) ((uint32_t)(((uint64_t)(uintptr_t)(val)) >> 32))
#define U64_LO(val) ((uint32_t)(((uint64_t)(uintptr_t)(val)) & 0xffffffff))
#define HILO_U64(hi, lo) ((uintptr_t)((((uint64_t)(hi)) << 32) + (lo)))

#define QELR_MAX_RQ_WQE_SIZE (RDMA_MAX_SGE_PER_RQ_WQE)
#define QELR_MAX_SQ_WQE_SIZE (ROCE_REQ_MAX_SINGLE_SQ_WQE_SIZE /	\
			      ROCE_WQE_ELEM_SIZE)

#endif /* __QELR_H__ */