opensourcemirror
/
opnsense-src
mirror of https://github.com/opnsense/src.git
1
0
Fork 0
Code Issues Releases Wiki Activity

irdma(4): Upgrade to 1.2.36-k 

Update Intel irdma driver to version 1.2.36-k.

Notable changes:

- Start using ib_sge directly instead of irdma_sge
- Turn off flush completion generator for libirdma
- Minor formatting changes

Signed-off-by: Bartosz Sobczak <bartosz.sobczak@intel.com>
Signed-off-by: Eric Joyner <erj@FreeBSD.org>

Reviewed by:	erj@
Sponsored by:	Intel Corporation
Differential Revision:	https://reviews.freebsd.org/D43567

(cherry picked from commit 5b5f7d0e77a9eee73eb5d596f43aef4e1a3674d8)
This commit is contained in:
Bartosz Sobczak 2024-02-16 13:55:08 -08:00 committed by Franco Fichtner
parent 65147f6e3d
commit 0475af2433
37 changed files with 422 additions and 789 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
contrib/ofed/libirdma/abi.h
View File

@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
* Copyright (C) 2019 - 2022 Intel Corporation
* Copyright (C) 2019 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@ -85,6 +85,7 @@ struct irdma_ucreate_qp {
struct ibv_create_qp ibv_cmd;
__aligned_u64 user_wqe_bufs;
__aligned_u64 user_compl_ctx;
__aligned_u64 comp_mask;
};
struct irdma_ucreate_qp_resp {
@ -97,6 +98,9 @@ struct irdma_ucreate_qp_resp {
__u8 lsmm;
__u8 rsvd;
__u32 qp_caps;
__aligned_u64 comp_mask;
__u8 start_wqe_idx;
__u8 rsvd2[7];
};
struct irdma_umodify_qp_resp {
@ -137,6 +141,8 @@ struct irdma_get_context_resp {
__u8 hw_rev;
__u8 rsvd2;
__aligned_u64 comp_mask;
__u16 min_hw_wq_size;
__u8 rsvd3[6];
};
struct irdma_ureg_mr {

11
contrib/ofed/libirdma/irdma-abi.h
View File

@ -54,6 +54,11 @@ enum irdma_memreg_type {
enum {
IRDMA_ALLOC_UCTX_USE_RAW_ATTR = 1 << 0,
IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE = 1 << 1,
};
enum {
IRDMA_CREATE_QP_USE_START_WQE_IDX = 1 << 0,
};
struct irdma_alloc_ucontext_req {
@ -82,6 +87,8 @@ struct irdma_alloc_ucontext_resp {
__u8 hw_rev;
__u8 rsvd2;
__aligned_u64 comp_mask;
__u16 min_hw_wq_size;
__u8 rsvd3[6];
};
struct irdma_alloc_pd_resp {
@ -101,6 +108,7 @@ struct irdma_create_cq_req {
struct irdma_create_qp_req {
__aligned_u64 user_wqe_bufs;
__aligned_u64 user_compl_ctx;
__aligned_u64 comp_mask;
};
struct irdma_mem_reg_req {
@ -130,6 +138,9 @@ struct irdma_create_qp_resp {
__u8 lsmm;
__u8 rsvd;
__u32 qp_caps;
__aligned_u64 comp_mask;
__u8 start_wqe_idx;
__u8 rsvd2[7];
};
struct irdma_modify_qp_resp {

1
contrib/ofed/libirdma/irdma.h
View File

@ -46,7 +46,6 @@ enum irdma_vers {
IRDMA_GEN_RSVD = 0,
IRDMA_GEN_1 = 1,
IRDMA_GEN_2 = 2,
IRDMA_GEN_MAX = 2,
};
struct irdma_uk_attrs {

13
contrib/ofed/libirdma/irdma_defs.h
View File

@ -456,19 +456,6 @@ enum irdma_qp_wqe_size {
IRDMA_WQE_SIZE_256 = 256,
};
enum irdma_ws_op_type {
IRDMA_WS_OP_TYPE_NODE = 0,
IRDMA_WS_OP_TYPE_LEAF_NODE_GROUP,
};
enum irdma_ws_rate_limit_flags {
IRDMA_WS_RATE_LIMIT_FLAGS_VALID = 0x1,
IRDMA_WS_NO_RDMA_RATE_LIMIT = 0x2,
IRDMA_WS_LEAF_NODE_IS_PART_GROUP = 0x4,
IRDMA_WS_TREE_RATE_LIMITING = 0x8,
IRDMA_WS_PACING_CONTROL = 0x10,
};
/**
* set_64bit_val - set 64 bit value to hw wqe
* @wqe_words: wqe addr to write

106
contrib/ofed/libirdma/irdma_uk.c
View File

@ -45,16 +45,16 @@
* @valid: The wqe valid
*/
static void
irdma_set_fragment(__le64 * wqe, u32 offset, struct irdma_sge *sge,
irdma_set_fragment(__le64 * wqe, u32 offset, struct ibv_sge *sge,
u8 valid)
{
if (sge) {
set_64bit_val(wqe, offset,
FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off));
FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->addr));
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
FIELD_PREP(IRDMAQPSQ_VALID, valid) |
FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge->len) |
FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge->stag));
FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge->length) |
FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge->lkey));
} else {
set_64bit_val(wqe, offset, 0);
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
@ -71,14 +71,14 @@ irdma_set_fragment(__le64 * wqe, u32 offset, struct irdma_sge *sge,
*/
static void
irdma_set_fragment_gen_1(__le64 * wqe, u32 offset,
struct irdma_sge *sge, u8 valid)
struct ibv_sge *sge, u8 valid)
{
if (sge) {
set_64bit_val(wqe, offset,
FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off));
FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->addr));
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge->len) |
FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge->stag));
FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge->length) |
FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge->lkey));
} else {
set_64bit_val(wqe, offset, 0);
set_64bit_val(wqe, offset + IRDMA_BYTE_8, 0);
@ -209,8 +209,7 @@ irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 * wqe, u16 quanta,
if (IRDMA_RING_CURRENT_HEAD(qp->initial_ring) !=
IRDMA_RING_CURRENT_TAIL(qp->sq_ring) &&
!qp->push_mode) {
if (post_sq)
irdma_uk_qp_post_wr(qp);
irdma_uk_qp_post_wr(qp);
} else {
push = (__le64 *) ((uintptr_t)qp->push_wqe +
(wqe_idx & 0x7) * 0x20);
@ -338,7 +337,7 @@ irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
return EINVAL;
for (i = 0; i < op_info->num_lo_sges; i++)
total_size += op_info->lo_sg_list[i].len;
total_size += op_info->lo_sg_list[i].length;
read_fence |= info->read_fence;
@ -357,7 +356,7 @@ irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled;
set_64bit_val(wqe, IRDMA_BYTE_16,
FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr));
if (info->imm_data_valid) {
set_64bit_val(wqe, IRDMA_BYTE_0,
@ -386,7 +385,7 @@ irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
++addl_frag_cnt;
}
hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) |
FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid) |
FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt) |
@ -437,7 +436,7 @@ irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
return EINVAL;
for (i = 0; i < op_info->num_lo_sges; i++)
total_size += op_info->lo_sg_list[i].len;
total_size += op_info->lo_sg_list[i].length;
ret_code = irdma_fragcnt_to_quanta_sq(op_info->num_lo_sges, &quanta);
if (ret_code)
@ -475,8 +474,8 @@ irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
++addl_frag_cnt;
}
set_64bit_val(wqe, IRDMA_BYTE_16,
FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr));
hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) |
FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) |
FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
FIELD_PREP(IRDMAQPSQ_OPCODE,
@ -525,7 +524,7 @@ irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
return EINVAL;
for (i = 0; i < op_info->num_sges; i++)
total_size += op_info->sg_list[i].len;
total_size += op_info->sg_list[i].length;
if (info->imm_data_valid)
frag_cnt = op_info->num_sges + 1;
@ -620,15 +619,15 @@ irdma_set_mw_bind_wqe_gen_1(__le64 * wqe,
* @polarity: compatibility parameter
*/
static void
irdma_copy_inline_data_gen_1(u8 *wqe, struct irdma_sge *sge_list,
irdma_copy_inline_data_gen_1(u8 *wqe, struct ibv_sge *sge_list,
u32 num_sges, u8 polarity)
{
u32 quanta_bytes_remaining = 16;
u32 i;
for (i = 0; i < num_sges; i++) {
u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].tag_off;
u32 sge_len = sge_list[i].len;
u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr;
u32 sge_len = sge_list[i].length;
while (sge_len) {
u32 bytes_copied;
@ -683,7 +682,7 @@ irdma_set_mw_bind_wqe(__le64 * wqe,
* @polarity: polarity of wqe valid bit
*/
static void
irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list,
irdma_copy_inline_data(u8 *wqe, struct ibv_sge *sge_list,
u32 num_sges, u8 polarity)
{
u8 inline_valid = polarity << IRDMA_INLINE_VALID_S;
@ -694,8 +693,8 @@ irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list,
wqe += 8;
for (i = 0; i < num_sges; i++) {
u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].tag_off;
u32 sge_len = sge_list[i].len;
u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr;
u32 sge_len = sge_list[i].length;
while (sge_len) {
u32 bytes_copied;
@ -775,7 +774,7 @@ irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
return EINVAL;
for (i = 0; i < op_info->num_lo_sges; i++)
total_size += op_info->lo_sg_list[i].len;
total_size += op_info->lo_sg_list[i].length;
if (unlikely(total_size > qp->max_inline_data))
return EINVAL;
@ -788,9 +787,9 @@ irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled;
read_fence |= info->read_fence;
set_64bit_val(wqe, IRDMA_BYTE_16,
FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr));
hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) |
FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) |
FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt ? 1 : 0) |
@ -846,7 +845,7 @@ irdma_uk_inline_send(struct irdma_qp_uk *qp,
return EINVAL;
for (i = 0; i < op_info->num_sges; i++)
total_size += op_info->sg_list[i].len;
total_size += op_info->sg_list[i].length;
if (unlikely(total_size > qp->max_inline_data))
return EINVAL;
@ -911,7 +910,7 @@ irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
u64 hdr;
u32 wqe_idx;
bool local_fence = false;
struct irdma_sge sge = {0};
struct ibv_sge sge = {0};
u16 quanta = IRDMA_QP_WQE_MIN_QUANTA;
info->push_wqe = qp->push_db ? true : false;
@ -922,7 +921,7 @@ irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
if (!wqe)
return ENOSPC;
sge.stag = op_info->target_stag;
sge.lkey = op_info->target_stag;
qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, &sge, 0);
set_64bit_val(wqe, IRDMA_BYTE_16, 0);
@ -1436,8 +1435,7 @@ irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq,
IRDMA_RING_MOVE_TAIL(cq->cq_ring);
set_64bit_val(cq->shadow_area, IRDMA_BYTE_0,
IRDMA_RING_CURRENT_HEAD(cq->cq_ring));
memset(info, 0,
sizeof(struct irdma_cq_poll_info));
memset(info, 0, sizeof(*info));
return irdma_uk_cq_poll_cmpl(cq, info);
}
}
@ -1510,7 +1508,6 @@ exit:
if (pring && IRDMA_RING_MORE_WORK(*pring))
move_cq_head = false;
}
if (move_cq_head) {
IRDMA_RING_MOVE_HEAD_NOCHECK(cq->cq_ring);
if (!IRDMA_RING_CURRENT_HEAD(cq->cq_ring))
@ -1591,10 +1588,12 @@ irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge,
int
irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *sqdepth)
{
u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift;
*sqdepth = irdma_round_up_wq((sq_size << shift) + IRDMA_SQ_RSVD);
if (*sqdepth < ((u32)uk_attrs->min_hw_wq_size << shift))
*sqdepth = uk_attrs->min_hw_wq_size << shift;
if (*sqdepth < min_size)
*sqdepth = min_size;
else if (*sqdepth > uk_attrs->max_hw_wq_quanta)
return EINVAL;
@ -1608,10 +1607,12 @@ irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *s
int
irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u32 *rqdepth)
{
u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift;
*rqdepth = irdma_round_up_wq((rq_size << shift) + IRDMA_RQ_RSVD);
if (*rqdepth < ((u32)uk_attrs->min_hw_wq_size << shift))
*rqdepth = uk_attrs->min_hw_wq_size << shift;
if (*rqdepth < min_size)
*rqdepth = min_size;
else if (*rqdepth > uk_attrs->max_hw_rq_quanta)
return EINVAL;
@ -1644,41 +1645,16 @@ irdma_setup_connection_wqes(struct irdma_qp_uk *qp,
{
u16 move_cnt = 1;
if (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE)
if (info->start_wqe_idx)
move_cnt = info->start_wqe_idx;
else if (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE)
move_cnt = 3;
qp->conn_wqes = move_cnt;
IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, move_cnt);
IRDMA_RING_MOVE_TAIL_BY_COUNT(qp->sq_ring, move_cnt);
IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->initial_ring, move_cnt);
}
/**
* irdma_uk_calc_shift_wq - calculate WQE shift for both SQ and RQ
* @ukinfo: qp initialization info
* @sq_shift: Returns shift of SQ
* @rq_shift: Returns shift of RQ
*/
void
irdma_uk_calc_shift_wq(struct irdma_qp_uk_init_info *ukinfo, u8 *sq_shift,
u8 *rq_shift)
{
bool imm_support = ukinfo->uk_attrs->hw_rev >= IRDMA_GEN_2 ? true : false;
irdma_get_wqe_shift(ukinfo->uk_attrs,
imm_support ? ukinfo->max_sq_frag_cnt + 1 :
ukinfo->max_sq_frag_cnt,
ukinfo->max_inline_data, sq_shift);
irdma_get_wqe_shift(ukinfo->uk_attrs, ukinfo->max_rq_frag_cnt, 0,
rq_shift);
if (ukinfo->uk_attrs->hw_rev == IRDMA_GEN_1) {
if (ukinfo->abi_ver > 4)
*rq_shift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1;
}
}
/**
* irdma_uk_calc_depth_shift_sq - calculate depth and shift for SQ size.
* @ukinfo: qp initialization info
@ -1786,6 +1762,8 @@ irdma_uk_qp_init(struct irdma_qp_uk *qp, struct irdma_qp_uk_init_info *info)
qp->wqe_ops = iw_wqe_uk_ops_gen_1;
else
qp->wqe_ops = iw_wqe_uk_ops;
qp->start_wqe_idx = info->start_wqe_idx;
return ret_code;
}

2
contrib/ofed/libirdma/irdma_umain.c
View File

@ -48,7 +48,7 @@
/**
* Driver version
*/
char libirdma_version[] = "1.2.17-k";
char libirdma_version[] = "1.2.36-k";
unsigned int irdma_dbg;

2
contrib/ofed/libirdma/irdma_umain.h
View File

@ -96,7 +96,6 @@ struct irdma_cq_buf {
LIST_ENTRY(irdma_cq_buf) list;
struct irdma_cq_uk cq;
struct verbs_mr vmr;
size_t buf_size;
};
extern pthread_mutex_t sigusr1_wait_mutex;
@ -142,7 +141,6 @@ struct irdma_uqp {
struct ibv_recv_wr *pend_rx_wr;
struct irdma_qp_uk qp;
enum ibv_qp_type qp_type;
struct irdma_sge *recv_sges;
};
/* irdma_uverbs.c */

198
contrib/ofed/libirdma/irdma_user.h
View File

@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
* Copyright (c) 2015 - 2022 Intel Corporation
* Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@ -50,7 +50,7 @@
#define irdma_access_privileges u32
#define irdma_physical_fragment u64
#define irdma_address_list u64 *
#define irdma_sgl struct irdma_sge *
#define irdma_sgl struct ibv_sge *
#define IRDMA_MAX_MR_SIZE 0x200000000000ULL
@ -80,96 +80,6 @@
#define IRDMA_OP_TYPE_REC_IMM 0x3f
#define IRDMA_FLUSH_MAJOR_ERR 1
#define IRDMA_SRQFLUSH_RSVD_MAJOR_ERR 0xfffe
/* Async Events codes */
#define IRDMA_AE_AMP_UNALLOCATED_STAG 0x0102
#define IRDMA_AE_AMP_INVALID_STAG 0x0103
#define IRDMA_AE_AMP_BAD_QP 0x0104
#define IRDMA_AE_AMP_BAD_PD 0x0105
#define IRDMA_AE_AMP_BAD_STAG_KEY 0x0106
#define IRDMA_AE_AMP_BAD_STAG_INDEX 0x0107
#define IRDMA_AE_AMP_BOUNDS_VIOLATION 0x0108
#define IRDMA_AE_AMP_RIGHTS_VIOLATION 0x0109
#define IRDMA_AE_AMP_TO_WRAP 0x010a
#define IRDMA_AE_AMP_FASTREG_VALID_STAG 0x010c
#define IRDMA_AE_AMP_FASTREG_MW_STAG 0x010d
#define IRDMA_AE_AMP_FASTREG_INVALID_RIGHTS 0x010e
#define IRDMA_AE_AMP_FASTREG_INVALID_LENGTH 0x0110
#define IRDMA_AE_AMP_INVALIDATE_SHARED 0x0111
#define IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS 0x0112
#define IRDMA_AE_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS 0x0113
#define IRDMA_AE_AMP_MWBIND_VALID_STAG 0x0114
#define IRDMA_AE_AMP_MWBIND_OF_MR_STAG 0x0115
#define IRDMA_AE_AMP_MWBIND_TO_ZERO_BASED_STAG 0x0116
#define IRDMA_AE_AMP_MWBIND_TO_MW_STAG 0x0117
#define IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS 0x0118
#define IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS 0x0119
#define IRDMA_AE_AMP_MWBIND_TO_INVALID_PARENT 0x011a
#define IRDMA_AE_AMP_MWBIND_BIND_DISABLED 0x011b
#define IRDMA_AE_PRIV_OPERATION_DENIED 0x011c
#define IRDMA_AE_AMP_INVALIDATE_TYPE1_MW 0x011d
#define IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW 0x011e
#define IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG 0x011f
#define IRDMA_AE_AMP_MWBIND_WRONG_TYPE 0x0120
#define IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH 0x0121
#define IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG 0x0132
#define IRDMA_AE_UDA_XMIT_BAD_PD 0x0133
#define IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT 0x0134
#define IRDMA_AE_UDA_L4LEN_INVALID 0x0135
#define IRDMA_AE_BAD_CLOSE 0x0201
#define IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE 0x0202
#define IRDMA_AE_CQ_OPERATION_ERROR 0x0203
#define IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO 0x0205
#define IRDMA_AE_STAG_ZERO_INVALID 0x0206
#define IRDMA_AE_IB_RREQ_AND_Q1_FULL 0x0207
#define IRDMA_AE_IB_INVALID_REQUEST 0x0208
#define IRDMA_AE_WQE_UNEXPECTED_OPCODE 0x020a
#define IRDMA_AE_WQE_INVALID_PARAMETER 0x020b
#define IRDMA_AE_WQE_INVALID_FRAG_DATA 0x020c
#define IRDMA_AE_IB_REMOTE_ACCESS_ERROR 0x020d
#define IRDMA_AE_IB_REMOTE_OP_ERROR 0x020e
#define IRDMA_AE_WQE_LSMM_TOO_LONG 0x0220
#define IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN 0x0301
#define IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER 0x0303
#define IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION 0x0304
#define IRDMA_AE_DDP_UBE_INVALID_MO 0x0305
#define IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE 0x0306
#define IRDMA_AE_DDP_UBE_INVALID_QN 0x0307
#define IRDMA_AE_DDP_NO_L_BIT 0x0308
#define IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION 0x0311
#define IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE 0x0312
#define IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST 0x0313
#define IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP 0x0314
#define IRDMA_AE_ROCE_RSP_LENGTH_ERROR 0x0316
#define IRDMA_AE_ROCE_EMPTY_MCG 0x0380
#define IRDMA_AE_ROCE_BAD_MC_IP_ADDR 0x0381
#define IRDMA_AE_ROCE_BAD_MC_QPID 0x0382
#define IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH 0x0383
#define IRDMA_AE_INVALID_ARP_ENTRY 0x0401
#define IRDMA_AE_INVALID_TCP_OPTION_RCVD 0x0402
#define IRDMA_AE_STALE_ARP_ENTRY 0x0403
#define IRDMA_AE_INVALID_AH_ENTRY 0x0406
#define IRDMA_AE_LLP_CLOSE_COMPLETE 0x0501
#define IRDMA_AE_LLP_CONNECTION_RESET 0x0502
#define IRDMA_AE_LLP_FIN_RECEIVED 0x0503
#define IRDMA_AE_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH 0x0504
#define IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR 0x0505
#define IRDMA_AE_LLP_SEGMENT_TOO_SMALL 0x0507
#define IRDMA_AE_LLP_SYN_RECEIVED 0x0508
#define IRDMA_AE_LLP_TERMINATE_RECEIVED 0x0509
#define IRDMA_AE_LLP_TOO_MANY_RETRIES 0x050a
#define IRDMA_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES 0x050b
#define IRDMA_AE_LLP_DOUBT_REACHABILITY 0x050c
#define IRDMA_AE_LLP_CONNECTION_ESTABLISHED 0x050e
#define IRDMA_AE_RESOURCE_EXHAUSTION 0x0520
#define IRDMA_AE_RESET_SENT 0x0601
#define IRDMA_AE_TERMINATE_SENT 0x0602
#define IRDMA_AE_RESET_NOT_SENT 0x0603
#define IRDMA_AE_LCE_QP_CATASTROPHIC 0x0700
#define IRDMA_AE_LCE_FUNCTION_CATASTROPHIC 0x0701
#define IRDMA_AE_LCE_CQ_CATASTROPHIC 0x0702
#define IRDMA_AE_QP_SUSPEND_COMPLETE 0x0900
enum irdma_device_caps_const {
IRDMA_WQE_SIZE = 4,
@ -201,8 +111,7 @@ enum irdma_device_caps_const {
IRDMA_MAX_OUTBOUND_MSG_SIZE = 65537,
/* 64K +1 */
IRDMA_MAX_INBOUND_MSG_SIZE = 65537,
IRDMA_MAX_PUSH_PAGE_COUNT = 1024,
IRDMA_MAX_PE_ENA_VF_COUNT = 32,
IRDMA_MAX_PE_ENA_VF_COUNT = 32,
IRDMA_MAX_VF_FPM_ID = 47,
IRDMA_MAX_SQ_PAYLOAD_SIZE = 2145386496,
IRDMA_MAX_INLINE_DATA_SIZE = 101,
@ -229,12 +138,7 @@ enum irdma_flush_opcode {
FLUSH_RETRY_EXC_ERR,
FLUSH_MW_BIND_ERR,
FLUSH_REM_INV_REQ_ERR,
};
enum irdma_qp_event_type {
IRDMA_QP_EVENT_CATASTROPHIC,
IRDMA_QP_EVENT_ACCESS_ERR,
IRDMA_QP_EVENT_REQ_ERR,
FLUSH_RNR_RETRY_EXC_ERR,
};
enum irdma_cmpl_status {
@ -282,12 +186,6 @@ struct irdma_cq_uk;
struct irdma_qp_uk_init_info;
struct irdma_cq_uk_init_info;
struct irdma_sge {
irdma_tagged_offset tag_off;
u32 len;
irdma_stag stag;
};
struct irdma_ring {
volatile u32 head;
volatile u32 tail; /* effective tail */
@ -319,13 +217,13 @@ struct irdma_post_rq_info {
struct irdma_rdma_write {
irdma_sgl lo_sg_list;
u32 num_lo_sges;
struct irdma_sge rem_addr;
struct ibv_sge rem_addr;
};
struct irdma_rdma_read {
irdma_sgl lo_sg_list;
u32 num_lo_sges;
struct irdma_sge rem_addr;
struct ibv_sge rem_addr;
};
struct irdma_bind_window {
@ -399,11 +297,6 @@ struct irdma_cq_poll_info {
} stat;
};
struct qp_err_code {
enum irdma_flush_opcode flush_code;
enum irdma_qp_event_type event_type;
};
int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
struct irdma_post_sq_info *info, bool post_sq);
int irdma_uk_inline_send(struct irdma_qp_uk *qp,
@ -426,9 +319,9 @@ int irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
bool post_sq);
struct irdma_wqe_uk_ops {
void (*iw_copy_inline_data)(u8 *dest, struct irdma_sge *sge_list, u32 num_sges, u8 polarity);
void (*iw_copy_inline_data)(u8 *dest, struct ibv_sge *sge_list, u32 num_sges, u8 polarity);
u16 (*iw_inline_data_size_to_quanta)(u32 data_size);
void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct irdma_sge *sge,
void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct ibv_sge *sge,
u8 valid);
void (*iw_set_mw_bind_wqe)(__le64 *wqe,
struct irdma_bind_window *op_info);
@ -444,8 +337,6 @@ int irdma_uk_cq_init(struct irdma_cq_uk *cq,
struct irdma_cq_uk_init_info *info);
int irdma_uk_qp_init(struct irdma_qp_uk *qp,
struct irdma_qp_uk_init_info *info);
void irdma_uk_calc_shift_wq(struct irdma_qp_uk_init_info *ukinfo, u8 *sq_shift,
u8 *rq_shift);
int irdma_uk_calc_depth_shift_sq(struct irdma_qp_uk_init_info *ukinfo,
u32 *sq_depth, u8 *sq_shift);
int irdma_uk_calc_depth_shift_rq(struct irdma_qp_uk_init_info *ukinfo,
@ -494,6 +385,7 @@ struct irdma_qp_uk {
u8 rwqe_polarity;
u8 rq_wqe_size;
u8 rq_wqe_size_multiplier;
u8 start_wqe_idx;
bool deferred_flag:1;
bool push_mode:1; /* whether the last post wqe was pushed */
bool push_dropped:1;
@ -541,6 +433,7 @@ struct irdma_qp_uk_init_info {
u32 sq_depth;
u32 rq_depth;
u8 first_sq_wq;
u8 start_wqe_idx;
u8 type;
u8 sq_shift;
u8 rq_shift;
@ -574,75 +467,4 @@ int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u3
void irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 *wqe, u16 quanta,
u32 wqe_idx, bool post_sq);
void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx);
static inline struct qp_err_code irdma_ae_to_qp_err_code(u16 ae_id)
{
struct qp_err_code qp_err = { 0 };
switch (ae_id) {
case IRDMA_AE_AMP_BOUNDS_VIOLATION:
case IRDMA_AE_AMP_INVALID_STAG:
case IRDMA_AE_AMP_RIGHTS_VIOLATION:
case IRDMA_AE_AMP_UNALLOCATED_STAG:
case IRDMA_AE_AMP_BAD_PD:
case IRDMA_AE_AMP_BAD_QP:
case IRDMA_AE_AMP_BAD_STAG_KEY:
case IRDMA_AE_AMP_BAD_STAG_INDEX:
case IRDMA_AE_AMP_TO_WRAP:
case IRDMA_AE_PRIV_OPERATION_DENIED:
qp_err.flush_code = FLUSH_PROT_ERR;
qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;
break;
case IRDMA_AE_UDA_XMIT_BAD_PD:
case IRDMA_AE_WQE_UNEXPECTED_OPCODE:
qp_err.flush_code = FLUSH_LOC_QP_OP_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
break;
case IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT:
case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG:
case IRDMA_AE_UDA_L4LEN_INVALID:
case IRDMA_AE_DDP_UBE_INVALID_MO:
case IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
qp_err.flush_code = FLUSH_LOC_LEN_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
break;
case IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
case IRDMA_AE_IB_REMOTE_ACCESS_ERROR:
qp_err.flush_code = FLUSH_REM_ACCESS_ERR;
qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;
break;
case IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS:
case IRDMA_AE_AMP_MWBIND_BIND_DISABLED:
case IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS:
case IRDMA_AE_AMP_MWBIND_VALID_STAG:
qp_err.flush_code = FLUSH_MW_BIND_ERR;
qp_err.event_type = IRDMA_QP_EVENT_ACCESS_ERR;
break;
case IRDMA_AE_LLP_TOO_MANY_RETRIES:
qp_err.flush_code = FLUSH_RETRY_EXC_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
break;
case IRDMA_AE_IB_INVALID_REQUEST:
qp_err.flush_code = FLUSH_REM_INV_REQ_ERR;
qp_err.event_type = IRDMA_QP_EVENT_REQ_ERR;
break;
case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:
case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:
case IRDMA_AE_ROCE_RSP_LENGTH_ERROR:
case IRDMA_AE_IB_REMOTE_OP_ERROR:
qp_err.flush_code = FLUSH_REM_OP_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
break;
case IRDMA_AE_LCE_QP_CATASTROPHIC:
qp_err.flush_code = FLUSH_FATAL_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
break;
default:
qp_err.flush_code = FLUSH_GENERAL_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
break;
}
return qp_err;
}
#endif /* IRDMA_USER_H */

89
contrib/ofed/libirdma/irdma_uverbs.c
View File

@ -152,6 +152,7 @@ irdma_ualloc_pd(struct ibv_context *context)
err_free:
free(iwupd);
errno = err;
return NULL;
}
@ -163,7 +164,6 @@ err_free:
int
irdma_ufree_pd(struct ibv_pd *pd)
{
struct irdma_uvcontext *iwvctx = container_of(pd->context, struct irdma_uvcontext, ibv_ctx);
struct irdma_upd *iwupd;
int ret;
@ -374,12 +374,12 @@ irdma_free_hw_buf(void *buf, size_t size)
* @cqe_64byte_ena: enable 64byte cqe
*/
static inline int
get_cq_size(int ncqe, u8 hw_rev, bool cqe_64byte_ena)
get_cq_size(int ncqe, u8 hw_rev)
{
ncqe++;
/* Completions with immediate require 1 extra entry */
if (!cqe_64byte_ena && hw_rev > IRDMA_GEN_1)
if (hw_rev > IRDMA_GEN_1)
ncqe *= 2;
if (ncqe < IRDMA_U_MINCQ_SIZE)
@ -388,11 +388,8 @@ get_cq_size(int ncqe, u8 hw_rev, bool cqe_64byte_ena)
return ncqe;
}
static inline size_t get_cq_total_bytes(u32 cq_size, bool cqe_64byte_ena){
if (cqe_64byte_ena)
return roundup(cq_size * sizeof(struct irdma_extended_cqe), IRDMA_HW_PAGE_SIZE);
else
return roundup(cq_size * sizeof(struct irdma_cqe), IRDMA_HW_PAGE_SIZE);
static inline size_t get_cq_total_bytes(u32 cq_size) {
return roundup(cq_size * sizeof(struct irdma_cqe), IRDMA_HW_PAGE_SIZE);
}
/**
@ -420,7 +417,6 @@ ucreate_cq(struct ibv_context *context,
u32 cq_pages;
int ret, ncqe;
u8 hw_rev;
bool cqe_64byte_ena;
iwvctx = container_of(context, struct irdma_uvcontext, ibv_ctx);
uk_attrs = &iwvctx->uk_attrs;
@ -454,11 +450,10 @@ ucreate_cq(struct ibv_context *context,
return NULL;
}
cqe_64byte_ena = uk_attrs->feature_flags & IRDMA_FEATURE_64_BYTE_CQE ? true : false;
info.cq_size = get_cq_size(attr_ex->cqe, hw_rev, cqe_64byte_ena);
info.cq_size = get_cq_size(attr_ex->cqe, hw_rev);
total_size = get_cq_total_bytes(info.cq_size);
iwucq->comp_vector = attr_ex->comp_vector;
LIST_INIT(&iwucq->resize_list);
total_size = get_cq_total_bytes(info.cq_size, cqe_64byte_ena);
cq_pages = total_size >> IRDMA_HW_PAGE_SHIFT;
if (!(uk_attrs->feature_flags & IRDMA_FEATURE_CQ_RESIZE))
@ -528,8 +523,6 @@ ucreate_cq(struct ibv_context *context,
info.cq_id = resp.cq_id;
/* Do not report the CQE's reserved for immediate and burned by HW */
iwucq->verbs_cq.cq.cqe = ncqe;
if (cqe_64byte_ena)
info.avoid_mem_cflct = true;
info.cqe_alloc_db = (u32 *)((u8 *)iwvctx->db + IRDMA_DB_CQ_OFFSET);
irdma_uk_cq_init(&iwucq->cq, &info);
return &iwucq->verbs_cq.cq_ex;
@ -585,7 +578,7 @@ static void
irdma_free_cq_buf(struct irdma_cq_buf *cq_buf)
{
ibv_cmd_dereg_mr(&cq_buf->vmr.ibv_mr);
irdma_free_hw_buf(cq_buf->cq.cq_base, cq_buf->buf_size);
irdma_free_hw_buf(cq_buf->cq.cq_base, get_cq_total_bytes(cq_buf->cq.cq_size));
free(cq_buf);
}
@ -1322,6 +1315,8 @@ irdma_vmapped_qp(struct irdma_uqp *iwuqp, struct ibv_pd *pd,
cmd.user_wqe_bufs = (__u64) ((uintptr_t)info->sq);
cmd.user_compl_ctx = (__u64) (uintptr_t)&iwuqp->qp;
cmd.comp_mask |= IRDMA_CREATE_QP_USE_START_WQE_IDX;
ret = ibv_cmd_create_qp(pd, &iwuqp->ibv_qp, attr, &cmd.ibv_cmd,
sizeof(cmd), &resp.ibv_resp,
sizeof(struct irdma_ucreate_qp_resp));
@ -1331,6 +1326,8 @@ irdma_vmapped_qp(struct irdma_uqp *iwuqp, struct ibv_pd *pd,
info->sq_size = resp.actual_sq_size;
info->rq_size = resp.actual_rq_size;
info->first_sq_wq = legacy_mode ? 1 : resp.lsmm;
if (resp.comp_mask & IRDMA_CREATE_QP_USE_START_WQE_IDX)
info->start_wqe_idx = resp.start_wqe_idx;
info->qp_caps = resp.qp_caps;
info->qp_id = resp.qp_id;
iwuqp->irdma_drv_opt = resp.irdma_drv_opt;
@ -1379,6 +1376,8 @@ irdma_ucreate_qp(struct ibv_pd *pd,
if (attr->cap.max_send_sge > uk_attrs->max_hw_wq_frags ||
attr->cap.max_recv_sge > uk_attrs->max_hw_wq_frags ||
attr->cap.max_send_wr > uk_attrs->max_hw_wq_quanta ||
attr->cap.max_recv_wr > uk_attrs->max_hw_rq_quanta ||
attr->cap.max_inline_data > uk_attrs->max_hw_inline) {
errno = EINVAL;
return NULL;
@ -1430,18 +1429,12 @@ irdma_ucreate_qp(struct ibv_pd *pd,
attr->cap.max_recv_wr = info.rq_size;
}
iwuqp->recv_sges = calloc(attr->cap.max_recv_sge, sizeof(*iwuqp->recv_sges));
if (!iwuqp->recv_sges) {
status = errno; /* preserve errno */
goto err_destroy_lock;
}
info.wqe_alloc_db = (u32 *)iwvctx->db;
info.legacy_mode = iwvctx->legacy_mode;
info.sq_wrtrk_array = calloc(info.sq_depth, sizeof(*info.sq_wrtrk_array));
if (!info.sq_wrtrk_array) {
status = errno; /* preserve errno */
goto err_free_rsges;
goto err_destroy_lock;
}
info.rq_wrid_array = calloc(info.rq_depth, sizeof(*info.rq_wrid_array));
@ -1475,8 +1468,6 @@ err_free_rq_wrid:
free(info.rq_wrid_array);
err_free_sq_wrtrk:
free(info.sq_wrtrk_array);
err_free_rsges:
free(iwuqp->recv_sges);
err_destroy_lock:
pthread_spin_destroy(&iwuqp->lock);
err_free_qp:
@ -1635,7 +1626,6 @@ irdma_udestroy_qp(struct ibv_qp *qp)
free(iwuqp->qp.rq_wrid_array);
irdma_free_hw_buf(iwuqp->qp.sq_base, iwuqp->buf_size);
free(iwuqp->recv_sges);
free(iwuqp);
return 0;
@ -1645,26 +1635,6 @@ err:
return ret;
}
/**
* irdma_copy_sg_list - copy sg list for qp
* @sg_list: copied into sg_list
* @sgl: copy from sgl
* @num_sges: count of sg entries
* @max_sges: count of max supported sg entries
*/
static void
irdma_copy_sg_list(struct irdma_sge *sg_list, struct ibv_sge *sgl,
int num_sges)
{
int i;
for (i = 0; i < num_sges; i++) {
sg_list[i].tag_off = sgl[i].addr;
sg_list[i].len = sgl[i].length;
sg_list[i].stag = sgl[i].lkey;
}
}
/**
* calc_type2_mw_stag - calculate type 2 MW stag
* @rkey: desired rkey of the MW
@ -1743,7 +1713,7 @@ irdma_upost_send(struct ibv_qp *ib_qp, struct ibv_send_wr *ib_wr,
info.stag_to_inv = ib_wr->imm_data;
}
info.op.send.num_sges = ib_wr->num_sge;
info.op.send.sg_list = (struct irdma_sge *)ib_wr->sg_list;
info.op.send.sg_list = (struct ibv_sge *)ib_wr->sg_list;
if (ib_qp->qp_type == IBV_QPT_UD) {
struct irdma_uah *ah = container_of(ib_wr->wr.ud.ah,
struct irdma_uah, ibv_ah);
@ -1774,9 +1744,9 @@ irdma_upost_send(struct ibv_qp *ib_qp, struct ibv_send_wr *ib_wr,
info.op_type = IRDMA_OP_TYPE_RDMA_WRITE;
info.op.rdma_write.num_lo_sges = ib_wr->num_sge;
info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list;
info.op.rdma_write.rem_addr.tag_off = ib_wr->wr.rdma.remote_addr;
info.op.rdma_write.rem_addr.stag = ib_wr->wr.rdma.rkey;
info.op.rdma_write.lo_sg_list = ib_wr->sg_list;
info.op.rdma_write.rem_addr.addr = ib_wr->wr.rdma.remote_addr;
info.op.rdma_write.rem_addr.lkey = ib_wr->wr.rdma.rkey;
if (ib_wr->send_flags & IBV_SEND_INLINE)
err = irdma_uk_inline_rdma_write(&iwuqp->qp, &info, false);
else
@ -1788,10 +1758,10 @@ irdma_upost_send(struct ibv_qp *ib_qp, struct ibv_send_wr *ib_wr,
break;
}
info.op_type = IRDMA_OP_TYPE_RDMA_READ;
info.op.rdma_read.rem_addr.tag_off = ib_wr->wr.rdma.remote_addr;
info.op.rdma_read.rem_addr.stag = ib_wr->wr.rdma.rkey;
info.op.rdma_read.rem_addr.addr = ib_wr->wr.rdma.remote_addr;
info.op.rdma_read.rem_addr.lkey = ib_wr->wr.rdma.rkey;
info.op.rdma_read.lo_sg_list = (void *)ib_wr->sg_list;
info.op.rdma_read.lo_sg_list = ib_wr->sg_list;
info.op.rdma_read.num_lo_sges = ib_wr->num_sge;
err = irdma_uk_rdma_read(&iwuqp->qp, &info, false, false);
break;
@ -1873,14 +1843,11 @@ irdma_upost_recv(struct ibv_qp *ib_qp, struct ibv_recv_wr *ib_wr,
struct ibv_recv_wr **bad_wr)
{
struct irdma_post_rq_info post_recv = {};
struct irdma_sge *sg_list;
struct irdma_uqp *iwuqp;
bool reflush = false;
int err = 0;
iwuqp = container_of(ib_qp, struct irdma_uqp, ibv_qp);
sg_list = iwuqp->recv_sges;
err = pthread_spin_lock(&iwuqp->lock);
if (err)
return err;
@ -1897,8 +1864,7 @@ irdma_upost_recv(struct ibv_qp *ib_qp, struct ibv_recv_wr *ib_wr,
}
post_recv.num_sges = ib_wr->num_sge;
post_recv.wr_id = ib_wr->wr_id;
irdma_copy_sg_list(sg_list, ib_wr->sg_list, ib_wr->num_sge);
post_recv.sg_list = sg_list;
post_recv.sg_list = ib_wr->sg_list;
err = irdma_uk_post_receive(&iwuqp->qp, &post_recv);
if (err) {
*bad_wr = ib_wr;
@ -2022,7 +1988,6 @@ irdma_uresize_cq(struct ibv_cq *cq, int cqe)
u32 cq_pages;
int cqe_needed;
int ret = 0;
bool cqe_64byte_ena;
iwucq = container_of(cq, struct irdma_ucq, verbs_cq.cq);
iwvctx = container_of(cq->context, struct irdma_uvcontext, ibv_ctx);
@ -2034,14 +1999,11 @@ irdma_uresize_cq(struct ibv_cq *cq, int cqe)
if (cqe < uk_attrs->min_hw_cq_size || cqe > uk_attrs->max_hw_cq_size - 1)
return EINVAL;
cqe_64byte_ena = uk_attrs->feature_flags & IRDMA_FEATURE_64_BYTE_CQE ? true : false;
cqe_needed = get_cq_size(cqe, uk_attrs->hw_rev, cqe_64byte_ena);
cqe_needed = get_cq_size(cqe, uk_attrs->hw_rev);
if (cqe_needed == iwucq->cq.cq_size)
return 0;
cq_size = get_cq_total_bytes(cqe_needed, cqe_64byte_ena);
cq_size = get_cq_total_bytes(cqe_needed);
cq_pages = cq_size >> IRDMA_HW_PAGE_SHIFT;
cq_base = irdma_alloc_hw_buf(cq_size);
if (!cq_base)
@ -2077,7 +2039,6 @@ irdma_uresize_cq(struct ibv_cq *cq, int cqe)
goto err_resize;
memcpy(&cq_buf->cq, &iwucq->cq, sizeof(cq_buf->cq));
cq_buf->buf_size = cq_size;
cq_buf->vmr = iwucq->vmr;
iwucq->vmr = new_mr;
irdma_uk_cq_resize(&iwucq->cq, cq_base, cqe_needed);

1
contrib/ofed/libirdma/osdep.h
View File

@ -77,7 +77,6 @@
/* constants */
#define STATS_TIMER_DELAY 60000
/* a couple of linux size defines */
#define BIT_ULL(a) (1ULL << (a))
#define min(a, b) ((a) > (b) ? (b) : (a))
#ifndef likely

5
sys/dev/irdma/fbsd_kcompat.c
View File

@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
* Copyright (c) 2021 - 2022 Intel Corporation
* Copyright (c) 2021 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@ -660,8 +660,7 @@ static const struct irdma_sw_stats_tunable_info irdma_sws_list[] = {
"manage_hmc_pm_func_table", 0},
{IRDMA_OP_SUSPEND, "suspend", "suspend", 0},
{IRDMA_OP_RESUME, "resume", "resume", 0},
{IRDMA_OP_MANAGE_VCHNL_REQ_PBLE_BP, "manage_vchnl_req_pble_bp",
"manage_vchnl_req_pble_bp", 0},
{25, "manage_vchnl_req_pble_bp", "manage_vchnl_req_pble_bp", 0},
{IRDMA_OP_QUERY_FPM_VAL, "query_fpm_val", "query_fpm_val", 0},
{IRDMA_OP_COMMIT_FPM_VAL, "commit_fpm_val", "commit_fpm_val", 0},
{IRDMA_OP_AH_CREATE, "ah_create", "ah_create", 0},

25
sys/dev/irdma/fbsd_kcompat.h
View File

@ -53,15 +53,12 @@
ibdev.dma_device = (dev)
#define set_max_sge(props, rf) \
((props)->max_sge = (rf)->sc_dev.hw_attrs.uk_attrs.max_hw_wq_frags)
#define rdma_query_gid(ibdev, port, index, gid) \
ib_get_cached_gid(ibdev, port, index, gid, NULL)
#define kmap(pg) page_address(pg)
#define kmap_local_page(pg) page_address(pg)
#define kunmap(pg)
#define kunmap_local(pg)
#define IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION IB_CQ_FLAGS_TIMESTAMP_COMPLETION
#define kc_irdma_destroy_qp(ibqp, udata) irdma_destroy_qp(ibqp)
#ifndef IB_QP_ATTR_STANDARD_BITS
#define IB_QP_ATTR_STANDARD_BITS GENMASK(20, 0)
#endif
@ -71,34 +68,26 @@
#define IRDMA_VER_LEN 24
#ifndef EVNT_HNDLR_CRITERR
#if ICE_RDMA_MAJOR_VERSION == 1 && ICE_RDMA_MINOR_VERSION == 1
#define EVNT_HNDLR_CRITERR
#else
#undef EVNT_HNDLR_CRITERR
#endif
#endif
void kc_set_roce_uverbs_cmd_mask(struct irdma_device *iwdev);
void kc_set_rdma_uverbs_cmd_mask(struct irdma_device *iwdev);
struct irdma_tunable_info {
struct sysctl_ctx_list irdma_sysctl_ctx;
struct sysctl_oid *irdma_sysctl_tree;
struct sysctl_oid *qos_sysctl_tree;
struct sysctl_oid *sws_sysctl_tree;
char drv_ver[IRDMA_VER_LEN];
u8 roce_ena;
};
typedef u_int if_addr_cb_t(void *, struct ifaddr *, u_int);
u_int if_foreach_addr_type(if_t ifp, int type, if_addr_cb_t cb, void *cb_arg);
typedef int (*if_foreach_cb_t)(if_t, void *);
int if_foreach(if_foreach_cb_t cb, void *cb_arg);
#ifndef if_iter
struct if_iter {
void *context[4];
};
#endif
u_int if_foreach_addr_type(if_t ifp, int type, if_addr_cb_t cb, void *cb_arg);
int if_foreach(if_foreach_cb_t cb, void *cb_arg);
if_t if_iter_start(struct if_iter *iter);
if_t if_iter_next(struct if_iter *iter);
void if_iter_finish(struct if_iter *iter);
@ -241,8 +230,8 @@ void irdma_cq_free_rsrc(struct irdma_pci_f *rf, struct irdma_cq *iwcq);
int irdma_validate_qp_attrs(struct ib_qp_init_attr *init_attr,
struct irdma_device *iwdev);
void irdma_setup_virt_qp(struct irdma_device *iwdev,
struct irdma_qp *iwqp,
struct irdma_qp_init_info *init_info);
struct irdma_qp *iwqp,
struct irdma_qp_init_info *init_info);
int irdma_setup_kmode_qp(struct irdma_device *iwdev,
struct irdma_qp *iwqp,
struct irdma_qp_init_info *info,
@ -258,7 +247,7 @@ void irdma_iw_fill_and_set_qpctx_info(struct irdma_qp *iwqp,
struct irdma_qp_host_ctx_info *ctx_info);
int irdma_cqp_create_qp_cmd(struct irdma_qp *iwqp);
void irdma_dealloc_push_page(struct irdma_pci_f *rf,
struct irdma_sc_qp *qp);
struct irdma_qp *iwqp);
int irdma_process_resize_list(struct irdma_cq *iwcq, struct irdma_device *iwdev,
struct irdma_cq_buf *lcqe_buf);
int irdma_destroy_cq(struct ib_cq *ib_cq);

2
sys/dev/irdma/icrdma.c
View File

@ -52,7 +52,7 @@
/**
* Driver version
*/
char irdma_driver_version[] = "1.2.17-k";
char irdma_driver_version[] = "1.2.36-k";
/**
* irdma_init_tunable - prepare tunables

7
sys/dev/irdma/icrdma_hw.c
View File

@ -76,7 +76,7 @@ static u64 icrdma_masks[IRDMA_MAX_MASKS] = {
ICRDMA_CQPSQ_CQ_CEQID,
ICRDMA_CQPSQ_CQ_CQID,
ICRDMA_COMMIT_FPM_CQCNT,
ICRDMA_CQPSQ_UPESD_HMCFNID
ICRDMA_CQPSQ_UPESD_HMCFNID,
};
static u8 icrdma_shifts[IRDMA_MAX_SHIFTS] = {
@ -86,7 +86,7 @@ static u8 icrdma_shifts[IRDMA_MAX_SHIFTS] = {
ICRDMA_CQPSQ_CQ_CEQID_S,
ICRDMA_CQPSQ_CQ_CQID_S,
ICRDMA_COMMIT_FPM_CQCNT_S,
ICRDMA_CQPSQ_UPESD_HMCFNID_S
ICRDMA_CQPSQ_UPESD_HMCFNID_S,
};
/**
@ -210,8 +210,6 @@ icrdma_init_hw(struct irdma_sc_dev *dev)
dev->hw_regs[i] = (u32 IOMEM *) (hw_addr + icrdma_regs[i]);
}
dev->hw_attrs.max_hw_vf_fpm_id = IRDMA_MAX_VF_FPM_ID;
dev->hw_attrs.first_hw_vf_fpm_id = IRDMA_FIRST_VF_FPM_ID;
for (i = 0; i < IRDMA_MAX_SHIFTS; ++i)
dev->hw_shifts[i] = icrdma_shifts[i];
@ -231,6 +229,7 @@ icrdma_init_hw(struct irdma_sc_dev *dev)
dev->hw_attrs.max_hw_ord = ICRDMA_MAX_ORD_SIZE;
dev->hw_attrs.max_stat_inst = ICRDMA_MAX_STATS_COUNT;
dev->hw_attrs.max_stat_idx = IRDMA_HW_STAT_INDEX_MAX_GEN_2;
dev->hw_attrs.max_hw_device_pages = ICRDMA_MAX_PUSH_PAGE_COUNT;
dev->hw_attrs.uk_attrs.max_hw_wq_frags = ICRDMA_MAX_WQ_FRAGMENT_COUNT;
dev->hw_attrs.uk_attrs.max_hw_read_sges = ICRDMA_MAX_SGE_RD;

3
sys/dev/irdma/icrdma_hw.h
View File

@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
* Copyright (c) 2017 - 2022 Intel Corporation
* Copyright (c) 2017 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@ -123,6 +123,7 @@ enum icrdma_device_caps_const {
ICRDMA_MAX_IRD_SIZE = 32,
ICRDMA_MAX_ORD_SIZE = 32,
ICRDMA_MIN_WQ_SIZE = 8 /* WQEs */,
ICRDMA_MAX_PUSH_PAGE_COUNT = 256,
};

11
sys/dev/irdma/irdma-abi.h
View File

@ -54,6 +54,11 @@ enum irdma_memreg_type {
enum {
IRDMA_ALLOC_UCTX_USE_RAW_ATTR = 1 << 0,
IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE = 1 << 1,
};
enum {
IRDMA_CREATE_QP_USE_START_WQE_IDX = 1 << 0,
};
struct irdma_alloc_ucontext_req {
@ -82,6 +87,8 @@ struct irdma_alloc_ucontext_resp {
__u8 hw_rev;
__u8 rsvd2;
__aligned_u64 comp_mask;
__u16 min_hw_wq_size;
__u8 rsvd3[6];
};
struct irdma_alloc_pd_resp {
@ -101,6 +108,7 @@ struct irdma_create_cq_req {
struct irdma_create_qp_req {
__aligned_u64 user_wqe_bufs;
__aligned_u64 user_compl_ctx;
__aligned_u64 comp_mask;
};
struct irdma_mem_reg_req {
@ -130,6 +138,9 @@ struct irdma_create_qp_resp {
__u8 lsmm;
__u8 rsvd;
__u32 qp_caps;
__aligned_u64 comp_mask;
__u8 start_wqe_idx;
__u8 rsvd2[7];
};
struct irdma_modify_qp_resp {

2
sys/dev/irdma/irdma.h
View File

@ -183,7 +183,7 @@ enum irdma_vers {
IRDMA_GEN_RSVD = 0,
IRDMA_GEN_1 = 1,
IRDMA_GEN_2 = 2,
IRDMA_GEN_MAX = 2,
IRDMA_GEN_MAX = IRDMA_GEN_2,
};
struct irdma_uk_attrs {

47
sys/dev/irdma/irdma_cm.c
View File

@ -1079,14 +1079,14 @@ irdma_parse_mpa(struct irdma_cm_node *cm_node, u8 *buf, u32 *type,
*type = IRDMA_MPA_REQUEST_ACCEPT;
if (len < sizeof(struct ietf_mpa_v1)) {
if (len < sizeof(*mpa_frame)) {
irdma_debug(&cm_node->iwdev->rf->sc_dev, IRDMA_DEBUG_CM,
"ietf buffer small (%x)\n", len);
return -EINVAL;
}
mpa_frame = (struct ietf_mpa_v1 *)buf;
mpa_hdr_len = sizeof(struct ietf_mpa_v1);
mpa_hdr_len = sizeof(*mpa_frame);
priv_data_len = ntohs(mpa_frame->priv_data_len);
if (priv_data_len > IETF_MAX_PRIV_DATA_LEN) {
@ -1616,7 +1616,8 @@ static u8 irdma_iw_get_vlan_prio(u32 *loc_addr, u8 prio, bool ipv4)
}
/**
* irdma_netdev_vlan_ipv6 - Gets the netdev and mac
* irdma_get_vlan_mac_ipv6 - Get the vlan and mac for an IPv6
* address
* @addr: local IPv6 address
* @vlan_id: vlan id for the given IPv6 address
* @mac: mac address for the given IPv6 address
@ -1624,14 +1625,12 @@ static u8 irdma_iw_get_vlan_prio(u32 *loc_addr, u8 prio, bool ipv4)
* Returns the net_device of the IPv6 address and also sets the
* vlan id and mac for that address.
*/
if_t
irdma_netdev_vlan_ipv6(struct iw_cm_id *cm_id, u32 *addr, u16 *vlan_id, u8 *mac)
void
irdma_get_vlan_mac_ipv6(struct iw_cm_id *cm_id, u32 *addr, u16 *vlan_id, u8 *mac)
{
if_t ip_dev = NULL;
struct in6_addr laddr6;
#ifdef VIMAGE
struct vnet *vnet = irdma_cmid_to_vnet(cm_id);
#endif
struct vnet *vnet = &init_net;
struct ifaddr *ifa;
u16 scope_id = 0;
@ -1646,10 +1645,9 @@ irdma_netdev_vlan_ipv6(struct iw_cm_id *cm_id, u32 *addr, u16 *vlan_id, u8 *mac)
scope_id = ntohs(laddr6.__u6_addr.__u6_addr16[1]);
#ifdef VIMAGE
ip_dev = ip6_ifp_find(vnet, laddr6, scope_id);
#else
ip_dev = ip6_ifp_find(&init_net, laddr6, scope_id);
vnet = irdma_cmid_to_vnet(cm_id);
#endif
ip_dev = ip6_ifp_find(vnet, laddr6, scope_id);
if (ip_dev) {
if (vlan_id)
*vlan_id = rdma_vlan_dev_vlan_id(ip_dev);
@ -1657,8 +1655,6 @@ irdma_netdev_vlan_ipv6(struct iw_cm_id *cm_id, u32 *addr, u16 *vlan_id, u8 *mac)
if (ifa && ifa->ifa_addr && mac)
ether_addr_copy(mac, if_getlladdr(ip_dev));
}
return ip_dev;
}
/**
@ -1669,16 +1665,13 @@ u16
irdma_get_vlan_ipv4(struct iw_cm_id *cm_id, u32 *addr)
{
if_t netdev;
#ifdef VIMAGE
struct vnet *vnet = irdma_cmid_to_vnet(cm_id);
#endif
struct vnet *vnet = &init_net;
u16 vlan_id = 0xFFFF;
#ifdef VIMAGE
netdev = ip_ifp_find(vnet, htonl(addr[0]));
#else
netdev = ip_ifp_find(&init_net, htonl(addr[0]));
vnet = irdma_cmid_to_vnet(cm_id);
#endif
netdev = ip_ifp_find(vnet, htonl(addr[0]));
if (netdev) {
vlan_id = rdma_vlan_dev_vlan_id(netdev);
dev_put(netdev);
@ -2546,7 +2539,7 @@ irdma_handle_syn_pkt(struct irdma_cm_node *cm_node,
u32 inc_sequence;
int optionsize;
optionsize = (tcph->th_off << 2) - sizeof(struct tcphdr);
optionsize = (tcph->th_off << 2) - sizeof(*tcph);
inc_sequence = ntohl(tcph->th_seq);
switch (cm_node->state) {
@ -2613,7 +2606,7 @@ irdma_handle_synack_pkt(struct irdma_cm_node *cm_node,
u32 inc_sequence;
int optionsize;
optionsize = (tcph->th_off << 2) - sizeof(struct tcphdr);
optionsize = (tcph->th_off << 2) - sizeof(*tcph);
inc_sequence = ntohl(tcph->th_seq);
switch (cm_node->state) {
case IRDMA_CM_STATE_SYN_SENT:
@ -2687,7 +2680,7 @@ irdma_handle_ack_pkt(struct irdma_cm_node *cm_node,
int optionsize;
u32 datasize = rbuf->datalen;
optionsize = (tcph->th_off << 2) - sizeof(struct tcphdr);
optionsize = (tcph->th_off << 2) - sizeof(*tcph);
if (irdma_check_seq(cm_node, tcph))
return -EINVAL;
@ -3537,8 +3530,8 @@ irdma_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
cm_node->vlan_id = irdma_get_vlan_ipv4(cm_id, cm_node->loc_addr);
} else {
cm_node->ipv4 = false;
irdma_netdev_vlan_ipv6(cm_id, cm_node->loc_addr,
&cm_node->vlan_id, NULL);
irdma_get_vlan_mac_ipv6(cm_id, cm_node->loc_addr, &cm_node->vlan_id,
NULL);
}
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_CM, "Accept vlan_id=%d\n",
cm_node->vlan_id);
@ -3745,7 +3738,7 @@ irdma_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
raddr6->sin6_addr.__u6_addr.__u6_addr32);
cm_info.loc_port = ntohs(laddr6->sin6_port);
cm_info.rem_port = ntohs(raddr6->sin6_port);
irdma_netdev_vlan_ipv6(cm_id, cm_info.loc_addr, &cm_info.vlan_id, NULL);
irdma_get_vlan_mac_ipv6(cm_id, cm_info.loc_addr, &cm_info.vlan_id, NULL);
}
cm_info.cm_id = cm_id;
cm_info.qh_qpid = iwdev->vsi.ilq->qp_id;
@ -3874,8 +3867,8 @@ irdma_create_listen(struct iw_cm_id *cm_id, int backlog)
laddr6->sin6_addr.__u6_addr.__u6_addr32);
cm_info.loc_port = ntohs(laddr6->sin6_port);
if (!IN6_IS_ADDR_UNSPECIFIED(&laddr6->sin6_addr)) {
irdma_netdev_vlan_ipv6(cm_id, cm_info.loc_addr,
&cm_info.vlan_id, NULL);
irdma_get_vlan_mac_ipv6(cm_id, cm_info.loc_addr,
&cm_info.vlan_id, NULL);
} else {
cm_info.vlan_id = 0xFFFF;
wildcard = true;

2
sys/dev/irdma/irdma_cm.h
View File

@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
* Copyright (c) 2015 - 2022 Intel Corporation
* Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU

95
sys/dev/irdma/irdma_ctrl.c
View File

@ -785,14 +785,14 @@ irdma_sc_qp_setctx_roce(struct irdma_sc_qp *qp, __le64 * qp_ctx,
set_64bit_val(qp_ctx, IRDMA_BYTE_144,
FIELD_PREP(IRDMAQPC_STAT_INDEX, info->stats_idx));
set_64bit_val(qp_ctx, IRDMA_BYTE_152,
FIELD_PREP(IRDMAQPC_MACADDRESS, irdma_mac_to_u64(roce_info->mac_addr)));
FIELD_PREP(IRDMAQPC_MACADDRESS,
irdma_mac_to_u64(roce_info->mac_addr)));
set_64bit_val(qp_ctx, IRDMA_BYTE_160,
FIELD_PREP(IRDMAQPC_ORDSIZE, roce_info->ord_size) |
FIELD_PREP(IRDMAQPC_IRDSIZE, irdma_sc_get_encoded_ird_size(roce_info->ird_size)) |
FIELD_PREP(IRDMAQPC_WRRDRSPOK, roce_info->wr_rdresp_en) |
FIELD_PREP(IRDMAQPC_RDOK, roce_info->rd_en) |
FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info->stats_idx_valid) |
FIELD_PREP(IRDMAQPC_BINDEN, roce_info->bind_en) |
FIELD_PREP(IRDMAQPC_FASTREGEN, roce_info->fast_reg_en) |
FIELD_PREP(IRDMAQPC_DCQCNENABLE, roce_info->dcqcn_en) |
FIELD_PREP(IRDMAQPC_RCVNOICRC, roce_info->rcv_no_icrc) |
@ -1016,7 +1016,6 @@ irdma_sc_qp_setctx(struct irdma_sc_qp *qp, __le64 * qp_ctx,
FIELD_PREP(IRDMAQPC_WRRDRSPOK, iw->wr_rdresp_en) |
FIELD_PREP(IRDMAQPC_RDOK, iw->rd_en) |
FIELD_PREP(IRDMAQPC_SNDMARKERS, iw->snd_mark_en) |
FIELD_PREP(IRDMAQPC_BINDEN, iw->bind_en) |
FIELD_PREP(IRDMAQPC_FASTREGEN, iw->fast_reg_en) |
FIELD_PREP(IRDMAQPC_PRIVEN, iw->priv_mode_en) |
FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info->stats_idx_valid) |
@ -1466,6 +1465,15 @@ irdma_sc_gen_rts_ae(struct irdma_sc_qp *qp)
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
irdma_debug_buf(qp->dev, IRDMA_DEBUG_QP, "CONN EST WQE", wqe,
IRDMA_QP_WQE_MIN_SIZE);
if (qp->qp_uk.start_wqe_idx) {
wqe = qp_uk->sq_base[3].elem;
hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) |
FIELD_PREP(IRDMAQPSQ_LOCALFENCE, 1) |
FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
irdma_wmb(); /* make sure WQE is written before valid bit is set */
set_64bit_val(wqe, IRDMA_BYTE_24, hdr);
}
}
/**
@ -1831,7 +1839,7 @@ irdma_bld_terminate_hdr(struct irdma_sc_qp *qp,
if (copy_len)
irdma_memcpy(termhdr + 1, pkt, copy_len);
return sizeof(struct irdma_terminate_hdr) + copy_len;
return sizeof(*termhdr) + copy_len;
}
/**
@ -2124,9 +2132,6 @@ irdma_sc_gather_stats(struct irdma_sc_cqp *cqp,
IRDMA_CQP_WQE_SIZE * 8);
irdma_sc_cqp_post_sq(cqp);
irdma_debug(cqp->dev, IRDMA_DEBUG_STATS,
"CQP SQ head 0x%x tail 0x%x size 0x%x\n", cqp->sq_ring.head,
cqp->sq_ring.tail, cqp->sq_ring.size);
return 0;
}
@ -2607,17 +2612,18 @@ irdma_sc_cq_create(struct irdma_sc_cq *cq, u64 scratch,
set_64bit_val(wqe, IRDMA_BYTE_0, cq->cq_uk.cq_size);
set_64bit_val(wqe, IRDMA_BYTE_8, RS_64_1(cq, 1));
set_64bit_val(wqe, IRDMA_BYTE_16,
FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, cq->shadow_read_threshold));
set_64bit_val(wqe, IRDMA_BYTE_32, (cq->virtual_map ? 0 : cq->cq_pa));
FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD,
cq->shadow_read_threshold));
set_64bit_val(wqe, IRDMA_BYTE_32, cq->virtual_map ? 0 : cq->cq_pa);
set_64bit_val(wqe, IRDMA_BYTE_40, cq->shadow_area_pa);
set_64bit_val(wqe, IRDMA_BYTE_48,
FIELD_PREP(IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX, (cq->virtual_map ? cq->first_pm_pbl_idx : 0)));
FIELD_PREP(IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX,
cq->virtual_map ? cq->first_pm_pbl_idx : 0));
set_64bit_val(wqe, IRDMA_BYTE_56,
FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) |
FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx));
hdr = FLD_LS_64(cq->dev, cq->cq_uk.cq_id, IRDMA_CQPSQ_CQ_CQID) |
FLD_LS_64(cq->dev, (cq->ceq_id_valid ? cq->ceq_id : 0),
FLD_LS_64(cq->dev, cq->ceq_id_valid ? cq->ceq_id : 0,
IRDMA_CQPSQ_CQ_CEQID) |
FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CQ) |
FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, cq->pbl_chunk_size) |
@ -2935,10 +2941,12 @@ irdma_sc_parse_fpm_commit_buf(struct irdma_sc_dev *dev, __le64 * buf,
IRDMA_HMC_IW_HDR);
irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_152, info,
IRDMA_HMC_IW_MD);
irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_160, info,
IRDMA_HMC_IW_OOISC);
irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_168, info,
IRDMA_HMC_IW_OOISCFFL);
if (dev->cqp->protocol_used == IRDMA_IWARP_PROTOCOL_ONLY) {
irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_160, info,
IRDMA_HMC_IW_OOISC);
irdma_sc_decode_fpm_commit(dev, buf, IRDMA_BYTE_168, info,
IRDMA_HMC_IW_OOISCFFL);
}
}
/* searching for the last object in HMC to find the size of the HMC area. */
@ -3071,15 +3079,18 @@ irdma_sc_parse_fpm_query_buf(struct irdma_sc_dev *dev, __le64 * buf,
irdma_sc_decode_fpm_query(buf, 144, obj_info, IRDMA_HMC_IW_HDR);
irdma_sc_decode_fpm_query(buf, 152, obj_info, IRDMA_HMC_IW_MD);
irdma_sc_decode_fpm_query(buf, 160, obj_info, IRDMA_HMC_IW_OOISC);
get_64bit_val(buf, IRDMA_BYTE_168, &temp);
obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt = (u32)temp;
obj_info[IRDMA_HMC_IW_OOISCFFL].size = 4;
hmc_fpm_misc->ooiscf_block_size = FIELD_GET(IRDMA_QUERY_FPM_OOISCFBLOCKSIZE, temp);
if (!hmc_fpm_misc->ooiscf_block_size &&
obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt)
return -EINVAL;
if (dev->cqp->protocol_used == IRDMA_IWARP_PROTOCOL_ONLY) {
irdma_sc_decode_fpm_query(buf, 160, obj_info, IRDMA_HMC_IW_OOISC);
get_64bit_val(buf, IRDMA_BYTE_168, &temp);
obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt = (u32)temp;
obj_info[IRDMA_HMC_IW_OOISCFFL].size = 4;
hmc_fpm_misc->ooiscf_block_size = FIELD_GET(IRDMA_QUERY_FPM_OOISCFBLOCKSIZE, temp);
if (!hmc_fpm_misc->ooiscf_block_size &&
obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt)
return -EINVAL;
}
return 0;
}
@ -3251,6 +3262,7 @@ irdma_sc_cqp_create(struct irdma_sc_cqp *cqp, u16 *maj_err, u16 *min_err)
temp = FIELD_PREP(IRDMA_CQPHC_ENABLED_VFS, cqp->ena_vf_count) |
FIELD_PREP(IRDMA_CQPHC_HMC_PROFILE, cqp->hmc_profile);
if (hw_rev >= IRDMA_GEN_2)
temp |= FIELD_PREP(IRDMA_CQPHC_EN_REM_ENDPOINT_TRK,
cqp->en_rem_endpoint_trk);
@ -3399,11 +3411,13 @@ irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp, bool free_hwcqp)
void
irdma_sc_ccq_arm(struct irdma_sc_cq *ccq)
{
unsigned long flags;
u64 temp_val;
u16 sw_cq_sel;
u8 arm_next_se;
u8 arm_seq_num;
spin_lock_irqsave(&ccq->dev->cqp_lock, flags);
get_64bit_val(ccq->cq_uk.shadow_area, IRDMA_BYTE_32, &temp_val);
sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val);
arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val);
@ -3414,6 +3428,7 @@ irdma_sc_ccq_arm(struct irdma_sc_cq *ccq)
FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) |
FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, 1);
set_64bit_val(ccq->cq_uk.shadow_area, IRDMA_BYTE_32, temp_val);
spin_unlock_irqrestore(&ccq->dev->cqp_lock, flags);
irdma_wmb(); /* make sure shadow area is updated before arming */
@ -3436,6 +3451,7 @@ irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq,
u32 error;
u8 polarity;
int ret_code = 0;
unsigned long flags;
if (ccq->cq_uk.avoid_mem_cflct)
cqe = IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(&ccq->cq_uk);
@ -3484,7 +3500,9 @@ irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq,
irdma_wmb(); /* make sure shadow area is updated before moving tail */
spin_lock_irqsave(&cqp->dev->cqp_lock, flags);
IRDMA_RING_MOVE_TAIL(cqp->sq_ring);
spin_unlock_irqrestore(&cqp->dev->cqp_lock, flags);
atomic64_inc(&cqp->completed_ops);
return ret_code;
@ -4152,6 +4170,7 @@ irdma_sc_get_next_aeqe(struct irdma_sc_aeq *aeq,
case IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST:
case IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
case IRDMA_AE_ROCE_RSP_LENGTH_ERROR:
case IRDMA_AE_ROCE_REQ_LENGTH_ERROR:
case IRDMA_AE_INVALID_ARP_ENTRY:
case IRDMA_AE_INVALID_TCP_OPTION_RCVD:
case IRDMA_AE_STALE_ARP_ENTRY:
@ -4704,10 +4723,11 @@ static u32 irdma_est_sd(struct irdma_sc_dev *dev,
u64 sd;
int i;
for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++)
for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++) {
if (i != IRDMA_HMC_IW_PBLE)
size += round_up(hmc_info->hmc_obj[i].cnt *
hmc_info->hmc_obj[i].size, 512);
}
pble_info = &hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE];
size += round_up(pble_info->cnt * pble_info->size, 512);
@ -4717,7 +4737,7 @@ static u32 irdma_est_sd(struct irdma_sc_dev *dev,
sd = size >> 21;
if (sd > 0xFFFFFFFF) {
irdma_debug(dev, IRDMA_DEBUG_HMC, "sd overflow[%ld]\n", sd);
sd = 0xFFFFFFFF - 1;
sd = 0xFFFFFFFE;
}
return (u32)sd;
@ -4773,10 +4793,9 @@ irdma_sc_query_rdma_features(struct irdma_sc_cqp *cqp,
int
irdma_get_rdma_features(struct irdma_sc_dev *dev)
{
int ret_code;
int ret_code, byte_idx, feat_type, feat_cnt, feat_idx;
struct irdma_dma_mem feat_buf;
u64 temp;
u16 byte_idx, feat_type, feat_cnt, feat_idx;
feat_buf.size = IRDMA_FEATURE_BUF_SIZE;
feat_buf.va = irdma_allocate_dma_mem(dev->hw, &feat_buf, feat_buf.size,
@ -4819,7 +4838,7 @@ irdma_get_rdma_features(struct irdma_sc_dev *dev)
irdma_debug_buf(dev, IRDMA_DEBUG_WQE, "QUERY RDMA FEATURES", feat_buf.va,
feat_cnt * 8);
for (byte_idx = 0, feat_idx = 0; feat_idx < min(feat_cnt, (u16)IRDMA_MAX_FEATURES);
for (byte_idx = 0, feat_idx = 0; feat_idx < min(feat_cnt, IRDMA_MAX_FEATURES);
feat_idx++, byte_idx += 8) {
get_64bit_val(feat_buf.va, byte_idx, &temp);
feat_type = FIELD_GET(IRDMA_FEATURE_TYPE, temp);
@ -5421,14 +5440,16 @@ irdma_cfg_aeq(struct irdma_sc_dev *dev, u32 idx, bool enable)
void
sc_vsi_update_stats(struct irdma_sc_vsi *vsi)
{
struct irdma_gather_stats *gather_stats;
struct irdma_gather_stats *last_gather_stats;
struct irdma_dev_hw_stats *hw_stats = &vsi->pestat->hw_stats;
struct irdma_gather_stats *gather_stats =
vsi->pestat->gather_info.gather_stats_va;
struct irdma_gather_stats *last_gather_stats =
vsi->pestat->gather_info.last_gather_stats_va;
const struct irdma_hw_stat_map *map = vsi->dev->hw_stats_map;
u16 max_stat_idx = vsi->dev->hw_attrs.max_stat_idx;
gather_stats = vsi->pestat->gather_info.gather_stats_va;
last_gather_stats = vsi->pestat->gather_info.last_gather_stats_va;
irdma_update_stats(&vsi->pestat->hw_stats, gather_stats,
last_gather_stats, vsi->dev->hw_stats_map,
vsi->dev->hw_attrs.max_stat_idx);
irdma_update_stats(hw_stats, gather_stats, last_gather_stats,
map, max_stat_idx);
}
/**
@ -5498,7 +5519,6 @@ irdma_sc_dev_init(struct irdma_sc_dev *dev, struct irdma_device_init_info *info)
dev->hw_attrs.max_hw_outbound_msg_size = IRDMA_MAX_OUTBOUND_MSG_SIZE;
dev->hw_attrs.max_mr_size = IRDMA_MAX_MR_SIZE;
dev->hw_attrs.max_hw_inbound_msg_size = IRDMA_MAX_INBOUND_MSG_SIZE;
dev->hw_attrs.max_hw_device_pages = IRDMA_MAX_PUSH_PAGE_COUNT;
dev->hw_attrs.uk_attrs.max_hw_inline = IRDMA_MAX_INLINE_DATA_SIZE;
dev->hw_attrs.max_hw_wqes = IRDMA_MAX_WQ_ENTRIES;
dev->hw_attrs.max_qp_wr = IRDMA_MAX_QP_WRS(IRDMA_MAX_QUANTA_PER_WR);
@ -5526,7 +5546,6 @@ irdma_sc_dev_init(struct irdma_sc_dev *dev, struct irdma_device_init_info *info)
val, db_size);
return -ENODEV;
}
dev->db_addr = dev->hw->hw_addr + (uintptr_t)dev->hw_regs[IRDMA_DB_ADDR_OFFSET];
return ret_code;
}

3
sys/dev/irdma/irdma_defs.h
View File

@ -249,7 +249,6 @@ enum irdma_cqp_op_type {
IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE = 22,
IRDMA_OP_SUSPEND = 23,
IRDMA_OP_RESUME = 24,
IRDMA_OP_MANAGE_VCHNL_REQ_PBLE_BP = 25,
IRDMA_OP_QUERY_FPM_VAL = 26,
IRDMA_OP_COMMIT_FPM_VAL = 27,
IRDMA_OP_AH_CREATE = 28,
@ -292,7 +291,6 @@ enum irdma_cqp_op_type {
#define IRDMA_CQP_OP_DEALLOC_STAG 0x0d
#define IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE 0x0e
#define IRDMA_CQP_OP_MANAGE_ARP 0x0f
#define IRDMA_CQP_OP_MANAGE_VCHNL_REQ_PBLE_BP 0x10
#define IRDMA_CQP_OP_MANAGE_PUSH_PAGES 0x11
#define IRDMA_CQP_OP_QUERY_RDMA_FEATURES 0x12
#define IRDMA_CQP_OP_UPLOAD_CONTEXT 0x13
@ -849,7 +847,6 @@ enum irdma_cqp_op_type {
#define IRDMA_CQPSQ_UCTX_RAWFORMAT BIT_ULL(61)
#define IRDMA_CQPSQ_UCTX_FREEZEQP_S 62
#define IRDMA_CQPSQ_UCTX_FREEZEQP BIT_ULL(62)
#define IRDMA_CQPSQ_MHMC_VFIDX_S 0
#define IRDMA_CQPSQ_MHMC_VFIDX GENMASK_ULL(15, 0)
#define IRDMA_CQPSQ_MHMC_FREEPMFN_S 62

43
sys/dev/irdma/irdma_hw.c
View File

@ -394,6 +394,7 @@ irdma_process_aeq(struct irdma_pci_f *rf)
case IRDMA_AE_LLP_TOO_MANY_RETRIES:
case IRDMA_AE_LCE_QP_CATASTROPHIC:
case IRDMA_AE_LCE_FUNCTION_CATASTROPHIC:
case IRDMA_AE_LLP_TOO_MANY_RNRS:
case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG:
default:
irdma_dev_err(&iwdev->ibdev,
@ -490,7 +491,7 @@ irdma_save_msix_info(struct irdma_pci_f *rf)
{
struct irdma_qvlist_info *iw_qvlist;
struct irdma_qv_info *iw_qvinfo;
u32 ceq_idx;
u16 ceq_idx;
u32 i;
u32 size;
@ -500,8 +501,8 @@ irdma_save_msix_info(struct irdma_pci_f *rf)
}
size = sizeof(struct irdma_msix_vector) * rf->msix_count;
size += sizeof(struct irdma_qvlist_info);
size += sizeof(struct irdma_qv_info) * rf->msix_count - 1;
size += sizeof(*iw_qvlist);
size += sizeof(*iw_qvinfo) * rf->msix_count - 1;
rf->iw_msixtbl = kzalloc(size, GFP_KERNEL);
if (!rf->iw_msixtbl)
return -ENOMEM;
@ -600,6 +601,13 @@ irdma_destroy_irq(struct irdma_pci_f *rf,
dev->irq_ops->irdma_dis_irq(dev, msix_vec->idx);
irdma_free_irq(rf, msix_vec);
if (rf == dev_id) {
tasklet_kill(&rf->dpc_tasklet);
} else {
struct irdma_ceq *iwceq = (struct irdma_ceq *)dev_id;
tasklet_kill(&iwceq->dpc_tasklet);
}
}
/**
@ -964,13 +972,13 @@ irdma_create_cqp(struct irdma_pci_f *rf)
u16 maj_err, min_err;
int i, status;
cqp->cqp_requests = kcalloc(sqsize, sizeof(*cqp->cqp_requests), GFP_KERNEL);
memset(cqp->cqp_requests, 0, sqsize * sizeof(*cqp->cqp_requests));
cqp->cqp_requests = kcalloc(sqsize, sizeof(*cqp->cqp_requests),
GFP_KERNEL);
if (!cqp->cqp_requests)
return -ENOMEM;
cqp->scratch_array = kcalloc(sqsize, sizeof(*cqp->scratch_array), GFP_KERNEL);
memset(cqp->scratch_array, 0, sqsize * sizeof(*cqp->scratch_array));
cqp->scratch_array = kcalloc(sqsize, sizeof(*cqp->scratch_array),
GFP_KERNEL);
if (!cqp->scratch_array) {
status = -ENOMEM;
goto err_scratch;
@ -1190,7 +1198,7 @@ fail_intr:
*/
static int
irdma_cfg_ceq_vector(struct irdma_pci_f *rf, struct irdma_ceq *iwceq,
u32 ceq_id, struct irdma_msix_vector *msix_vec)
u16 ceq_id, struct irdma_msix_vector *msix_vec)
{
int status;
@ -1264,7 +1272,7 @@ irdma_cfg_aeq_vector(struct irdma_pci_f *rf)
*/
static int
irdma_create_ceq(struct irdma_pci_f *rf, struct irdma_ceq *iwceq,
u32 ceq_id, struct irdma_sc_vsi *vsi)
u16 ceq_id, struct irdma_sc_vsi *vsi)
{
int status;
struct irdma_ceq_init_info info = {0};
@ -1381,7 +1389,7 @@ static int
irdma_setup_ceqs(struct irdma_pci_f *rf, struct irdma_sc_vsi *vsi)
{
u32 i;
u32 ceq_id;
u16 ceq_id;
struct irdma_ceq *iwceq;
struct irdma_msix_vector *msix_vec;
int status;
@ -2212,14 +2220,19 @@ irdma_cqp_ce_handler(struct irdma_pci_f *rf, struct irdma_sc_cq *cq)
cqp_request = (struct irdma_cqp_request *)
(uintptr_t)info.scratch;
if (info.error && irdma_cqp_crit_err(dev, cqp_request->info.cqp_cmd,
if (info.error && irdma_cqp_crit_err(dev,
cqp_request->info.cqp_cmd,
info.maj_err_code,
info.min_err_code))
irdma_dev_err(&rf->iwdev->ibdev, "cqp opcode = 0x%x maj_err_code = 0x%x min_err_code = 0x%x\n",
info.op_code, info.maj_err_code, info.min_err_code);
irdma_dev_err(&rf->iwdev->ibdev,
"cqp opcode = 0x%x maj_err_code = 0x%x min_err_code = 0x%x\n",
info.op_code, info.maj_err_code,
info.min_err_code);
if (cqp_request) {
cqp_request->compl_info.maj_err_code = info.maj_err_code;
cqp_request->compl_info.min_err_code = info.min_err_code;
cqp_request->compl_info.maj_err_code =
info.maj_err_code;
cqp_request->compl_info.min_err_code =
info.min_err_code;
cqp_request->compl_info.op_ret_val = info.op_ret_val;
cqp_request->compl_info.error = info.error;
irdma_complete_cqp_request(&rf->cqp, cqp_request);

96
sys/dev/irdma/irdma_kcompat.c
View File

@ -160,6 +160,7 @@ err:
#define IRDMA_ALLOC_UCTX_MIN_REQ_LEN offsetofend(struct irdma_alloc_ucontext_req, rsvd8)
#define IRDMA_ALLOC_UCTX_MIN_RESP_LEN offsetofend(struct irdma_alloc_ucontext_resp, rsvd)
/**
* irdma_alloc_ucontext - Allocate the user context data structure
* @ibdev: ib device pointer
@ -228,6 +229,8 @@ irdma_alloc_ucontext(struct ib_device *ibdev, struct ib_udata *udata)
uresp.min_hw_cq_size = uk_attrs->min_hw_cq_size;
uresp.hw_rev = uk_attrs->hw_rev;
uresp.comp_mask |= IRDMA_ALLOC_UCTX_USE_RAW_ATTR;
uresp.min_hw_wq_size = uk_attrs->min_hw_wq_size;
uresp.comp_mask |= IRDMA_ALLOC_UCTX_MIN_HW_WQ_SIZE;
bar_off =
(uintptr_t)iwdev->rf->sc_dev.hw_regs[IRDMA_DB_ADDR_OFFSET];
@ -285,6 +288,7 @@ irdma_dealloc_ucontext(struct ib_ucontext *context)
}
#define IRDMA_ALLOC_PD_MIN_RESP_LEN offsetofend(struct irdma_alloc_pd_resp, rsvd)
/**
* irdma_alloc_pd - allocate protection domain
* @ibdev: IB device
@ -481,23 +485,41 @@ static int
irdma_create_ah_wait(struct irdma_pci_f *rf,
struct irdma_sc_ah *sc_ah, bool sleep)
{
int ret;
if (!sleep) {
int cnt = rf->sc_dev.hw_attrs.max_cqp_compl_wait_time_ms *
CQP_TIMEOUT_THRESHOLD;
struct irdma_cqp_request *cqp_request =
sc_ah->ah_info.cqp_request;
do {
irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq);
mdelay(1);
} while (!sc_ah->ah_info.ah_valid && --cnt);
} while (!READ_ONCE(cqp_request->request_done) && --cnt);
if (!cnt)
return -ETIMEDOUT;
if (cnt && !cqp_request->compl_info.op_ret_val) {
irdma_put_cqp_request(&rf->cqp, cqp_request);
sc_ah->ah_info.ah_valid = true;
} else {
ret = !cnt ? -ETIMEDOUT : -EINVAL;
irdma_dev_err(&rf->iwdev->ibdev, "CQP create AH error ret = %d opt_ret_val = %d",
ret, cqp_request->compl_info.op_ret_val);
irdma_put_cqp_request(&rf->cqp, cqp_request);
if (!cnt && !rf->reset) {
rf->reset = true;
rf->gen_ops.request_reset(rf);
}
return ret;
}
}
return 0;
}
#define IRDMA_CREATE_AH_MIN_RESP_LEN offsetofend(struct irdma_create_ah_resp, rsvd)
void
irdma_ether_copy(u8 *dmac, struct ib_ah_attr *attr)
{
@ -610,17 +632,15 @@ irdma_create_ah(struct ib_pd *ibpd,
goto err_gid_l2;
err = irdma_ah_cqp_op(iwdev->rf, sc_ah, IRDMA_OP_AH_CREATE,
sleep, irdma_gsi_ud_qp_ah_cb, sc_ah);
sleep, NULL, sc_ah);
if (err) {
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "CQP-OP Create AH fail");
goto err_gid_l2;
}
err = irdma_create_ah_wait(rf, sc_ah, sleep);
if (err) {
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_DEV, "CQP create AH timed out");
if (err)
goto err_gid_l2;
}
if (udata) {
uresp.ah_id = ah->sc_ah.ah_info.ah_idx;
@ -652,7 +672,7 @@ irdma_free_qp_rsrc(struct irdma_qp *iwqp)
u32 qp_num = iwqp->ibqp.qp_num;
irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp);
irdma_dealloc_push_page(rf, &iwqp->sc_qp);
irdma_dealloc_push_page(rf, iwqp);
if (iwqp->sc_qp.vsi) {
irdma_qp_rem_qos(&iwqp->sc_qp);
iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi,
@ -846,12 +866,17 @@ irdma_create_qp(struct ib_pd *ibpd,
if (udata) {
/* GEN_1 legacy support with libi40iw does not have expanded uresp struct */
if (udata->outlen < sizeof(uresp)) {
if (udata->outlen == IRDMA_CREATE_QP_MIN_RESP_LEN) {
uresp.lsmm = 1;
uresp.push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX_GEN_1;
} else {
if (rdma_protocol_iwarp(&iwdev->ibdev, 1))
if (rdma_protocol_iwarp(&iwdev->ibdev, 1)) {
uresp.lsmm = 1;
if (qp->qp_uk.start_wqe_idx) {
uresp.comp_mask |= IRDMA_CREATE_QP_USE_START_WQE_IDX;
uresp.start_wqe_idx = qp->qp_uk.start_wqe_idx;
}
}
}
uresp.actual_sq_size = init_info.qp_uk_init_info.sq_size;
uresp.actual_rq_size = init_info.qp_uk_init_info.rq_size;
@ -862,7 +887,7 @@ irdma_create_qp(struct ib_pd *ibpd,
min(sizeof(uresp), udata->outlen));
if (err_code) {
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS, "copy_to_udata failed\n");
kc_irdma_destroy_qp(&iwqp->ibqp, udata);
irdma_destroy_qp(&iwqp->ibqp);
return ERR_PTR(err_code);
}
}
@ -1357,50 +1382,6 @@ irdma_destroy_cq(struct ib_cq *ib_cq)
return 0;
}
/**
* irdma_alloc_mw - Allocate memory window
* @pd: Protection domain
* @type: Window type
* @udata: user data pointer
*/
struct ib_mw *
irdma_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
struct ib_udata *udata)
{
struct irdma_device *iwdev = to_iwdev(pd->device);
struct irdma_mr *iwmr;
int err_code;
u32 stag;
if (type != IB_MW_TYPE_1 && type != IB_MW_TYPE_2)
return ERR_PTR(-EINVAL);
iwmr = kzalloc(sizeof(*iwmr), GFP_KERNEL);
if (!iwmr)
return ERR_PTR(-ENOMEM);
stag = irdma_create_stag(iwdev);
if (!stag) {
kfree(iwmr);
return ERR_PTR(-ENOMEM);
}
iwmr->stag = stag;
iwmr->ibmw.rkey = stag;
iwmr->ibmw.pd = pd;
iwmr->ibmw.type = type;
iwmr->ibmw.device = pd->device;
err_code = irdma_hw_alloc_mw(iwdev, iwmr);
if (err_code) {
irdma_free_stag(iwdev, stag);
kfree(iwmr);
return ERR_PTR(err_code);
}
return &iwmr->ibmw;
}
/**
* kc_set_loc_seq_num_mss - Set local seq number and mss
* @cm_node: cm node info
@ -1569,7 +1550,7 @@ irdma_query_gid_roce(struct ib_device *ibdev, u8 port, int index,
{
int ret;
ret = rdma_query_gid(ibdev, port, index, gid);
ret = ib_get_cached_gid(ibdev, port, index, gid, NULL);
if (ret == -EAGAIN) {
memcpy(gid, &zgid, sizeof(*gid));
return 0;
@ -1861,9 +1842,6 @@ kc_set_rdma_uverbs_cmd_mask(struct irdma_device *iwdev)
BIT_ULL(IB_USER_VERBS_CMD_QUERY_QP) |
BIT_ULL(IB_USER_VERBS_CMD_POLL_CQ) |
BIT_ULL(IB_USER_VERBS_CMD_DESTROY_QP) |
BIT_ULL(IB_USER_VERBS_CMD_ALLOC_MW) |
BIT_ULL(IB_USER_VERBS_CMD_BIND_MW) |
BIT_ULL(IB_USER_VERBS_CMD_DEALLOC_MW) |
BIT_ULL(IB_USER_VERBS_CMD_POST_RECV) |
BIT_ULL(IB_USER_VERBS_CMD_POST_SEND);
iwdev->ibdev.uverbs_ex_cmd_mask =

13
sys/dev/irdma/irdma_main.h
View File

@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
* Copyright (c) 2015 - 2022 Intel Corporation
* Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@ -181,7 +181,7 @@ struct irdma_cqp_request {
void (*callback_fcn)(struct irdma_cqp_request *cqp_request);
void *param;
struct irdma_cqp_compl_info compl_info;
bool request_done; /* READ/WRITE_ONCE macros operate on it */
u8 request_done; /* READ/WRITE_ONCE macros operate on it */
bool waiting:1;
bool dynamic:1;
};
@ -232,7 +232,7 @@ struct irdma_msix_vector {
u32 idx;
u32 irq;
u32 cpu_affinity;
u32 ceq_id;
u16 ceq_id;
char name[IRDMA_IRQ_NAME_STR_LEN];
struct resource *res;
void *tag;
@ -374,7 +374,6 @@ struct irdma_device {
u32 roce_ackcreds;
u32 vendor_id;
u32 vendor_part_id;
u32 push_mode;
u32 rcv_wnd;
u16 mac_ip_table_idx;
u16 vsi_num;
@ -388,6 +387,7 @@ struct irdma_device {
bool override_ooo:1;
bool override_rd_fence_rate:1;
bool override_rtomin:1;
bool push_mode:1;
bool roce_mode:1;
bool roce_dcqcn_en:1;
bool dcb_vlan_mode:1;
@ -573,8 +573,8 @@ void irdma_gen_ae(struct irdma_pci_f *rf, struct irdma_sc_qp *qp,
void irdma_copy_ip_ntohl(u32 *dst, __be32 *src);
void irdma_copy_ip_htonl(__be32 *dst, u32 *src);
u16 irdma_get_vlan_ipv4(struct iw_cm_id *cm_id, u32 *addr);
if_t irdma_netdev_vlan_ipv6(struct iw_cm_id *cm_id, u32 *addr, u16 *vlan_id,
u8 *mac);
void irdma_get_vlan_mac_ipv6(struct iw_cm_id *cm_id, u32 *addr, u16 *vlan_id,
u8 *mac);
struct ib_mr *irdma_reg_phys_mr(struct ib_pd *ib_pd, u64 addr, u64 size,
int acc, u64 *iova_start);
int irdma_upload_qp_context(struct irdma_qp *iwqp, bool freeze, bool raw);
@ -586,7 +586,6 @@ int irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd,
bool wait,
void (*callback_fcn)(struct irdma_cqp_request *cqp_request),
void *cb_param);
void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request);
void irdma_udqp_qs_worker(struct work_struct *work);
bool irdma_cq_empty(struct irdma_cq *iwcq);
int irdma_netdevice_event(struct notifier_block *notifier, unsigned long event,

2
sys/dev/irdma/irdma_pble.h
View File

@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
* Copyright (c) 2015 - 2022 Intel Corporation
* Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU

2
sys/dev/irdma/irdma_protos.h
View File

@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
* Copyright (c) 2016 - 2022 Intel Corporation
* Copyright (c) 2016 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU

8
sys/dev/irdma/irdma_puda.c
View File

@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
* Copyright (c) 2015 - 2022 Intel Corporation
* Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@ -182,7 +182,7 @@ irdma_puda_alloc_buf(struct irdma_sc_dev *dev,
struct irdma_puda_buf *buf;
struct irdma_virt_mem buf_mem;
buf_mem.size = sizeof(struct irdma_puda_buf);
buf_mem.size = sizeof(*buf);
buf_mem.va = kzalloc(buf_mem.size, GFP_KERNEL);
if (!buf_mem.va)
return NULL;
@ -1010,7 +1010,7 @@ irdma_puda_allocbufs(struct irdma_puda_rsrc *rsrc, u32 count)
bool virtdma = false;
unsigned long flags;
buf_mem.size = count * sizeof(struct irdma_puda_buf);
buf_mem.size = count * sizeof(*buf);
buf_mem.va = kzalloc(buf_mem.size, GFP_KERNEL);
if (!buf_mem.va) {
irdma_debug(rsrc->dev, IRDMA_DEBUG_PUDA,
@ -1105,7 +1105,7 @@ irdma_puda_create_rsrc(struct irdma_sc_vsi *vsi,
struct irdma_virt_mem *vmem;
info->count = 1;
pudasize = sizeof(struct irdma_puda_rsrc);
pudasize = sizeof(*rsrc);
sqwridsize = info->sq_size * sizeof(struct irdma_sq_uk_wr_trk_info);
rqwridsize = info->rq_size * 8;
switch (info->type) {

29
sys/dev/irdma/irdma_type.h
View File

@ -42,6 +42,8 @@
#include "irdma_hmc.h"
#include "irdma_uda.h"
#include "irdma_ws.h"
#include "irdma_pble.h"
enum irdma_debug_flag {
IRDMA_DEBUG_NONE = 0x00000000,
IRDMA_DEBUG_ERR = 0x00000001,
@ -70,6 +72,8 @@ enum irdma_debug_flag {
IRDMA_DEBUG_ALL = 0xFFFFFFFF,
};
#define RSVD_OFFSET 0xFFFFFFFF
enum irdma_page_size {
IRDMA_PAGE_SIZE_4K = 0,
IRDMA_PAGE_SIZE_2M,
@ -472,7 +476,7 @@ struct irdma_sc_cq {
bool virtual_map:1;
bool check_overflow:1;
bool ceq_id_valid:1;
bool tph_en;
bool tph_en:1;
};
struct irdma_sc_qp {
@ -520,9 +524,9 @@ struct irdma_sc_qp {
};
struct irdma_stats_inst_info {
bool use_hmc_fcn_index;
u16 hmc_fn_id;
u16 stats_idx;
bool use_hmc_fcn_index:1;
};
struct irdma_up_info {
@ -570,7 +574,7 @@ struct irdma_qos {
u8 traffic_class;
u8 rel_bw;
u8 prio_type;
bool valid;
bool valid:1;
};
struct irdma_config_check {
@ -623,7 +627,6 @@ struct irdma_sc_dev {
__le64 *fpm_query_buf;
__le64 *fpm_commit_buf;
struct irdma_hw *hw;
u8 IOMEM *db_addr;
u32 IOMEM *wqe_alloc_db;
u32 IOMEM *cq_arm_db;
u32 IOMEM *aeq_alloc_db;
@ -649,8 +652,6 @@ struct irdma_sc_dev {
u32 debug_mask;
u16 num_vfs;
u16 hmc_fn_id;
u8 vf_id;
bool vchnl_up:1;
bool ceq_valid:1;
u8 pci_rev;
int (*ws_add)(struct irdma_sc_vsi *vsi, u8 user_pri);
@ -666,7 +667,7 @@ struct irdma_modify_cq_info {
u8 pbl_chunk_size;
u32 first_pm_pbl_idx;
bool virtual_map:1;
bool check_overflow;
bool check_overflow:1;
bool cq_resize:1;
};
@ -676,7 +677,7 @@ struct irdma_create_qp_info {
bool cq_num_valid:1;
bool arp_cache_idx_valid:1;
bool mac_valid:1;
bool force_lpb;
bool force_lpb:1;
u8 next_iwarp_state;
};
@ -709,7 +710,7 @@ struct irdma_ccq_cqe_info {
u16 maj_err_code;
u16 min_err_code;
u8 op_code;
bool error;
bool error:1;
};
struct irdma_qos_tc_info {
@ -751,7 +752,7 @@ struct irdma_vsi_init_info {
struct irdma_vsi_stats_info {
struct irdma_vsi_pestat *pestat;
u8 fcn_id;
bool alloc_stats_inst;
bool alloc_stats_inst:1;
};
struct irdma_device_init_info {
@ -789,7 +790,7 @@ struct irdma_aeq_init_info {
u32 *aeqe_base;
void *pbl_list;
u32 elem_cnt;
bool virtual_map;
bool virtual_map:1;
u8 pbl_chunk_size;
u32 first_pm_pbl_idx;
u32 msix_idx;
@ -856,7 +857,6 @@ struct irdma_roce_offload_info {
bool dcqcn_en:1;
bool rcv_no_icrc:1;
bool wr_rdresp_en:1;
bool bind_en:1;
bool fast_reg_en:1;
bool priv_mode_en:1;
bool rd_en:1;
@ -888,7 +888,6 @@ struct irdma_iwarp_offload_info {
bool snd_mark_en:1;
bool rcv_mark_en:1;
bool wr_rdresp_en:1;
bool bind_en:1;
bool fast_reg_en:1;
bool priv_mode_en:1;
bool rd_en:1;
@ -1130,12 +1129,12 @@ struct irdma_add_arp_cache_entry_info {
u8 mac_addr[ETHER_ADDR_LEN];
u32 reach_max;
u16 arp_index;
bool permanent;
bool permanent:1;
};
struct irdma_apbvt_info {
u16 port;
bool add;
bool add:1;
};
struct irdma_qhash_table_info {

2
sys/dev/irdma/irdma_uda.c
View File

@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
* Copyright (c) 2016 - 2022 Intel Corporation
* Copyright (c) 2016 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU

3
sys/dev/irdma/irdma_uda.h
View File

@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
* Copyright (c) 2016 - 2021 Intel Corporation
* Copyright (c) 2016 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@ -43,6 +43,7 @@ struct irdma_sc_cqp;
struct irdma_ah_info {
struct irdma_sc_vsi *vsi;
struct irdma_cqp_request *cqp_request;
u32 pd_idx;
u32 dst_arpindex;
u32 dest_ip_addr[4];

80
sys/dev/irdma/irdma_uk.c
View File

@ -45,16 +45,16 @@
* @valid: The wqe valid
*/
static void
irdma_set_fragment(__le64 * wqe, u32 offset, struct irdma_sge *sge,
irdma_set_fragment(__le64 * wqe, u32 offset, struct ib_sge *sge,
u8 valid)
{
if (sge) {
set_64bit_val(wqe, offset,
FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off));
FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->addr));
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
FIELD_PREP(IRDMAQPSQ_VALID, valid) |
FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge->len) |
FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge->stag));
FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge->length) |
FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge->lkey));
} else {
set_64bit_val(wqe, offset, 0);
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
@ -71,14 +71,14 @@ irdma_set_fragment(__le64 * wqe, u32 offset, struct irdma_sge *sge,
*/
static void
irdma_set_fragment_gen_1(__le64 * wqe, u32 offset,
struct irdma_sge *sge, u8 valid)
struct ib_sge *sge, u8 valid)
{
if (sge) {
set_64bit_val(wqe, offset,
FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->tag_off));
FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->addr));
set_64bit_val(wqe, offset + IRDMA_BYTE_8,
FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge->len) |
FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge->stag));
FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge->length) |
FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge->lkey));
} else {
set_64bit_val(wqe, offset, 0);
set_64bit_val(wqe, offset + IRDMA_BYTE_8, 0);
@ -209,8 +209,7 @@ irdma_qp_push_wqe(struct irdma_qp_uk *qp, __le64 * wqe, u16 quanta,
if (IRDMA_RING_CURRENT_HEAD(qp->initial_ring) !=
IRDMA_RING_CURRENT_TAIL(qp->sq_ring) &&
!qp->push_mode) {
if (post_sq)
irdma_uk_qp_post_wr(qp);
irdma_uk_qp_post_wr(qp);
} else {
push = (__le64 *) ((uintptr_t)qp->push_wqe +
(wqe_idx & 0x7) * 0x20);
@ -338,7 +337,7 @@ irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
return -EINVAL;
for (i = 0; i < op_info->num_lo_sges; i++)
total_size += op_info->lo_sg_list[i].len;
total_size += op_info->lo_sg_list[i].length;
read_fence |= info->read_fence;
@ -357,7 +356,7 @@ irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled;
set_64bit_val(wqe, IRDMA_BYTE_16,
FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr));
if (info->imm_data_valid) {
set_64bit_val(wqe, IRDMA_BYTE_0,
@ -386,7 +385,7 @@ irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
++addl_frag_cnt;
}
hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) |
FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid) |
FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt) |
@ -437,7 +436,7 @@ irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
return -EINVAL;
for (i = 0; i < op_info->num_lo_sges; i++)
total_size += op_info->lo_sg_list[i].len;
total_size += op_info->lo_sg_list[i].length;
ret_code = irdma_fragcnt_to_quanta_sq(op_info->num_lo_sges, &quanta);
if (ret_code)
@ -475,8 +474,8 @@ irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
++addl_frag_cnt;
}
set_64bit_val(wqe, IRDMA_BYTE_16,
FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr));
hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) |
FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) |
FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
FIELD_PREP(IRDMAQPSQ_OPCODE,
@ -525,7 +524,7 @@ irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
return -EINVAL;
for (i = 0; i < op_info->num_sges; i++)
total_size += op_info->sg_list[i].len;
total_size += op_info->sg_list[i].length;
if (info->imm_data_valid)
frag_cnt = op_info->num_sges + 1;
@ -604,15 +603,15 @@ irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
* @polarity: compatibility parameter
*/
static void
irdma_copy_inline_data_gen_1(u8 *wqe, struct irdma_sge *sge_list,
irdma_copy_inline_data_gen_1(u8 *wqe, struct ib_sge *sge_list,
u32 num_sges, u8 polarity)
{
u32 quanta_bytes_remaining = 16;
u32 i;
for (i = 0; i < num_sges; i++) {
u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].tag_off;
u32 sge_len = sge_list[i].len;
u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr;
u32 sge_len = sge_list[i].length;
while (sge_len) {
u32 bytes_copied;
@ -651,7 +650,7 @@ static inline u16 irdma_inline_data_size_to_quanta_gen_1(u32 data_size) {
* @polarity: polarity of wqe valid bit
*/
static void
irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list,
irdma_copy_inline_data(u8 *wqe, struct ib_sge *sge_list,
u32 num_sges, u8 polarity)
{
u8 inline_valid = polarity << IRDMA_INLINE_VALID_S;
@ -662,8 +661,8 @@ irdma_copy_inline_data(u8 *wqe, struct irdma_sge *sge_list,
wqe += 8;
for (i = 0; i < num_sges; i++) {
u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].tag_off;
u32 sge_len = sge_list[i].len;
u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr;
u32 sge_len = sge_list[i].length;
while (sge_len) {
u32 bytes_copied;
@ -743,7 +742,7 @@ irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
return -EINVAL;
for (i = 0; i < op_info->num_lo_sges; i++)
total_size += op_info->lo_sg_list[i].len;
total_size += op_info->lo_sg_list[i].length;
if (unlikely(total_size > qp->max_inline_data))
return -EINVAL;
@ -756,9 +755,9 @@ irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
qp->sq_wrtrk_array[wqe_idx].signaled = info->signaled;
read_fence |= info->read_fence;
set_64bit_val(wqe, IRDMA_BYTE_16,
FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.tag_off));
FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr));
hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.stag) |
hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) |
FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) |
FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt ? 1 : 0) |
@ -814,7 +813,7 @@ irdma_uk_inline_send(struct irdma_qp_uk *qp,
return -EINVAL;
for (i = 0; i < op_info->num_sges; i++)
total_size += op_info->sg_list[i].len;
total_size += op_info->sg_list[i].length;
if (unlikely(total_size > qp->max_inline_data))
return -EINVAL;
@ -879,7 +878,7 @@ irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
u64 hdr;
u32 wqe_idx;
bool local_fence = false;
struct irdma_sge sge = {0};
struct ib_sge sge = {0};
u16 quanta = IRDMA_QP_WQE_MIN_QUANTA;
info->push_wqe = qp->push_db ? true : false;
@ -890,7 +889,7 @@ irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
if (!wqe)
return -ENOSPC;
sge.stag = op_info->target_stag;
sge.lkey = op_info->target_stag;
qp->wqe_ops.iw_set_fragment(wqe, IRDMA_BYTE_0, &sge, 0);
set_64bit_val(wqe, IRDMA_BYTE_16, 0);
@ -1327,8 +1326,7 @@ irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq,
IRDMA_RING_MOVE_TAIL(cq->cq_ring);
set_64bit_val(cq->shadow_area, IRDMA_BYTE_0,
IRDMA_RING_CURRENT_HEAD(cq->cq_ring));
memset(info, 0,
sizeof(struct irdma_cq_poll_info));
memset(info, 0, sizeof(*info));
return irdma_uk_cq_poll_cmpl(cq, info);
}
}
@ -1403,7 +1401,6 @@ exit:
if (pring && IRDMA_RING_MORE_WORK(*pring))
move_cq_head = false;
}
if (move_cq_head) {
IRDMA_RING_MOVE_HEAD_NOCHECK(cq->cq_ring);
if (!IRDMA_RING_CURRENT_HEAD(cq->cq_ring))
@ -1484,10 +1481,12 @@ irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge,
int
irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *sqdepth)
{
u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift;
*sqdepth = irdma_round_up_wq((sq_size << shift) + IRDMA_SQ_RSVD);
if (*sqdepth < ((u32)uk_attrs->min_hw_wq_size << shift))
*sqdepth = uk_attrs->min_hw_wq_size << shift;
if (*sqdepth < min_size)
*sqdepth = min_size;
else if (*sqdepth > uk_attrs->max_hw_wq_quanta)
return -EINVAL;
@ -1501,10 +1500,12 @@ irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift, u32 *s
int
irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift, u32 *rqdepth)
{
u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift;
*rqdepth = irdma_round_up_wq((rq_size << shift) + IRDMA_RQ_RSVD);
if (*rqdepth < ((u32)uk_attrs->min_hw_wq_size << shift))
*rqdepth = uk_attrs->min_hw_wq_size << shift;
if (*rqdepth < min_size)
*rqdepth = min_size;
else if (*rqdepth > uk_attrs->max_hw_rq_quanta)
return -EINVAL;
@ -1535,9 +1536,10 @@ irdma_setup_connection_wqes(struct irdma_qp_uk *qp,
{
u16 move_cnt = 1;
if (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE)
if (info->start_wqe_idx)
move_cnt = info->start_wqe_idx;
else if (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE)
move_cnt = 3;
qp->conn_wqes = move_cnt;
IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, move_cnt);
IRDMA_RING_MOVE_TAIL_BY_COUNT(qp->sq_ring, move_cnt);
@ -1677,6 +1679,8 @@ irdma_uk_qp_init(struct irdma_qp_uk *qp, struct irdma_qp_uk_init_info *info)
qp->wqe_ops = iw_wqe_uk_ops_gen_1;
else
qp->wqe_ops = iw_wqe_uk_ops;
qp->start_wqe_idx = info->start_wqe_idx;
return ret_code;
}

35
sys/dev/irdma/irdma_user.h
View File

@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
* Copyright (c) 2015 - 2022 Intel Corporation
* Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@ -35,6 +35,8 @@
#ifndef IRDMA_USER_H
#define IRDMA_USER_H
#include <rdma/ib_verbs.h>
#define irdma_handle void *
#define irdma_adapter_handle irdma_handle
#define irdma_qp_handle irdma_handle
@ -48,7 +50,7 @@
#define irdma_access_privileges u32
#define irdma_physical_fragment u64
#define irdma_address_list u64 *
#define irdma_sgl struct irdma_sge *
#define irdma_sgl struct ib_sge *
#define IRDMA_MAX_MR_SIZE 0x200000000000ULL
@ -78,8 +80,6 @@
#define IRDMA_OP_TYPE_REC_IMM 0x3f
#define IRDMA_FLUSH_MAJOR_ERR 1
#define IRDMA_SRQFLUSH_RSVD_MAJOR_ERR 0xfffe
/* Async Events codes */
#define IRDMA_AE_AMP_UNALLOCATED_STAG 0x0102
#define IRDMA_AE_AMP_INVALID_STAG 0x0103
@ -140,6 +140,7 @@
#define IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST 0x0313
#define IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP 0x0314
#define IRDMA_AE_ROCE_RSP_LENGTH_ERROR 0x0316
#define IRDMA_AE_ROCE_REQ_LENGTH_ERROR 0x0318
#define IRDMA_AE_ROCE_EMPTY_MCG 0x0380
#define IRDMA_AE_ROCE_BAD_MC_IP_ADDR 0x0381
#define IRDMA_AE_ROCE_BAD_MC_QPID 0x0382
@ -160,6 +161,7 @@
#define IRDMA_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES 0x050b
#define IRDMA_AE_LLP_DOUBT_REACHABILITY 0x050c
#define IRDMA_AE_LLP_CONNECTION_ESTABLISHED 0x050e
#define IRDMA_AE_LLP_TOO_MANY_RNRS 0x050f
#define IRDMA_AE_RESOURCE_EXHAUSTION 0x0520
#define IRDMA_AE_RESET_SENT 0x0601
#define IRDMA_AE_TERMINATE_SENT 0x0602
@ -199,8 +201,7 @@ enum irdma_device_caps_const {
IRDMA_MAX_OUTBOUND_MSG_SIZE = 65537,
/* 64K +1 */
IRDMA_MAX_INBOUND_MSG_SIZE = 65537,
IRDMA_MAX_PUSH_PAGE_COUNT = 1024,
IRDMA_MAX_PE_ENA_VF_COUNT = 32,
IRDMA_MAX_PE_ENA_VF_COUNT = 32,
IRDMA_MAX_VF_FPM_ID = 47,
IRDMA_MAX_SQ_PAYLOAD_SIZE = 2145386496,
IRDMA_MAX_INLINE_DATA_SIZE = 101,
@ -227,6 +228,7 @@ enum irdma_flush_opcode {
FLUSH_RETRY_EXC_ERR,
FLUSH_MW_BIND_ERR,
FLUSH_REM_INV_REQ_ERR,
FLUSH_RNR_RETRY_EXC_ERR,
};
enum irdma_qp_event_type {
@ -280,12 +282,6 @@ struct irdma_cq_uk;
struct irdma_qp_uk_init_info;
struct irdma_cq_uk_init_info;
struct irdma_sge {
irdma_tagged_offset tag_off;
u32 len;
irdma_stag stag;
};
struct irdma_ring {
volatile u32 head;
volatile u32 tail; /* effective tail */
@ -317,13 +313,13 @@ struct irdma_post_rq_info {
struct irdma_rdma_write {
irdma_sgl lo_sg_list;
u32 num_lo_sges;
struct irdma_sge rem_addr;
struct ib_sge rem_addr;
};
struct irdma_rdma_read {
irdma_sgl lo_sg_list;
u32 num_lo_sges;
struct irdma_sge rem_addr;
struct ib_sge rem_addr;
};
struct irdma_bind_window {
@ -422,9 +418,9 @@ int irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
bool post_sq);
struct irdma_wqe_uk_ops {
void (*iw_copy_inline_data)(u8 *dest, struct irdma_sge *sge_list, u32 num_sges, u8 polarity);
void (*iw_copy_inline_data)(u8 *dest, struct ib_sge *sge_list, u32 num_sges, u8 polarity);
u16 (*iw_inline_data_size_to_quanta)(u32 data_size);
void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct irdma_sge *sge,
void (*iw_set_fragment)(__le64 *wqe, u32 offset, struct ib_sge *sge,
u8 valid);
void (*iw_set_mw_bind_wqe)(__le64 *wqe,
struct irdma_bind_window *op_info);
@ -490,6 +486,7 @@ struct irdma_qp_uk {
u8 rwqe_polarity;
u8 rq_wqe_size;
u8 rq_wqe_size_multiplier;
u8 start_wqe_idx;
bool deferred_flag:1;
bool push_mode:1; /* whether the last post wqe was pushed */
bool push_dropped:1;
@ -537,6 +534,7 @@ struct irdma_qp_uk_init_info {
u32 sq_depth;
u32 rq_depth;
u8 first_sq_wq;
u8 start_wqe_idx;
u8 type;
u8 sq_shift;
u8 rq_shift;
@ -625,10 +623,15 @@ static inline struct qp_err_code irdma_ae_to_qp_err_code(u16 ae_id)
case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:
case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:
case IRDMA_AE_ROCE_RSP_LENGTH_ERROR:
case IRDMA_AE_ROCE_REQ_LENGTH_ERROR:
case IRDMA_AE_IB_REMOTE_OP_ERROR:
qp_err.flush_code = FLUSH_REM_OP_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
break;
case IRDMA_AE_LLP_TOO_MANY_RNRS:
qp_err.flush_code = FLUSH_RNR_RETRY_EXC_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;
break;
case IRDMA_AE_LCE_QP_CATASTROPHIC:
qp_err.flush_code = FLUSH_FATAL_ERR;
qp_err.event_type = IRDMA_QP_EVENT_CATASTROPHIC;

53
sys/dev/irdma/irdma_utils.c
View File

@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
* Copyright (c) 2015 - 2022 Intel Corporation
* Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@ -164,6 +164,7 @@ static const struct ae_desc ae_desc_list[] = {
"Connection error: Doubt reachability (usually occurs after the max number of retries has been reached)"},
{IRDMA_AE_LLP_CONNECTION_ESTABLISHED,
"iWARP event: Connection established"},
{IRDMA_AE_LLP_TOO_MANY_RNRS, "RoCEv2: Too many RNR NACKs"},
{IRDMA_AE_RESOURCE_EXHAUSTION,
"QP error: Resource exhaustion"},
{IRDMA_AE_RESET_SENT,
@ -437,11 +438,11 @@ static void
irdma_free_pending_cqp_request(struct irdma_cqp *cqp,
struct irdma_cqp_request *cqp_request)
{
if (cqp_request->waiting) {
cqp_request->compl_info.error = true;
WRITE_ONCE(cqp_request->request_done, true);
cqp_request->compl_info.error = true;
WRITE_ONCE(cqp_request->request_done, true);
if (cqp_request->waiting)
wake_up(&cqp_request->waitq);
}
wait_event_timeout(cqp->remove_wq,
atomic_read(&cqp_request->refcnt) == 1, 1000);
irdma_put_cqp_request(cqp, cqp_request);
@ -558,8 +559,6 @@ static const char *const irdma_cqp_cmd_names[IRDMA_MAX_CQP_OPS] = {
[IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd",
[IRDMA_OP_SUSPEND] = "Suspend QP Cmd",
[IRDMA_OP_RESUME] = "Resume QP Cmd",
[IRDMA_OP_MANAGE_VCHNL_REQ_PBLE_BP] =
"Manage Virtual Channel Requester Function PBLE Backing Pages Cmd",
[IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd",
[IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd",
[IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd",
@ -591,7 +590,7 @@ static const struct irdma_cqp_err_info irdma_noncrit_err_list[] = {
{0xffff, 0x8007, "Modify QP Bad Close"},
{0xffff, 0x8009, "LLP Closed"},
{0xffff, 0x800a, "Reset Not Sent"},
{0xffff, 0x200, "Failover Pending"}
{0xffff, 0x0200, "Failover Pending"},
};
/**
@ -1055,15 +1054,16 @@ irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
/**
* irdma_dealloc_push_page - free a push page for qp
* @rf: RDMA PCI function
* @qp: hardware control qp
* @iwqp: QP pointer
*/
void
irdma_dealloc_push_page(struct irdma_pci_f *rf,
struct irdma_sc_qp *qp)
struct irdma_qp *iwqp)
{
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
int status;
struct irdma_sc_qp *qp = &iwqp->sc_qp;
if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX)
return;
@ -1564,16 +1564,6 @@ irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi)
del_timer_sync(&devstat->stats_timer);
}
/**
* irdma_process_stats - Checking for wrap and update stats
* @pestat: stats structure pointer
*/
static inline void
irdma_process_stats(struct irdma_vsi_pestat *pestat)
{
sc_vsi_update_stats(pestat->vsi);
}
/**
* irdma_process_cqp_stats - Checking for wrap and update stats
* @cqp_request: cqp_request structure pointer
@ -1583,7 +1573,7 @@ irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request)
{
struct irdma_vsi_pestat *pestat = cqp_request->param;
irdma_process_stats(pestat);
sc_vsi_update_stats(pestat->vsi);
}
/**
@ -1619,7 +1609,7 @@ irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev,
cqp_request->callback_fcn = irdma_process_cqp_stats;
status = irdma_handle_cqp_op(rf, cqp_request);
if (wait)
irdma_process_stats(pestat);
sc_vsi_update_stats(pestat->vsi);
irdma_put_cqp_request(&rf->cqp, cqp_request);
return status;
@ -1814,6 +1804,10 @@ irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd,
cqp_info->cqp_cmd = cmd;
cqp_info->post_sq = 1;
if (cmd == IRDMA_OP_AH_CREATE) {
if (!wait)
irdma_get_cqp_request(cqp_request);
sc_ah->ah_info.cqp_request = cqp_request;
cqp_info->in.u.ah_create.info = sc_ah->ah_info;
cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request;
cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp;
@ -1948,21 +1942,6 @@ irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah)
kfree(ah);
}
/**
* irdma_gsi_ud_qp_ah_cb - callback after creation of AH for GSI/ID QP
* @cqp_request: pointer to cqp_request of create AH
*/
void
irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request)
{
struct irdma_sc_ah *sc_ah = cqp_request->param;
if (!cqp_request->compl_info.op_ret_val)
sc_ah->ah_info.ah_valid = true;
else
sc_ah->ah_info.ah_valid = false;
}
/**
* irdma_prm_add_pble_mem - add moemory to pble resources
* @pprm: pble resource manager

202
sys/dev/irdma/irdma_verbs.c
View File

@ -72,7 +72,6 @@ irdma_query_device(struct ib_device *ibdev,
props->max_cq = rf->max_cq - rf->used_cqs;
props->max_cqe = rf->max_cqe - 1;
props->max_mr = rf->max_mr - rf->used_mrs;
props->max_mw = props->max_mr;
props->max_pd = rf->max_pd - rf->used_pds;
props->max_sge_rd = hw_attrs->uk_attrs.max_hw_read_sges;
props->max_qp_rd_atom = hw_attrs->max_hw_ird;
@ -500,6 +499,9 @@ irdma_setup_umode_qp(struct ib_udata *udata,
ukinfo->sq_size = ukinfo->sq_depth >> ukinfo->sq_shift;
ukinfo->rq_size = ukinfo->rq_depth >> ukinfo->rq_shift;
}
if (req.comp_mask & IRDMA_CREATE_QP_USE_START_WQE_IDX &&
iwdev->rf->sc_dev.hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE)
ukinfo->start_wqe_idx = 4;
irdma_setup_virt_qp(iwdev, iwqp, info);
return 0;
@ -642,7 +644,6 @@ irdma_roce_fill_and_set_qpctx_info(struct irdma_qp *iwqp,
roce_info->rd_en = true;
roce_info->wr_rdresp_en = true;
roce_info->bind_en = true;
roce_info->dcqcn_en = false;
roce_info->rtomin = iwdev->roce_rtomin;
@ -674,7 +675,6 @@ irdma_iw_fill_and_set_qpctx_info(struct irdma_qp *iwqp,
ether_addr_copy(iwarp_info->mac_addr, if_getlladdr(iwdev->netdev));
iwarp_info->rd_en = true;
iwarp_info->wr_rdresp_en = true;
iwarp_info->bind_en = true;
iwarp_info->ecn_en = true;
iwarp_info->rtomin = 5;
@ -705,6 +705,8 @@ irdma_validate_qp_attrs(struct ib_qp_init_attr *init_attr,
if (init_attr->cap.max_inline_data > uk_attrs->max_hw_inline ||
init_attr->cap.max_send_sge > uk_attrs->max_hw_wq_frags ||
init_attr->cap.max_send_wr > uk_attrs->max_hw_wq_quanta ||
init_attr->cap.max_recv_wr > uk_attrs->max_hw_rq_quanta ||
init_attr->cap.max_recv_sge > uk_attrs->max_hw_wq_frags)
return -EINVAL;
@ -759,8 +761,6 @@ irdma_get_ib_acc_flags(struct irdma_qp *iwqp)
}
if (iwqp->roce_info.rd_en)
acc_flags |= IB_ACCESS_REMOTE_READ;
if (iwqp->roce_info.bind_en)
acc_flags |= IB_ACCESS_MW_BIND;
} else {
if (iwqp->iwarp_info.wr_rdresp_en) {
acc_flags |= IB_ACCESS_LOCAL_WRITE;
@ -768,8 +768,6 @@ irdma_get_ib_acc_flags(struct irdma_qp *iwqp)
}
if (iwqp->iwarp_info.rd_en)
acc_flags |= IB_ACCESS_REMOTE_READ;
if (iwqp->iwarp_info.bind_en)
acc_flags |= IB_ACCESS_MW_BIND;
}
return acc_flags;
}
@ -1181,7 +1179,8 @@ irdma_modify_qp_roce(struct ib_qp *ibqp, struct ib_qp_attr *attr,
udata->outlen));
if (ret) {
irdma_remove_push_mmap_entries(iwqp);
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
irdma_debug(&iwdev->rf->sc_dev,
IRDMA_DEBUG_VERBS,
"copy_to_udata failed\n");
return ret;
}
@ -1647,8 +1646,10 @@ error:
/**
* irdma_get_mr_access - get hw MR access permissions from IB access flags
* @access: IB access flags
* @hw_rev: Hardware version
*/
static inline u16 irdma_get_mr_access(int access){
static inline u16 irdma_get_mr_access(int access, u8 hw_rev)
{
u16 hw_access = 0;
hw_access |= (access & IB_ACCESS_LOCAL_WRITE) ?
@ -1657,8 +1658,6 @@ static inline u16 irdma_get_mr_access(int access){
IRDMA_ACCESS_FLAGS_REMOTEWRITE : 0;
hw_access |= (access & IB_ACCESS_REMOTE_READ) ?
IRDMA_ACCESS_FLAGS_REMOTEREAD : 0;
hw_access |= (access & IB_ACCESS_MW_BIND) ?
IRDMA_ACCESS_FLAGS_BIND_WINDOW : 0;
hw_access |= (access & IB_ZERO_BASED) ?
IRDMA_ACCESS_FLAGS_ZERO_BASED : 0;
hw_access |= IRDMA_ACCESS_FLAGS_LOCALREAD;
@ -1893,81 +1892,6 @@ irdma_handle_q_mem(struct irdma_device *iwdev,
return err;
}
/**
* irdma_hw_alloc_mw - create the hw memory window
* @iwdev: irdma device
* @iwmr: pointer to memory window info
*/
int
irdma_hw_alloc_mw(struct irdma_device *iwdev, struct irdma_mr *iwmr)
{
struct irdma_mw_alloc_info *info;
struct irdma_pd *iwpd = to_iwpd(iwmr->ibmr.pd);
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
int status;
cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true);
if (!cqp_request)
return -ENOMEM;
cqp_info = &cqp_request->info;
info = &cqp_info->in.u.mw_alloc.info;
memset(info, 0, sizeof(*info));
if (iwmr->ibmw.type == IB_MW_TYPE_1)
info->mw_wide = true;
info->page_size = PAGE_SIZE;
info->mw_stag_index = iwmr->stag >> IRDMA_CQPSQ_STAG_IDX_S;
info->pd_id = iwpd->sc_pd.pd_id;
info->remote_access = true;
cqp_info->cqp_cmd = IRDMA_OP_MW_ALLOC;
cqp_info->post_sq = 1;
cqp_info->in.u.mw_alloc.dev = &iwdev->rf->sc_dev;
cqp_info->in.u.mw_alloc.scratch = (uintptr_t)cqp_request;
status = irdma_handle_cqp_op(iwdev->rf, cqp_request);
irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);
return status;
}
/**
* irdma_dealloc_mw - Dealloc memory window
* @ibmw: memory window structure.
*/
static int
irdma_dealloc_mw(struct ib_mw *ibmw)
{
struct ib_pd *ibpd = ibmw->pd;
struct irdma_pd *iwpd = to_iwpd(ibpd);
struct irdma_mr *iwmr = to_iwmr((struct ib_mr *)ibmw);
struct irdma_device *iwdev = to_iwdev(ibmw->device);
struct irdma_cqp_request *cqp_request;
struct cqp_cmds_info *cqp_info;
struct irdma_dealloc_stag_info *info;
cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, true);
if (!cqp_request)
return -ENOMEM;
cqp_info = &cqp_request->info;
info = &cqp_info->in.u.dealloc_stag.info;
memset(info, 0, sizeof(*info));
info->pd_id = iwpd->sc_pd.pd_id;
info->stag_idx = RS_64_1(ibmw->rkey, IRDMA_CQPSQ_STAG_IDX_S);
info->mr = false;
cqp_info->cqp_cmd = IRDMA_OP_DEALLOC_STAG;
cqp_info->post_sq = 1;
cqp_info->in.u.dealloc_stag.dev = &iwdev->rf->sc_dev;
cqp_info->in.u.dealloc_stag.scratch = (uintptr_t)cqp_request;
irdma_handle_cqp_op(iwdev->rf, cqp_request);
irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);
irdma_free_stag(iwdev, iwmr->stag);
kfree(iwmr);
return 0;
}
/**
* irdma_hw_alloc_stag - cqp command to allocate stag
* @iwdev: irdma device
@ -2088,7 +2012,8 @@ irdma_hwreg_mr(struct irdma_device *iwdev, struct irdma_mr *iwmr,
stag_info->stag_key = (u8)iwmr->stag;
stag_info->total_len = iwmr->len;
stag_info->all_memory = (pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY) ? true : false;
stag_info->access_rights = irdma_get_mr_access(access);
stag_info->access_rights = irdma_get_mr_access(access,
iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev);
stag_info->pd_id = iwpd->sc_pd.pd_id;
if (stag_info->access_rights & IRDMA_ACCESS_FLAGS_ZERO_BASED)
stag_info->addr_type = IRDMA_ADDR_TYPE_ZERO_BASED;
@ -2162,14 +2087,18 @@ irdma_free_iwmr(struct irdma_mr *iwmr)
}
/*
* irdma_reg_user_mr_type_mem - Handle memory registration @iwmr - irdma mr @access - access rights
* irdma_reg_user_mr_type_mem - Handle memory registration
* @iwmr - irdma mr
* @access - access rights
* @create_stag - flag to create stag or not
*/
static int
irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access)
irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access,
bool create_stag)
{
struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);
struct irdma_pbl *iwpbl = &iwmr->iwpbl;
u32 stag;
u32 stag = 0;
int err;
u8 lvl;
@ -2188,15 +2117,17 @@ irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access)
}
}
stag = irdma_create_stag(iwdev);
if (!stag) {
err = -ENOMEM;
goto free_pble;
}
if (create_stag) {
stag = irdma_create_stag(iwdev);
if (!stag) {
err = -ENOMEM;
goto free_pble;
}
iwmr->stag = stag;
iwmr->ibmr.rkey = stag;
iwmr->ibmr.lkey = stag;
iwmr->stag = stag;
iwmr->ibmr.rkey = stag;
iwmr->ibmr.lkey = stag;
}
iwmr->access = access;
err = irdma_hwreg_mr(iwdev, iwmr, access);
if (err)
@ -2205,7 +2136,8 @@ irdma_reg_user_mr_type_mem(struct irdma_mr *iwmr, int access)
return 0;
err_hwreg:
irdma_free_stag(iwdev, stag);
if (stag)
irdma_free_stag(iwdev, stag);
free_pble:
if (iwpbl->pble_alloc.level != PBLE_LEVEL_0 && iwpbl->pbl_allocated)
@ -2344,7 +2276,7 @@ irdma_reg_user_mr(struct ib_pd *pd, u64 start, u64 len,
break;
case IRDMA_MEMREG_TYPE_MEM:
err = irdma_reg_user_mr_type_mem(iwmr, access);
err = irdma_reg_user_mr_type_mem(iwmr, access, true);
if (err)
goto error;
@ -2421,10 +2353,8 @@ irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len,
{
struct irdma_device *iwdev = to_iwdev(iwmr->ibmr.device);
struct irdma_pbl *iwpbl = &iwmr->iwpbl;
struct irdma_pble_alloc *palloc = &iwpbl->pble_alloc;
struct ib_pd *pd = iwmr->ibmr.pd;
struct ib_umem *region;
u8 lvl;
int err;
region = ib_umem_get(pd->uobject->context, start, len, iwmr->access, 0);
@ -2445,35 +2375,14 @@ irdma_rereg_mr_trans(struct irdma_mr *iwmr, u64 start, u64 len,
iwmr->page_cnt = irdma_ib_umem_num_dma_blocks(region, iwmr->page_size,
virt);
lvl = iwmr->page_cnt != 1 ? PBLE_LEVEL_1 | PBLE_LEVEL_2 : PBLE_LEVEL_0;
err = irdma_setup_pbles(iwdev->rf, iwmr, lvl);
err = irdma_reg_user_mr_type_mem(iwmr, iwmr->access, false);
if (err)
goto error;
if (lvl) {
err = irdma_check_mr_contiguous(palloc,
iwmr->page_size);
if (err) {
irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
iwpbl->pbl_allocated = false;
}
}
err = irdma_hwreg_mr(iwdev, iwmr, iwmr->access);
if (err)
goto error;
goto err;
return &iwmr->ibmr;
error:
if (palloc->level != PBLE_LEVEL_0 && iwpbl->pbl_allocated) {
irdma_free_pble(iwdev->rf->pble_rsrc, palloc);
iwpbl->pbl_allocated = false;
}
err:
ib_umem_release(region);
iwmr->region = NULL;
return ERR_PTR(err);
}
@ -2580,25 +2489,6 @@ irdma_del_memlist(struct irdma_mr *iwmr,
}
}
/**
* irdma_copy_sg_list - copy sg list for qp
* @sg_list: copied into sg_list
* @sgl: copy from sgl
* @num_sges: count of sg entries
*/
static void
irdma_copy_sg_list(struct irdma_sge *sg_list, struct ib_sge *sgl,
int num_sges)
{
unsigned int i;
for (i = 0; i < num_sges; i++) {
sg_list[i].tag_off = sgl[i].addr;
sg_list[i].len = sgl[i].length;
sg_list[i].stag = sgl[i].lkey;
}
}
/**
* irdma_post_send - kernel application wr
* @ibqp: qp ptr for wr
@ -2659,7 +2549,7 @@ irdma_post_send(struct ib_qp *ibqp,
}
info.op.send.num_sges = ib_wr->num_sge;
info.op.send.sg_list = (struct irdma_sge *)ib_wr->sg_list;
info.op.send.sg_list = ib_wr->sg_list;
if (iwqp->ibqp.qp_type == IB_QPT_UD ||
iwqp->ibqp.qp_type == IB_QPT_GSI) {
ah = to_iwah(ud_wr(ib_wr)->ah);
@ -2690,8 +2580,8 @@ irdma_post_send(struct ib_qp *ibqp,
info.op.rdma_write.num_lo_sges = ib_wr->num_sge;
info.op.rdma_write.lo_sg_list = (void *)ib_wr->sg_list;
info.op.rdma_write.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr;
info.op.rdma_write.rem_addr.stag = rdma_wr(ib_wr)->rkey;
info.op.rdma_write.rem_addr.addr = rdma_wr(ib_wr)->remote_addr;
info.op.rdma_write.rem_addr.lkey = rdma_wr(ib_wr)->rkey;
if (ib_wr->send_flags & IB_SEND_INLINE)
err = irdma_uk_inline_rdma_write(ukqp, &info, false);
else
@ -2707,8 +2597,8 @@ irdma_post_send(struct ib_qp *ibqp,
break;
}
info.op_type = IRDMA_OP_TYPE_RDMA_READ;
info.op.rdma_read.rem_addr.tag_off = rdma_wr(ib_wr)->remote_addr;
info.op.rdma_read.rem_addr.stag = rdma_wr(ib_wr)->rkey;
info.op.rdma_read.rem_addr.addr = rdma_wr(ib_wr)->remote_addr;
info.op.rdma_read.rem_addr.lkey = rdma_wr(ib_wr)->rkey;
info.op.rdma_read.lo_sg_list = (void *)ib_wr->sg_list;
info.op.rdma_read.num_lo_sges = ib_wr->num_sge;
err = irdma_uk_rdma_read(ukqp, &info, inv_stag, false);
@ -2726,7 +2616,9 @@ irdma_post_send(struct ib_qp *ibqp,
stag_info.signaled = info.signaled;
stag_info.read_fence = info.read_fence;
stag_info.access_rights = irdma_get_mr_access(reg_wr(ib_wr)->access);
stag_info.access_rights =
irdma_get_mr_access(reg_wr(ib_wr)->access,
dev->hw_attrs.uk_attrs.hw_rev);
stag_info.stag_key = reg_wr(ib_wr)->key & 0xff;
stag_info.stag_idx = reg_wr(ib_wr)->key >> 8;
stag_info.page_size = reg_wr(ib_wr)->mr->page_size;
@ -2788,7 +2680,6 @@ irdma_post_recv(struct ib_qp *ibqp,
struct irdma_qp *iwqp = to_iwqp(ibqp);
struct irdma_qp_uk *ukqp = &iwqp->sc_qp.qp_uk;
struct irdma_post_rq_info post_recv = {0};
struct irdma_sge *sg_list = iwqp->sg_list;
unsigned long flags;
int err = 0;
@ -2801,8 +2692,7 @@ irdma_post_recv(struct ib_qp *ibqp,
}
post_recv.num_sges = ib_wr->num_sge;
post_recv.wr_id = ib_wr->wr_id;
irdma_copy_sg_list(sg_list, ib_wr->sg_list, ib_wr->num_sge);
post_recv.sg_list = sg_list;
post_recv.sg_list = ib_wr->sg_list;
err = irdma_uk_post_receive(ukqp, &post_recv);
if (err) {
irdma_debug(&iwqp->iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
@ -3217,7 +3107,7 @@ irdma_attach_mcast(struct ib_qp *ibqp, union ib_gid *ibgid, u16 lid)
if (!ipv6_addr_v4mapped((struct in6_addr *)ibgid)) {
irdma_copy_ip_ntohl(ip_addr,
sgid_addr.saddr_in6.sin6_addr.__u6_addr.__u6_addr32);
irdma_netdev_vlan_ipv6(iwqp->cm_id, ip_addr, &vlan_id, NULL);
irdma_get_vlan_mac_ipv6(iwqp->cm_id, ip_addr, &vlan_id, NULL);
ipv4 = false;
irdma_debug(&iwdev->rf->sc_dev, IRDMA_DEBUG_VERBS,
"qp_id=%d, IP6address=%x:%x:%x:%x\n", ibqp->qp_num,
@ -3440,12 +3330,10 @@ irdma_set_device_ops(struct ib_device *ibdev)
dev_ops->alloc_hw_stats = irdma_alloc_hw_stats;
dev_ops->alloc_mr = irdma_alloc_mr;
dev_ops->alloc_mw = irdma_alloc_mw;
dev_ops->alloc_pd = irdma_alloc_pd;
dev_ops->alloc_ucontext = irdma_alloc_ucontext;
dev_ops->create_cq = irdma_create_cq;
dev_ops->create_qp = irdma_create_qp;
dev_ops->dealloc_mw = irdma_dealloc_mw;
dev_ops->dealloc_pd = irdma_dealloc_pd;
dev_ops->dealloc_ucontext = irdma_dealloc_ucontext;
dev_ops->dereg_mr = irdma_dereg_mr;

4
sys/dev/irdma/irdma_verbs.h
View File

@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
* Copyright (c) 2015 - 2022 Intel Corporation
* Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
@ -254,7 +254,7 @@ struct irdma_qp {
struct irdma_dma_mem host_ctx;
struct timer_list terminate_timer;
struct irdma_pbl *iwpbl;
struct irdma_sge *sg_list;
struct ib_sge *sg_list;
struct irdma_dma_mem q2_ctx_mem;
struct irdma_dma_mem ietf_mem;
struct completion free_qp;

4
sys/dev/irdma/irdma_ws.c
View File

@ -57,7 +57,7 @@ irdma_alloc_node(struct irdma_sc_vsi *vsi,
struct irdma_ws_node *node;
u16 node_index = 0;
ws_mem.size = sizeof(struct irdma_ws_node);
ws_mem.size = sizeof(*node);
ws_mem.va = kzalloc(ws_mem.size, GFP_KERNEL);
if (!ws_mem.va)
return NULL;
@ -109,7 +109,7 @@ irdma_free_node(struct irdma_sc_vsi *vsi,
irdma_free_ws_node_id(vsi->dev, node->index);
ws_mem.va = node;
ws_mem.size = sizeof(struct irdma_ws_node);
ws_mem.size = sizeof(*node);
kfree(ws_mem.va);
}

2
sys/dev/irdma/irdma_ws.h
View File

@ -1,7 +1,7 @@
/*-
* SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
*
* Copyright (c) 2015 - 2022 Intel Corporation
* Copyright (c) 2015 - 2023 Intel Corporation
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU