1#include <linux/socket.h>
2#include <linux/in.h>
3#include <linux/in6.h>
4#include <rdma/ib_verbs.h>
5#include <rdma/rdma_cm.h>
6
7#define DRV_NAME	"isert"
8#define PFX		DRV_NAME ": "
9
10#define isert_dbg(fmt, arg...)				 \
11	do {						 \
12		if (unlikely(isert_debug_level > 2))	 \
13			printk(KERN_DEBUG PFX "%s: " fmt,\
14				__func__ , ## arg);	 \
15	} while (0)
16
17#define isert_warn(fmt, arg...)				\
18	do {						\
19		if (unlikely(isert_debug_level > 0))	\
20			pr_warn(PFX "%s: " fmt,         \
21				__func__ , ## arg);	\
22	} while (0)
23
24#define isert_info(fmt, arg...)				\
25	do {						\
26		if (unlikely(isert_debug_level > 1))	\
27			pr_info(PFX "%s: " fmt,         \
28				__func__ , ## arg);	\
29	} while (0)
30
31#define isert_err(fmt, arg...) \
32	pr_err(PFX "%s: " fmt, __func__ , ## arg)
33
34#define ISCSI_ISER_SG_TABLESIZE		256
35#define ISER_FASTREG_LI_WRID		0xffffffffffffffffULL
36#define ISER_BEACON_WRID               0xfffffffffffffffeULL
37
38enum isert_desc_type {
39	ISCSI_TX_CONTROL,
40	ISCSI_TX_DATAIN
41};
42
43enum iser_ib_op_code {
44	ISER_IB_RECV,
45	ISER_IB_SEND,
46	ISER_IB_RDMA_WRITE,
47	ISER_IB_RDMA_READ,
48};
49
50enum iser_conn_state {
51	ISER_CONN_INIT,
52	ISER_CONN_UP,
53	ISER_CONN_BOUND,
54	ISER_CONN_FULL_FEATURE,
55	ISER_CONN_TERMINATING,
56	ISER_CONN_DOWN,
57};
58
59struct iser_rx_desc {
60	struct iser_hdr iser_header;
61	struct iscsi_hdr iscsi_header;
62	char		data[ISER_RECV_DATA_SEG_LEN];
63	u64		dma_addr;
64	struct ib_sge	rx_sg;
65	char		pad[ISER_RX_PAD_SIZE];
66} __packed;
67
68struct iser_tx_desc {
69	struct iser_hdr iser_header;
70	struct iscsi_hdr iscsi_header;
71	enum isert_desc_type type;
72	u64		dma_addr;
73	struct ib_sge	tx_sg[2];
74	int		num_sge;
75	struct isert_cmd *isert_cmd;
76	struct ib_send_wr send_wr;
77} __packed;
78
79enum isert_indicator {
80	ISERT_PROTECTED		= 1 << 0,
81	ISERT_DATA_KEY_VALID	= 1 << 1,
82	ISERT_PROT_KEY_VALID	= 1 << 2,
83	ISERT_SIG_KEY_VALID	= 1 << 3,
84};
85
86struct pi_context {
87	struct ib_mr		       *prot_mr;
88	struct ib_fast_reg_page_list   *prot_frpl;
89	struct ib_mr		       *sig_mr;
90};
91
92struct fast_reg_descriptor {
93	struct list_head		list;
94	struct ib_mr		       *data_mr;
95	struct ib_fast_reg_page_list   *data_frpl;
96	u8				ind;
97	struct pi_context	       *pi_ctx;
98};
99
100struct isert_data_buf {
101	struct scatterlist     *sg;
102	int			nents;
103	u32			sg_off;
104	u32			len; /* cur_rdma_length */
105	u32			offset;
106	unsigned int		dma_nents;
107	enum dma_data_direction dma_dir;
108};
109
110enum {
111	DATA = 0,
112	PROT = 1,
113	SIG = 2,
114};
115
116struct isert_rdma_wr {
117	struct list_head	wr_list;
118	struct isert_cmd	*isert_cmd;
119	enum iser_ib_op_code	iser_ib_op;
120	struct ib_sge		*ib_sge;
121	struct ib_sge		s_ib_sge;
122	int			send_wr_num;
123	struct ib_send_wr	*send_wr;
124	struct ib_send_wr	s_send_wr;
125	struct ib_sge		ib_sg[3];
126	struct isert_data_buf	data;
127	struct isert_data_buf	prot;
128	struct fast_reg_descriptor *fr_desc;
129};
130
131struct isert_cmd {
132	uint32_t		read_stag;
133	uint32_t		write_stag;
134	uint64_t		read_va;
135	uint64_t		write_va;
136	u64			pdu_buf_dma;
137	u32			pdu_buf_len;
138	u32			read_va_off;
139	u32			write_va_off;
140	u32			rdma_wr_num;
141	struct isert_conn	*conn;
142	struct iscsi_cmd	*iscsi_cmd;
143	struct iser_tx_desc	tx_desc;
144	struct isert_rdma_wr	rdma_wr;
145	struct work_struct	comp_work;
146};
147
148struct isert_device;
149
150struct isert_conn {
151	enum iser_conn_state	state;
152	int			post_recv_buf_count;
153	u32			responder_resources;
154	u32			initiator_depth;
155	bool			pi_support;
156	u32			max_sge;
157	char			*login_buf;
158	char			*login_req_buf;
159	char			*login_rsp_buf;
160	u64			login_req_dma;
161	int			login_req_len;
162	u64			login_rsp_dma;
163	unsigned int		rx_desc_head;
164	struct iser_rx_desc	*rx_descs;
165	struct ib_recv_wr	rx_wr[ISERT_MIN_POSTED_RX];
166	struct iscsi_conn	*conn;
167	struct list_head	accept_node;
168	struct completion	login_comp;
169	struct completion	login_req_comp;
170	struct iser_tx_desc	login_tx_desc;
171	struct rdma_cm_id	*cm_id;
172	struct ib_qp		*qp;
173	struct isert_device	*device;
174	struct mutex		mutex;
175	struct completion	wait;
176	struct completion	wait_comp_err;
177	struct kref		kref;
178	struct list_head	fr_pool;
179	int			fr_pool_size;
180	/* lock to protect fastreg pool */
181	spinlock_t		pool_lock;
182	struct work_struct	release_work;
183	struct ib_recv_wr       beacon;
184	bool                    logout_posted;
185};
186
187#define ISERT_MAX_CQ 64
188
189/**
190 * struct isert_comp - iSER completion context
191 *
192 * @device:     pointer to device handle
193 * @cq:         completion queue
194 * @wcs:        work completion array
195 * @active_qps: Number of active QPs attached
196 *              to completion context
197 * @work:       completion work handle
198 */
199struct isert_comp {
200	struct isert_device     *device;
201	struct ib_cq		*cq;
202	struct ib_wc		 wcs[16];
203	int                      active_qps;
204	struct work_struct	 work;
205};
206
207struct isert_device {
208	int			use_fastreg;
209	bool			pi_capable;
210	int			refcount;
211	struct ib_device	*ib_device;
212	struct ib_pd		*pd;
213	struct ib_mr		*mr;
214	struct isert_comp	*comps;
215	int                     comps_used;
216	struct list_head	dev_node;
217	struct ib_device_attr	dev_attr;
218	int			(*reg_rdma_mem)(struct iscsi_conn *conn,
219						    struct iscsi_cmd *cmd,
220						    struct isert_rdma_wr *wr);
221	void			(*unreg_rdma_mem)(struct isert_cmd *isert_cmd,
222						  struct isert_conn *isert_conn);
223};
224
225struct isert_np {
226	struct iscsi_np         *np;
227	struct semaphore	np_sem;
228	struct rdma_cm_id	*np_cm_id;
229	struct mutex		np_accept_mutex;
230	struct list_head	np_accept_list;
231	struct completion	np_login_comp;
232};
233