1/*
2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses.  You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 *     Redistribution and use in source and binary forms, with or
11 *     without modification, are permitted provided that the following
12 *     conditions are met:
13 *
14 *      - Redistributions of source code must retain the above
15 *        copyright notice, this list of conditions and the following
16 *        disclaimer.
17 *
18 *      - Redistributions in binary form must reproduce the above
19 *        copyright notice, this list of conditions and the following
20 *        disclaimer in the documentation and/or other materials
21 *        provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33
34#include <linux/gfp.h>
35#include <linux/export.h>
36#include <linux/mlx5/cmd.h>
37#include <linux/mlx5/qp.h>
38#include <linux/mlx5/driver.h>
39
40#include "mlx5_core.h"
41
42static struct mlx5_core_rsc_common *mlx5_get_rsc(struct mlx5_core_dev *dev,
43						 u32 rsn)
44{
45	struct mlx5_qp_table *table = &dev->priv.qp_table;
46	struct mlx5_core_rsc_common *common;
47
48	spin_lock(&table->lock);
49
50	common = radix_tree_lookup(&table->tree, rsn);
51	if (common)
52		atomic_inc(&common->refcount);
53
54	spin_unlock(&table->lock);
55
56	if (!common) {
57		mlx5_core_warn(dev, "Async event for bogus resource 0x%x\n",
58			       rsn);
59		return NULL;
60	}
61	return common;
62}
63
64void mlx5_core_put_rsc(struct mlx5_core_rsc_common *common)
65{
66	if (atomic_dec_and_test(&common->refcount))
67		complete(&common->free);
68}
69
70void mlx5_rsc_event(struct mlx5_core_dev *dev, u32 rsn, int event_type)
71{
72	struct mlx5_core_rsc_common *common = mlx5_get_rsc(dev, rsn);
73	struct mlx5_core_qp *qp;
74
75	if (!common)
76		return;
77
78	switch (common->res) {
79	case MLX5_RES_QP:
80		qp = (struct mlx5_core_qp *)common;
81		qp->event(qp, event_type);
82		break;
83
84	default:
85		mlx5_core_warn(dev, "invalid resource type for 0x%x\n", rsn);
86	}
87
88	mlx5_core_put_rsc(common);
89}
90
91#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
92void mlx5_eq_pagefault(struct mlx5_core_dev *dev, struct mlx5_eqe *eqe)
93{
94	struct mlx5_eqe_page_fault *pf_eqe = &eqe->data.page_fault;
95	int qpn = be32_to_cpu(pf_eqe->flags_qpn) & MLX5_QPN_MASK;
96	struct mlx5_core_rsc_common *common = mlx5_get_rsc(dev, qpn);
97	struct mlx5_core_qp *qp =
98		container_of(common, struct mlx5_core_qp, common);
99	struct mlx5_pagefault pfault;
100
101	if (!qp) {
102		mlx5_core_warn(dev, "ODP event for non-existent QP %06x\n",
103			       qpn);
104		return;
105	}
106
107	pfault.event_subtype = eqe->sub_type;
108	pfault.flags = (be32_to_cpu(pf_eqe->flags_qpn) >> MLX5_QPN_BITS) &
109		(MLX5_PFAULT_REQUESTOR | MLX5_PFAULT_WRITE | MLX5_PFAULT_RDMA);
110	pfault.bytes_committed = be32_to_cpu(
111		pf_eqe->bytes_committed);
112
113	mlx5_core_dbg(dev,
114		      "PAGE_FAULT: subtype: 0x%02x, flags: 0x%02x,\n",
115		      eqe->sub_type, pfault.flags);
116
117	switch (eqe->sub_type) {
118	case MLX5_PFAULT_SUBTYPE_RDMA:
119		/* RDMA based event */
120		pfault.rdma.r_key =
121			be32_to_cpu(pf_eqe->rdma.r_key);
122		pfault.rdma.packet_size =
123			be16_to_cpu(pf_eqe->rdma.packet_length);
124		pfault.rdma.rdma_op_len =
125			be32_to_cpu(pf_eqe->rdma.rdma_op_len);
126		pfault.rdma.rdma_va =
127			be64_to_cpu(pf_eqe->rdma.rdma_va);
128		mlx5_core_dbg(dev,
129			      "PAGE_FAULT: qpn: 0x%06x, r_key: 0x%08x,\n",
130			      qpn, pfault.rdma.r_key);
131		mlx5_core_dbg(dev,
132			      "PAGE_FAULT: rdma_op_len: 0x%08x,\n",
133			      pfault.rdma.rdma_op_len);
134		mlx5_core_dbg(dev,
135			      "PAGE_FAULT: rdma_va: 0x%016llx,\n",
136			      pfault.rdma.rdma_va);
137		mlx5_core_dbg(dev,
138			      "PAGE_FAULT: bytes_committed: 0x%06x\n",
139			      pfault.bytes_committed);
140		break;
141
142	case MLX5_PFAULT_SUBTYPE_WQE:
143		/* WQE based event */
144		pfault.wqe.wqe_index =
145			be16_to_cpu(pf_eqe->wqe.wqe_index);
146		pfault.wqe.packet_size =
147			be16_to_cpu(pf_eqe->wqe.packet_length);
148		mlx5_core_dbg(dev,
149			      "PAGE_FAULT: qpn: 0x%06x, wqe_index: 0x%04x,\n",
150			      qpn, pfault.wqe.wqe_index);
151		mlx5_core_dbg(dev,
152			      "PAGE_FAULT: bytes_committed: 0x%06x\n",
153			      pfault.bytes_committed);
154		break;
155
156	default:
157		mlx5_core_warn(dev,
158			       "Unsupported page fault event sub-type: 0x%02hhx, QP %06x\n",
159			       eqe->sub_type, qpn);
160		/* Unsupported page faults should still be resolved by the
161		 * page fault handler
162		 */
163	}
164
165	if (qp->pfault_handler) {
166		qp->pfault_handler(qp, &pfault);
167	} else {
168		mlx5_core_err(dev,
169			      "ODP event for QP %08x, without a fault handler in QP\n",
170			      qpn);
171		/* Page fault will remain unresolved. QP will hang until it is
172		 * destroyed
173		 */
174	}
175
176	mlx5_core_put_rsc(common);
177}
178#endif
179
180int mlx5_core_create_qp(struct mlx5_core_dev *dev,
181			struct mlx5_core_qp *qp,
182			struct mlx5_create_qp_mbox_in *in,
183			int inlen)
184{
185	struct mlx5_qp_table *table = &dev->priv.qp_table;
186	struct mlx5_create_qp_mbox_out out;
187	struct mlx5_destroy_qp_mbox_in din;
188	struct mlx5_destroy_qp_mbox_out dout;
189	int err;
190
191	memset(&out, 0, sizeof(out));
192	in->hdr.opcode = cpu_to_be16(MLX5_CMD_OP_CREATE_QP);
193
194	err = mlx5_cmd_exec(dev, in, inlen, &out, sizeof(out));
195	if (err) {
196		mlx5_core_warn(dev, "ret %d\n", err);
197		return err;
198	}
199
200	if (out.hdr.status) {
201		mlx5_core_warn(dev, "current num of QPs 0x%x\n",
202			       atomic_read(&dev->num_qps));
203		return mlx5_cmd_status_to_err(&out.hdr);
204	}
205
206	qp->qpn = be32_to_cpu(out.qpn) & 0xffffff;
207	mlx5_core_dbg(dev, "qpn = 0x%x\n", qp->qpn);
208
209	qp->common.res = MLX5_RES_QP;
210	spin_lock_irq(&table->lock);
211	err = radix_tree_insert(&table->tree, qp->qpn, qp);
212	spin_unlock_irq(&table->lock);
213	if (err) {
214		mlx5_core_warn(dev, "err %d\n", err);
215		goto err_cmd;
216	}
217
218	err = mlx5_debug_qp_add(dev, qp);
219	if (err)
220		mlx5_core_dbg(dev, "failed adding QP 0x%x to debug file system\n",
221			      qp->qpn);
222
223	qp->pid = current->pid;
224	atomic_set(&qp->common.refcount, 1);
225	atomic_inc(&dev->num_qps);
226	init_completion(&qp->common.free);
227
228	return 0;
229
230err_cmd:
231	memset(&din, 0, sizeof(din));
232	memset(&dout, 0, sizeof(dout));
233	din.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DESTROY_QP);
234	din.qpn = cpu_to_be32(qp->qpn);
235	mlx5_cmd_exec(dev, &din, sizeof(din), &out, sizeof(dout));
236
237	return err;
238}
239EXPORT_SYMBOL_GPL(mlx5_core_create_qp);
240
241int mlx5_core_destroy_qp(struct mlx5_core_dev *dev,
242			 struct mlx5_core_qp *qp)
243{
244	struct mlx5_destroy_qp_mbox_in in;
245	struct mlx5_destroy_qp_mbox_out out;
246	struct mlx5_qp_table *table = &dev->priv.qp_table;
247	unsigned long flags;
248	int err;
249
250	mlx5_debug_qp_remove(dev, qp);
251
252	spin_lock_irqsave(&table->lock, flags);
253	radix_tree_delete(&table->tree, qp->qpn);
254	spin_unlock_irqrestore(&table->lock, flags);
255
256	mlx5_core_put_rsc((struct mlx5_core_rsc_common *)qp);
257	wait_for_completion(&qp->common.free);
258
259	memset(&in, 0, sizeof(in));
260	memset(&out, 0, sizeof(out));
261	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DESTROY_QP);
262	in.qpn = cpu_to_be32(qp->qpn);
263	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
264	if (err)
265		return err;
266
267	if (out.hdr.status)
268		return mlx5_cmd_status_to_err(&out.hdr);
269
270	atomic_dec(&dev->num_qps);
271	return 0;
272}
273EXPORT_SYMBOL_GPL(mlx5_core_destroy_qp);
274
275int mlx5_core_qp_modify(struct mlx5_core_dev *dev, enum mlx5_qp_state cur_state,
276			enum mlx5_qp_state new_state,
277			struct mlx5_modify_qp_mbox_in *in, int sqd_event,
278			struct mlx5_core_qp *qp)
279{
280	static const u16 optab[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE] = {
281		[MLX5_QP_STATE_RST] = {
282			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
283			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
284			[MLX5_QP_STATE_INIT]	= MLX5_CMD_OP_RST2INIT_QP,
285		},
286		[MLX5_QP_STATE_INIT]  = {
287			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
288			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
289			[MLX5_QP_STATE_INIT]	= MLX5_CMD_OP_INIT2INIT_QP,
290			[MLX5_QP_STATE_RTR]	= MLX5_CMD_OP_INIT2RTR_QP,
291		},
292		[MLX5_QP_STATE_RTR]   = {
293			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
294			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
295			[MLX5_QP_STATE_RTS]	= MLX5_CMD_OP_RTR2RTS_QP,
296		},
297		[MLX5_QP_STATE_RTS]   = {
298			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
299			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
300			[MLX5_QP_STATE_RTS]	= MLX5_CMD_OP_RTS2RTS_QP,
301		},
302		[MLX5_QP_STATE_SQD] = {
303			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
304			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
305		},
306		[MLX5_QP_STATE_SQER] = {
307			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
308			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
309			[MLX5_QP_STATE_RTS]	= MLX5_CMD_OP_SQERR2RTS_QP,
310		},
311		[MLX5_QP_STATE_ERR] = {
312			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
313			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
314		}
315	};
316
317	struct mlx5_modify_qp_mbox_out out;
318	int err = 0;
319	u16 op;
320
321	if (cur_state >= MLX5_QP_NUM_STATE || new_state >= MLX5_QP_NUM_STATE ||
322	    !optab[cur_state][new_state])
323		return -EINVAL;
324
325	memset(&out, 0, sizeof(out));
326	op = optab[cur_state][new_state];
327	in->hdr.opcode = cpu_to_be16(op);
328	in->qpn = cpu_to_be32(qp->qpn);
329	err = mlx5_cmd_exec(dev, in, sizeof(*in), &out, sizeof(out));
330	if (err)
331		return err;
332
333	return mlx5_cmd_status_to_err(&out.hdr);
334}
335EXPORT_SYMBOL_GPL(mlx5_core_qp_modify);
336
337void mlx5_init_qp_table(struct mlx5_core_dev *dev)
338{
339	struct mlx5_qp_table *table = &dev->priv.qp_table;
340
341	spin_lock_init(&table->lock);
342	INIT_RADIX_TREE(&table->tree, GFP_ATOMIC);
343	mlx5_qp_debugfs_init(dev);
344}
345
346void mlx5_cleanup_qp_table(struct mlx5_core_dev *dev)
347{
348	mlx5_qp_debugfs_cleanup(dev);
349}
350
351int mlx5_core_qp_query(struct mlx5_core_dev *dev, struct mlx5_core_qp *qp,
352		       struct mlx5_query_qp_mbox_out *out, int outlen)
353{
354	struct mlx5_query_qp_mbox_in in;
355	int err;
356
357	memset(&in, 0, sizeof(in));
358	memset(out, 0, outlen);
359	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_QUERY_QP);
360	in.qpn = cpu_to_be32(qp->qpn);
361	err = mlx5_cmd_exec(dev, &in, sizeof(in), out, outlen);
362	if (err)
363		return err;
364
365	if (out->hdr.status)
366		return mlx5_cmd_status_to_err(&out->hdr);
367
368	return err;
369}
370EXPORT_SYMBOL_GPL(mlx5_core_qp_query);
371
372int mlx5_core_xrcd_alloc(struct mlx5_core_dev *dev, u32 *xrcdn)
373{
374	struct mlx5_alloc_xrcd_mbox_in in;
375	struct mlx5_alloc_xrcd_mbox_out out;
376	int err;
377
378	memset(&in, 0, sizeof(in));
379	memset(&out, 0, sizeof(out));
380	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_ALLOC_XRCD);
381	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
382	if (err)
383		return err;
384
385	if (out.hdr.status)
386		err = mlx5_cmd_status_to_err(&out.hdr);
387	else
388		*xrcdn = be32_to_cpu(out.xrcdn);
389
390	return err;
391}
392EXPORT_SYMBOL_GPL(mlx5_core_xrcd_alloc);
393
394int mlx5_core_xrcd_dealloc(struct mlx5_core_dev *dev, u32 xrcdn)
395{
396	struct mlx5_dealloc_xrcd_mbox_in in;
397	struct mlx5_dealloc_xrcd_mbox_out out;
398	int err;
399
400	memset(&in, 0, sizeof(in));
401	memset(&out, 0, sizeof(out));
402	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_DEALLOC_XRCD);
403	in.xrcdn = cpu_to_be32(xrcdn);
404	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
405	if (err)
406		return err;
407
408	if (out.hdr.status)
409		err = mlx5_cmd_status_to_err(&out.hdr);
410
411	return err;
412}
413EXPORT_SYMBOL_GPL(mlx5_core_xrcd_dealloc);
414
415#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
416int mlx5_core_page_fault_resume(struct mlx5_core_dev *dev, u32 qpn,
417				u8 flags, int error)
418{
419	struct mlx5_page_fault_resume_mbox_in in;
420	struct mlx5_page_fault_resume_mbox_out out;
421	int err;
422
423	memset(&in, 0, sizeof(in));
424	memset(&out, 0, sizeof(out));
425	in.hdr.opcode = cpu_to_be16(MLX5_CMD_OP_PAGE_FAULT_RESUME);
426	in.hdr.opmod = 0;
427	flags &= (MLX5_PAGE_FAULT_RESUME_REQUESTOR |
428		  MLX5_PAGE_FAULT_RESUME_WRITE	   |
429		  MLX5_PAGE_FAULT_RESUME_RDMA);
430	flags |= (error ? MLX5_PAGE_FAULT_RESUME_ERROR : 0);
431	in.flags_qpn = cpu_to_be32((qpn & MLX5_QPN_MASK) |
432				   (flags << MLX5_QPN_BITS));
433	err = mlx5_cmd_exec(dev, &in, sizeof(in), &out, sizeof(out));
434	if (err)
435		return err;
436
437	if (out.hdr.status)
438		err = mlx5_cmd_status_to_err(&out.hdr);
439
440	return err;
441}
442EXPORT_SYMBOL_GPL(mlx5_core_page_fault_resume);
443#endif
444