189 #define Q_IDX(q, p)			((p) & ((1 << (q)->max_n_shift) - 1))  argument
190 #define Q_WRP(q, p)			((p) & (1 << (q)->max_n_shift))  argument
192 #define Q_OVF(q, p)			((p) & Q_OVERFLOW_FLAG)  argument
193 #define Q_ENT(q, p)			((q)->base +			\  argument
194 					 Q_IDX(q, p) * (q)->ent_dwords)
499 	struct arm_smmu_queue		q;  member
504 	struct arm_smmu_queue		q;  member
509 	struct arm_smmu_queue		q;  member
664 static bool queue_full(struct arm_smmu_queue *q)  in queue_full()  argument
666 	return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&  in queue_full()
667 	       Q_WRP(q, q->prod) != Q_WRP(q, q->cons);  in queue_full()
670 static bool queue_empty(struct arm_smmu_queue *q)  in queue_empty()  argument
672 	return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&  in queue_empty()
673 	       Q_WRP(q, q->prod) == Q_WRP(q, q->cons);  in queue_empty()
676 static void queue_sync_cons(struct arm_smmu_queue *q)  in queue_sync_cons()  argument
678 	q->cons = readl_relaxed(q->cons_reg);  in queue_sync_cons()
681 static void queue_inc_cons(struct arm_smmu_queue *q)  in queue_inc_cons()  argument
683 	u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;  in queue_inc_cons()
685 	q->cons = Q_OVF(q, q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);  in queue_inc_cons()
686 	writel(q->cons, q->cons_reg);  in queue_inc_cons()
689 static int queue_sync_prod(struct arm_smmu_queue *q)  in queue_sync_prod()  argument
692 	u32 prod = readl_relaxed(q->prod_reg);  in queue_sync_prod()
694 	if (Q_OVF(q, prod) != Q_OVF(q, q->prod))  in queue_sync_prod()
697 	q->prod = prod;  in queue_sync_prod()
701 static void queue_inc_prod(struct arm_smmu_queue *q)  in queue_inc_prod()  argument
703 	u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + 1;  in queue_inc_prod()
705 	q->prod = Q_OVF(q, q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);  in queue_inc_prod()
706 	writel(q->prod, q->prod_reg);  in queue_inc_prod()
709 static bool __queue_cons_before(struct arm_smmu_queue *q, u32 until)  in __queue_cons_before()  argument
711 	if (Q_WRP(q, q->cons) == Q_WRP(q, until))  in __queue_cons_before()
712 		return Q_IDX(q, q->cons) < Q_IDX(q, until);  in __queue_cons_before()
714 	return Q_IDX(q, q->cons) >= Q_IDX(q, until);  in __queue_cons_before()
717 static int queue_poll_cons(struct arm_smmu_queue *q, u32 until, bool wfe)  in queue_poll_cons()  argument
721 	while (queue_sync_cons(q), __queue_cons_before(q, until)) {  in queue_poll_cons()
744 static int queue_insert_raw(struct arm_smmu_queue *q, u64 *ent)  in queue_insert_raw()  argument
746 	if (queue_full(q))  in queue_insert_raw()
749 	queue_write(Q_ENT(q, q->prod), ent, q->ent_dwords);  in queue_insert_raw()
750 	queue_inc_prod(q);  in queue_insert_raw()
762 static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)  in queue_remove_raw()  argument
764 	if (queue_empty(q))  in queue_remove_raw()
767 	queue_read(ent, Q_ENT(q, q->cons), q->ent_dwords);  in queue_remove_raw()
768 	queue_inc_cons(q);  in queue_remove_raw()
850 	struct arm_smmu_queue *q = &smmu->cmdq.q;  in arm_smmu_cmdq_skip_err()  local
851 	u32 cons = readl_relaxed(q->cons_reg);  in arm_smmu_cmdq_skip_err()
873 	queue_read(cmd, Q_ENT(q, idx), q->ent_dwords);  in arm_smmu_cmdq_skip_err()
884 	queue_write(cmd, Q_ENT(q, idx), q->ent_dwords);  in arm_smmu_cmdq_skip_err()
893 	struct arm_smmu_queue *q = &smmu->cmdq.q;  in arm_smmu_cmdq_issue_cmd()  local
902 	while (until = q->prod + 1, queue_insert_raw(q, cmd) == -ENOSPC) {  in arm_smmu_cmdq_issue_cmd()
908 		if (queue_poll_cons(q, until, wfe))  in arm_smmu_cmdq_issue_cmd()
912 	if (ent->opcode == CMDQ_OP_CMD_SYNC && queue_poll_cons(q, until, wfe))  in arm_smmu_cmdq_issue_cmd()
1145 	struct arm_smmu_queue *q = &smmu->evtq.q;  in arm_smmu_evtq_thread()  local
1148 	while (!queue_remove_raw(q, evt)) {  in arm_smmu_evtq_thread()
1158 	q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);  in arm_smmu_evtq_thread()
1166 	struct arm_smmu_queue *q = &smmu->evtq.q;  in arm_smmu_evtq_handler()  local
1172 	if (queue_sync_prod(q) == -EOVERFLOW)  in arm_smmu_evtq_handler()
1174 	else if (queue_empty(q))  in arm_smmu_evtq_handler()
1183 	struct arm_smmu_queue *q = &smmu->priq.q;  in arm_smmu_priq_thread()  local
1186 	while (!queue_remove_raw(q, evt)) {  in arm_smmu_priq_thread()
1224 	q->cons = Q_OVF(q, q->prod) | Q_WRP(q, q->cons) | Q_IDX(q, q->cons);  in arm_smmu_priq_thread()
1232 	struct arm_smmu_queue *q = &smmu->priq.q;  in arm_smmu_priq_handler()  local
1235 	if (queue_sync_prod(q) == -EOVERFLOW)  in arm_smmu_priq_handler()
1237 	else if (queue_empty(q))  in arm_smmu_priq_handler()
1933 				   struct arm_smmu_queue *q,  in arm_smmu_init_one_queue()  argument
1938 	size_t qsz = ((1 << q->max_n_shift) * dwords) << 3;  in arm_smmu_init_one_queue()
1940 	q->base = dma_alloc_coherent(smmu->dev, qsz, &q->base_dma, GFP_KERNEL);  in arm_smmu_init_one_queue()
1941 	if (!q->base) {  in arm_smmu_init_one_queue()
1947 	q->prod_reg	= smmu->base + prod_off;  in arm_smmu_init_one_queue()
1948 	q->cons_reg	= smmu->base + cons_off;  in arm_smmu_init_one_queue()
1949 	q->ent_dwords	= dwords;  in arm_smmu_init_one_queue()
1951 	q->q_base  = Q_BASE_RWA;  in arm_smmu_init_one_queue()
1952 	q->q_base |= q->base_dma & Q_BASE_ADDR_MASK << Q_BASE_ADDR_SHIFT;  in arm_smmu_init_one_queue()
1953 	q->q_base |= (q->max_n_shift & Q_BASE_LOG2SIZE_MASK)  in arm_smmu_init_one_queue()
1956 	q->prod = q->cons = 0;  in arm_smmu_init_one_queue()
1961 				    struct arm_smmu_queue *q)  in arm_smmu_free_one_queue()  argument
1963 	size_t qsz = ((1 << q->max_n_shift) * q->ent_dwords) << 3;  in arm_smmu_free_one_queue()
1965 	dma_free_coherent(smmu->dev, qsz, q->base, q->base_dma);  in arm_smmu_free_one_queue()
1970 	arm_smmu_free_one_queue(smmu, &smmu->cmdq.q);  in arm_smmu_free_queues()
1971 	arm_smmu_free_one_queue(smmu, &smmu->evtq.q);  in arm_smmu_free_queues()
1974 		arm_smmu_free_one_queue(smmu, &smmu->priq.q);  in arm_smmu_free_queues()
1983 	ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, ARM_SMMU_CMDQ_PROD,  in arm_smmu_init_queues()
1989 	ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, ARM_SMMU_EVTQ_PROD,  in arm_smmu_init_queues()
1998 	ret = arm_smmu_init_one_queue(smmu, &smmu->priq.q, ARM_SMMU_PRIQ_PROD,  in arm_smmu_init_queues()
2006 	arm_smmu_free_one_queue(smmu, &smmu->evtq.q);  in arm_smmu_init_queues()
2008 	arm_smmu_free_one_queue(smmu, &smmu->cmdq.q);  in arm_smmu_init_queues()
2259 			smmu->evtq.q.irq = desc->irq;  in arm_smmu_setup_msis()
2265 			smmu->priq.q.irq = desc->irq;  in arm_smmu_setup_msis()
2292 	irq = smmu->evtq.q.irq;  in arm_smmu_setup_irqs()
2302 	irq = smmu->cmdq.q.irq;  in arm_smmu_setup_irqs()
2320 		irq = smmu->priq.q.irq;  in arm_smmu_setup_irqs()
2390 	writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);  in arm_smmu_device_reset()
2391 	writel_relaxed(smmu->cmdq.q.prod, smmu->base + ARM_SMMU_CMDQ_PROD);  in arm_smmu_device_reset()
2392 	writel_relaxed(smmu->cmdq.q.cons, smmu->base + ARM_SMMU_CMDQ_CONS);  in arm_smmu_device_reset()
2420 	writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);  in arm_smmu_device_reset()
2421 	writel_relaxed(smmu->evtq.q.prod, smmu->base + ARM_SMMU_EVTQ_PROD);  in arm_smmu_device_reset()
2422 	writel_relaxed(smmu->evtq.q.cons, smmu->base + ARM_SMMU_EVTQ_CONS);  in arm_smmu_device_reset()
2434 		writeq_relaxed(smmu->priq.q.q_base,  in arm_smmu_device_reset()
2436 		writel_relaxed(smmu->priq.q.prod,  in arm_smmu_device_reset()
2438 		writel_relaxed(smmu->priq.q.cons,  in arm_smmu_device_reset()
2573 	smmu->cmdq.q.max_n_shift = min((u32)CMDQ_MAX_SZ_SHIFT,  in arm_smmu_device_probe()
2575 	if (!smmu->cmdq.q.max_n_shift) {  in arm_smmu_device_probe()
2581 	smmu->evtq.q.max_n_shift = min((u32)EVTQ_MAX_SZ_SHIFT,  in arm_smmu_device_probe()
2583 	smmu->priq.q.max_n_shift = min((u32)PRIQ_MAX_SZ_SHIFT,  in arm_smmu_device_probe()
2672 		smmu->evtq.q.irq = irq;  in arm_smmu_device_dt_probe()
2676 		smmu->priq.q.irq = irq;  in arm_smmu_device_dt_probe()
2680 		smmu->cmdq.q.irq = irq;  in arm_smmu_device_dt_probe()