1100 	if (iommu->qi) {  in free_iommu()
1101 		free_page((unsigned long)iommu->qi->desc);  in free_iommu()
1102 		kfree(iommu->qi->desc_status);  in free_iommu()
1103 		kfree(iommu->qi);  in free_iommu()
1116 static inline void reclaim_free_desc(struct q_inval *qi)  in reclaim_free_desc()  argument
1118 	while (qi->desc_status[qi->free_tail] == QI_DONE ||  in reclaim_free_desc()
1119 	       qi->desc_status[qi->free_tail] == QI_ABORT) {  in reclaim_free_desc()
1120 		qi->desc_status[qi->free_tail] = QI_FREE;  in reclaim_free_desc()
1121 		qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;  in reclaim_free_desc()
1122 		qi->free_cnt++;  in reclaim_free_desc()
1130 	struct q_inval *qi = iommu->qi;  in qi_check_fault()  local
1133 	if (qi->desc_status[wait_index] == QI_ABORT)  in qi_check_fault()
1148 				(unsigned long long)qi->desc[index].low,  in qi_check_fault()
1149 				(unsigned long long)qi->desc[index].high);  in qi_check_fault()
1150 			memcpy(&qi->desc[index], &qi->desc[wait_index],  in qi_check_fault()
1152 			__iommu_flush_cache(iommu, &qi->desc[index],  in qi_check_fault()
1173 			if (qi->desc_status[head] == QI_IN_USE)  in qi_check_fault()
1174 				qi->desc_status[head] = QI_ABORT;  in qi_check_fault()
1178 		if (qi->desc_status[wait_index] == QI_ABORT)  in qi_check_fault()
1195 	struct q_inval *qi = iommu->qi;  in qi_submit_sync()  local
1200 	if (!qi)  in qi_submit_sync()
1203 	hw = qi->desc;  in qi_submit_sync()
1208 	raw_spin_lock_irqsave(&qi->q_lock, flags);  in qi_submit_sync()
1209 	while (qi->free_cnt < 3) {  in qi_submit_sync()
1210 		raw_spin_unlock_irqrestore(&qi->q_lock, flags);  in qi_submit_sync()
1212 		raw_spin_lock_irqsave(&qi->q_lock, flags);  in qi_submit_sync()
1215 	index = qi->free_head;  in qi_submit_sync()
1218 	qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;  in qi_submit_sync()
1224 	wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);  in qi_submit_sync()
1231 	qi->free_head = (qi->free_head + 2) % QI_LENGTH;  in qi_submit_sync()
1232 	qi->free_cnt -= 2;  in qi_submit_sync()
1238 	writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);  in qi_submit_sync()
1240 	while (qi->desc_status[wait_index] != QI_DONE) {  in qi_submit_sync()
1252 		raw_spin_unlock(&qi->q_lock);  in qi_submit_sync()
1254 		raw_spin_lock(&qi->q_lock);  in qi_submit_sync()
1257 	qi->desc_status[index] = QI_DONE;  in qi_submit_sync()
1259 	reclaim_free_desc(qi);  in qi_submit_sync()
1260 	raw_spin_unlock_irqrestore(&qi->q_lock, flags);  in qi_submit_sync()
1379 	struct q_inval *qi = iommu->qi;  in __dmar_enable_qi()  local
1381 	qi->free_head = qi->free_tail = 0;  in __dmar_enable_qi()
1382 	qi->free_cnt = QI_LENGTH;  in __dmar_enable_qi()
1389 	dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));  in __dmar_enable_qi()
1407 	struct q_inval *qi;  in dmar_enable_qi()  local
1416 	if (iommu->qi)  in dmar_enable_qi()
1419 	iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);  in dmar_enable_qi()
1420 	if (!iommu->qi)  in dmar_enable_qi()
1423 	qi = iommu->qi;  in dmar_enable_qi()
1428 		kfree(qi);  in dmar_enable_qi()
1429 		iommu->qi = NULL;  in dmar_enable_qi()
1433 	qi->desc = page_address(desc_page);  in dmar_enable_qi()
1435 	qi->desc_status = kzalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);  in dmar_enable_qi()
1436 	if (!qi->desc_status) {  in dmar_enable_qi()
1437 		free_page((unsigned long) qi->desc);  in dmar_enable_qi()
1438 		kfree(qi);  in dmar_enable_qi()
1439 		iommu->qi = NULL;  in dmar_enable_qi()
1443 	raw_spin_lock_init(&qi->q_lock);  in dmar_enable_qi()
1717 	if (!iommu->qi)  in dmar_reenable_qi()