91 	struct its_node		*its;  member
109 	struct its_node *its = its_dev->its;  in dev_event_to_col()  local
111 	return its->collections + its_dev->event_map.col_map[event];  in dev_event_to_col()
354 static u64 its_cmd_ptr_to_offset(struct its_node *its,  in its_cmd_ptr_to_offset()  argument
357 	return (ptr - its->cmd_base) * sizeof(*ptr);  in its_cmd_ptr_to_offset()
360 static int its_queue_full(struct its_node *its)  in its_queue_full()  argument
365 	widx = its->cmd_write - its->cmd_base;  in its_queue_full()
366 	ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block);  in its_queue_full()
375 static struct its_cmd_block *its_allocate_entry(struct its_node *its)  in its_allocate_entry()  argument
380 	while (its_queue_full(its)) {  in its_allocate_entry()
390 	cmd = its->cmd_write++;  in its_allocate_entry()
393 	if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES))  in its_allocate_entry()
394 		its->cmd_write = its->cmd_base;  in its_allocate_entry()
399 static struct its_cmd_block *its_post_commands(struct its_node *its)  in its_post_commands()  argument
401 	u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write);  in its_post_commands()
403 	writel_relaxed(wr, its->base + GITS_CWRITER);  in its_post_commands()
405 	return its->cmd_write;  in its_post_commands()
408 static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd)  in its_flush_cmd()  argument
414 	if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING)  in its_flush_cmd()
420 static void its_wait_for_range_completion(struct its_node *its,  in its_wait_for_range_completion()  argument
427 	from_idx = its_cmd_ptr_to_offset(its, from);  in its_wait_for_range_completion()
428 	to_idx = its_cmd_ptr_to_offset(its, to);  in its_wait_for_range_completion()
431 		rd_idx = readl_relaxed(its->base + GITS_CREADR);  in its_wait_for_range_completion()
445 static void its_send_single_command(struct its_node *its,  in its_send_single_command()  argument
453 	raw_spin_lock_irqsave(&its->lock, flags);  in its_send_single_command()
455 	cmd = its_allocate_entry(its);  in its_send_single_command()
458 		raw_spin_unlock_irqrestore(&its->lock, flags);  in its_send_single_command()
462 	its_flush_cmd(its, cmd);  in its_send_single_command()
465 		sync_cmd = its_allocate_entry(its);  in its_send_single_command()
473 		its_flush_cmd(its, sync_cmd);  in its_send_single_command()
477 	next_cmd = its_post_commands(its);  in its_send_single_command()
478 	raw_spin_unlock_irqrestore(&its->lock, flags);  in its_send_single_command()
480 	its_wait_for_range_completion(its, cmd, next_cmd);  in its_send_single_command()
490 	its_send_single_command(dev->its, its_build_inv_cmd, &desc);  in its_send_inv()
500 	its_send_single_command(dev->its, its_build_mapd_cmd, &desc);  in its_send_mapd()
503 static void its_send_mapc(struct its_node *its, struct its_collection *col,  in its_send_mapc()  argument
511 	its_send_single_command(its, its_build_mapc_cmd, &desc);  in its_send_mapc()
522 	its_send_single_command(dev->its, its_build_mapvi_cmd, &desc);  in its_send_mapvi()
534 	its_send_single_command(dev->its, its_build_movi_cmd, &desc);  in its_send_movi()
544 	its_send_single_command(dev->its, its_build_discard_cmd, &desc);  in its_send_discard()
547 static void its_send_invall(struct its_node *its, struct its_collection *col)  in its_send_invall()  argument
553 	its_send_single_command(its, its_build_invall_cmd, &desc);  in its_send_invall()
611 	target_col = &its_dev->its->collections[cpu];  in its_set_affinity()
621 	struct its_node *its;  in its_irq_compose_msi_msg()  local
624 	its = its_dev->its;  in its_irq_compose_msi_msg()
625 	addr = its->phys_base + GITS_TRANSLATER;  in its_irq_compose_msi_msg()
800 static void its_free_tables(struct its_node *its)  in its_free_tables()  argument
805 		if (its->tables[i]) {  in its_free_tables()
806 			free_page((unsigned long)its->tables[i]);  in its_free_tables()
807 			its->tables[i] = NULL;  in its_free_tables()
812 static int its_alloc_tables(const char *node_name, struct its_node *its)  in its_alloc_tables()  argument
822 	if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375) {  in its_alloc_tables()
831 		typer	= readq_relaxed(its->base + GITS_TYPER);  in its_alloc_tables()
836 		u64 val = readq_relaxed(its->base + GITS_BASER + i * 8);  in its_alloc_tables()
887 		its->tables[i] = base;  in its_alloc_tables()
911 		writeq_relaxed(val, its->base + GITS_BASER + i * 8);  in its_alloc_tables()
912 		tmp = readq_relaxed(its->base + GITS_BASER + i * 8);  in its_alloc_tables()
964 	its_free_tables(its);  in its_alloc_tables()
969 static int its_alloc_collections(struct its_node *its)  in its_alloc_collections()  argument
971 	its->collections = kzalloc(nr_cpu_ids * sizeof(*its->collections),  in its_alloc_collections()
973 	if (!its->collections)  in its_alloc_collections()
1075 	struct its_node *its;  in its_cpu_init_collection()  local
1081 	list_for_each_entry(its, &its_nodes, entry) {  in its_cpu_init_collection()
1088 		if (readq_relaxed(its->base + GITS_TYPER) & GITS_TYPER_PTA) {  in its_cpu_init_collection()
1103 		its->collections[cpu].target_address = target;  in its_cpu_init_collection()
1104 		its->collections[cpu].col_id = cpu;  in its_cpu_init_collection()
1106 		its_send_mapc(its, &its->collections[cpu], 1);  in its_cpu_init_collection()
1107 		its_send_invall(its, &its->collections[cpu]);  in its_cpu_init_collection()
1113 static struct its_device *its_find_device(struct its_node *its, u32 dev_id)  in its_find_device()  argument
1118 	raw_spin_lock_irqsave(&its->lock, flags);  in its_find_device()
1120 	list_for_each_entry(tmp, &its->its_device_list, entry) {  in its_find_device()
1127 	raw_spin_unlock_irqrestore(&its->lock, flags);  in its_find_device()
1132 static struct its_device *its_create_device(struct its_node *its, u32 dev_id,  in its_create_device()  argument
1152 	sz = nr_ites * its->ite_size;  in its_create_device()
1169 	dev->its = its;  in its_create_device()
1179 	raw_spin_lock_irqsave(&its->lock, flags);  in its_create_device()
1180 	list_add(&dev->entry, &its->its_device_list);  in its_create_device()
1181 	raw_spin_unlock_irqrestore(&its->lock, flags);  in its_create_device()
1193 	raw_spin_lock_irqsave(&its_dev->its->lock, flags);  in its_free_device()
1195 	raw_spin_unlock_irqrestore(&its_dev->its->lock, flags);  in its_free_device()
1218 	struct its_node *its;  in its_msi_prepare()  local
1232 	its = msi_info->data;  in its_msi_prepare()
1234 	its_dev = its_find_device(its, dev_id);  in its_msi_prepare()
1245 	its_dev = its_create_device(its, dev_id, nvec);  in its_msi_prepare()
1398 	struct its_node *its = data;  in its_enable_quirk_cavium_22375()  local
1400 	its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375;  in its_enable_quirk_cavium_22375()
1416 static void its_enable_quirks(struct its_node *its)  in its_enable_quirks()  argument
1418 	u32 iidr = readl_relaxed(its->base + GITS_IIDR);  in its_enable_quirks()
1420 	gic_enable_quirks(iidr, its_quirks, its);  in its_enable_quirks()
1426 	struct its_node *its;  in its_probe()  local
1461 	its = kzalloc(sizeof(*its), GFP_KERNEL);  in its_probe()
1462 	if (!its) {  in its_probe()
1467 	raw_spin_lock_init(&its->lock);  in its_probe()
1468 	INIT_LIST_HEAD(&its->entry);  in its_probe()
1469 	INIT_LIST_HEAD(&its->its_device_list);  in its_probe()
1470 	its->base = its_base;  in its_probe()
1471 	its->phys_base = res.start;  in its_probe()
1472 	its->ite_size = ((readl_relaxed(its_base + GITS_TYPER) >> 4) & 0xf) + 1;  in its_probe()
1474 	its->cmd_base = kzalloc(ITS_CMD_QUEUE_SZ, GFP_KERNEL);  in its_probe()
1475 	if (!its->cmd_base) {  in its_probe()
1479 	its->cmd_write = its->cmd_base;  in its_probe()
1481 	its_enable_quirks(its);  in its_probe()
1483 	err = its_alloc_tables(node->full_name, its);  in its_probe()
1487 	err = its_alloc_collections(its);  in its_probe()
1491 	baser = (virt_to_phys(its->cmd_base)	|  in its_probe()
1497 	writeq_relaxed(baser, its->base + GITS_CBASER);  in its_probe()
1498 	tmp = readq_relaxed(its->base + GITS_CBASER);  in its_probe()
1510 			writeq_relaxed(baser, its->base + GITS_CBASER);  in its_probe()
1513 		its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING;  in its_probe()
1516 	writeq_relaxed(0, its->base + GITS_CWRITER);  in its_probe()
1517 	writel_relaxed(GITS_CTLR_ENABLE, its->base + GITS_CTLR);  in its_probe()
1528 		inner_domain = irq_domain_add_tree(node, &its_domain_ops, its);  in its_probe()
1538 		info->data = its;  in its_probe()
1543 	list_add(&its->entry, &its_nodes);  in its_probe()
1549 	its_free_tables(its);  in its_probe()
1551 	kfree(its->cmd_base);  in its_probe()
1553 	kfree(its);  in its_probe()