91 	struct its_node		*its;  member
109 	struct its_node *its = its_dev->its;  in dev_event_to_col()  local
111 	return its->collections + its_dev->event_map.col_map[event];  in dev_event_to_col()
354 static u64 its_cmd_ptr_to_offset(struct its_node *its,  in its_cmd_ptr_to_offset()  argument
357 	return (ptr - its->cmd_base) * sizeof(*ptr);  in its_cmd_ptr_to_offset()
360 static int its_queue_full(struct its_node *its)  in its_queue_full()  argument
365 	widx = its->cmd_write - its->cmd_base;  in its_queue_full()
366 	ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block);  in its_queue_full()
375 static struct its_cmd_block *its_allocate_entry(struct its_node *its)  in its_allocate_entry()  argument
380 	while (its_queue_full(its)) {  in its_allocate_entry()
390 	cmd = its->cmd_write++;  in its_allocate_entry()
393 	if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES))  in its_allocate_entry()
394 		its->cmd_write = its->cmd_base;  in its_allocate_entry()
399 static struct its_cmd_block *its_post_commands(struct its_node *its)  in its_post_commands()  argument
401 	u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write);  in its_post_commands()
403 	writel_relaxed(wr, its->base + GITS_CWRITER);  in its_post_commands()
405 	return its->cmd_write;  in its_post_commands()
408 static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd)  in its_flush_cmd()  argument
414 	if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING)  in its_flush_cmd()
420 static void its_wait_for_range_completion(struct its_node *its,  in its_wait_for_range_completion()  argument
427 	from_idx = its_cmd_ptr_to_offset(its, from);  in its_wait_for_range_completion()
428 	to_idx = its_cmd_ptr_to_offset(its, to);  in its_wait_for_range_completion()
431 		rd_idx = readl_relaxed(its->base + GITS_CREADR);  in its_wait_for_range_completion()
445 static void its_send_single_command(struct its_node *its,  in its_send_single_command()  argument
453 	raw_spin_lock_irqsave(&its->lock, flags);  in its_send_single_command()
455 	cmd = its_allocate_entry(its);  in its_send_single_command()
458 		raw_spin_unlock_irqrestore(&its->lock, flags);  in its_send_single_command()
462 	its_flush_cmd(its, cmd);  in its_send_single_command()
465 		sync_cmd = its_allocate_entry(its);  in its_send_single_command()
473 		its_flush_cmd(its, sync_cmd);  in its_send_single_command()
477 	next_cmd = its_post_commands(its);  in its_send_single_command()
478 	raw_spin_unlock_irqrestore(&its->lock, flags);  in its_send_single_command()
480 	its_wait_for_range_completion(its, cmd, next_cmd);  in its_send_single_command()
490 	its_send_single_command(dev->its, its_build_inv_cmd, &desc);  in its_send_inv()
500 	its_send_single_command(dev->its, its_build_mapd_cmd, &desc);  in its_send_mapd()
503 static void its_send_mapc(struct its_node *its, struct its_collection *col,  in its_send_mapc()  argument
511 	its_send_single_command(its, its_build_mapc_cmd, &desc);  in its_send_mapc()
522 	its_send_single_command(dev->its, its_build_mapvi_cmd, &desc);  in its_send_mapvi()
534 	its_send_single_command(dev->its, its_build_movi_cmd, &desc);  in its_send_movi()
544 	its_send_single_command(dev->its, its_build_discard_cmd, &desc);  in its_send_discard()
547 static void its_send_invall(struct its_node *its, struct its_collection *col)  in its_send_invall()  argument
553 	its_send_single_command(its, its_build_invall_cmd, &desc);  in its_send_invall()
616 	target_col = &its_dev->its->collections[cpu];  in its_set_affinity()
626 	struct its_node *its;  in its_irq_compose_msi_msg()  local
629 	its = its_dev->its;  in its_irq_compose_msi_msg()
630 	addr = its->phys_base + GITS_TRANSLATER;  in its_irq_compose_msi_msg()
822 static void its_free_tables(struct its_node *its)  in its_free_tables()  argument
827 		if (its->tables[i]) {  in its_free_tables()
828 			free_page((unsigned long)its->tables[i]);  in its_free_tables()
829 			its->tables[i] = NULL;  in its_free_tables()
834 static int its_alloc_tables(struct its_node *its)  in its_alloc_tables()  argument
843 		u64 val = readq_relaxed(its->base + GITS_BASER + i * 8);  in its_alloc_tables()
863 			u64 typer = readq_relaxed(its->base + GITS_TYPER);  in its_alloc_tables()
877 					its->msi_chip.of_node->full_name, order);  in its_alloc_tables()
888 		its->tables[i] = base;  in its_alloc_tables()
912 		writeq_relaxed(val, its->base + GITS_BASER + i * 8);  in its_alloc_tables()
913 		tmp = readq_relaxed(its->base + GITS_BASER + i * 8);  in its_alloc_tables()
949 			       its->msi_chip.of_node->full_name, i,  in its_alloc_tables()
965 	its_free_tables(its);  in its_alloc_tables()
970 static int its_alloc_collections(struct its_node *its)  in its_alloc_collections()  argument
972 	its->collections = kzalloc(nr_cpu_ids * sizeof(*its->collections),  in its_alloc_collections()
974 	if (!its->collections)  in its_alloc_collections()
1076 	struct its_node *its;  in its_cpu_init_collection()  local
1082 	list_for_each_entry(its, &its_nodes, entry) {  in its_cpu_init_collection()
1089 		if (readq_relaxed(its->base + GITS_TYPER) & GITS_TYPER_PTA) {  in its_cpu_init_collection()
1104 		its->collections[cpu].target_address = target;  in its_cpu_init_collection()
1105 		its->collections[cpu].col_id = cpu;  in its_cpu_init_collection()
1107 		its_send_mapc(its, &its->collections[cpu], 1);  in its_cpu_init_collection()
1108 		its_send_invall(its, &its->collections[cpu]);  in its_cpu_init_collection()
1114 static struct its_device *its_find_device(struct its_node *its, u32 dev_id)  in its_find_device()  argument
1119 	raw_spin_lock_irqsave(&its->lock, flags);  in its_find_device()
1121 	list_for_each_entry(tmp, &its->its_device_list, entry) {  in its_find_device()
1128 	raw_spin_unlock_irqrestore(&its->lock, flags);  in its_find_device()
1133 static struct its_device *its_create_device(struct its_node *its, u32 dev_id,  in its_create_device()  argument
1153 	sz = nr_ites * its->ite_size;  in its_create_device()
1170 	dev->its = its;  in its_create_device()
1180 	raw_spin_lock_irqsave(&its->lock, flags);  in its_create_device()
1181 	list_add(&dev->entry, &its->its_device_list);  in its_create_device()
1182 	raw_spin_unlock_irqrestore(&its->lock, flags);  in its_create_device()
1194 	raw_spin_lock_irqsave(&its_dev->its->lock, flags);  in its_free_device()
1196 	raw_spin_unlock_irqrestore(&its_dev->its->lock, flags);  in its_free_device()
1247 	struct its_node *its;  in its_msi_prepare()  local
1259 	its = domain->parent->host_data;  in its_msi_prepare()
1261 	its_dev = its_find_device(its, dev_alias.dev_id);  in its_msi_prepare()
1272 	its_dev = its_create_device(its, dev_alias.dev_id, dev_alias.count);  in its_msi_prepare()
1431 	struct its_node *its;  in its_probe()  local
1465 	its = kzalloc(sizeof(*its), GFP_KERNEL);  in its_probe()
1466 	if (!its) {  in its_probe()
1471 	raw_spin_lock_init(&its->lock);  in its_probe()
1472 	INIT_LIST_HEAD(&its->entry);  in its_probe()
1473 	INIT_LIST_HEAD(&its->its_device_list);  in its_probe()
1474 	its->base = its_base;  in its_probe()
1475 	its->phys_base = res.start;  in its_probe()
1476 	its->msi_chip.of_node = node;  in its_probe()
1477 	its->ite_size = ((readl_relaxed(its_base + GITS_TYPER) >> 4) & 0xf) + 1;  in its_probe()
1479 	its->cmd_base = kzalloc(ITS_CMD_QUEUE_SZ, GFP_KERNEL);  in its_probe()
1480 	if (!its->cmd_base) {  in its_probe()
1484 	its->cmd_write = its->cmd_base;  in its_probe()
1486 	err = its_alloc_tables(its);  in its_probe()
1490 	err = its_alloc_collections(its);  in its_probe()
1494 	baser = (virt_to_phys(its->cmd_base)	|  in its_probe()
1500 	writeq_relaxed(baser, its->base + GITS_CBASER);  in its_probe()
1501 	tmp = readq_relaxed(its->base + GITS_CBASER);  in its_probe()
1513 			writeq_relaxed(baser, its->base + GITS_CBASER);  in its_probe()
1516 		its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING;  in its_probe()
1519 	writeq_relaxed(0, its->base + GITS_CWRITER);  in its_probe()
1520 	writel_relaxed(GITS_CTLR_ENABLE, its->base + GITS_CTLR);  in its_probe()
1522 	if (of_property_read_bool(its->msi_chip.of_node, "msi-controller")) {  in its_probe()
1523 		its->domain = irq_domain_add_tree(NULL, &its_domain_ops, its);  in its_probe()
1524 		if (!its->domain) {  in its_probe()
1529 		its->domain->parent = parent;  in its_probe()
1531 		its->msi_chip.domain = pci_msi_create_irq_domain(node,  in its_probe()
1533 								 its->domain);  in its_probe()
1534 		if (!its->msi_chip.domain) {  in its_probe()
1539 		err = of_pci_msi_chip_add(&its->msi_chip);  in its_probe()
1545 	list_add(&its->entry, &its_nodes);  in its_probe()
1551 	if (its->msi_chip.domain)  in its_probe()
1552 		irq_domain_remove(its->msi_chip.domain);  in its_probe()
1553 	if (its->domain)  in its_probe()
1554 		irq_domain_remove(its->domain);  in its_probe()
1556 	its_free_tables(its);  in its_probe()
1558 	kfree(its->cmd_base);  in its_probe()
1560 	kfree(its);  in its_probe()