1/* 2 * Copyright (C) 2013, 2014 ARM Limited, All Rights Reserved. 3 * Author: Marc Zyngier <marc.zyngier@arm.com> 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License version 2 as 7 * published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope that it will be useful, 10 * but WITHOUT ANY WARRANTY; without even the implied warranty of 11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 * GNU General Public License for more details. 13 * 14 * You should have received a copy of the GNU General Public License 15 * along with this program. If not, see <http://www.gnu.org/licenses/>. 16 */ 17 18#include <linux/bitmap.h> 19#include <linux/cpu.h> 20#include <linux/delay.h> 21#include <linux/interrupt.h> 22#include <linux/log2.h> 23#include <linux/mm.h> 24#include <linux/msi.h> 25#include <linux/of.h> 26#include <linux/of_address.h> 27#include <linux/of_irq.h> 28#include <linux/of_pci.h> 29#include <linux/of_platform.h> 30#include <linux/percpu.h> 31#include <linux/slab.h> 32 33#include <linux/irqchip/arm-gic-v3.h> 34 35#include <asm/cacheflush.h> 36#include <asm/cputype.h> 37#include <asm/exception.h> 38 39#include "irqchip.h" 40 41#define ITS_FLAGS_CMDQ_NEEDS_FLUSHING (1 << 0) 42 43#define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING (1 << 0) 44 45/* 46 * Collection structure - just an ID, and a redistributor address to 47 * ping. We use one per CPU as a bag of interrupts assigned to this 48 * CPU. 49 */ 50struct its_collection { 51 u64 target_address; 52 u16 col_id; 53}; 54 55/* 56 * The ITS structure - contains most of the infrastructure, with the 57 * msi_controller, the command queue, the collections, and the list of 58 * devices writing to it. 59 */ 60struct its_node { 61 raw_spinlock_t lock; 62 struct list_head entry; 63 struct msi_controller msi_chip; 64 struct irq_domain *domain; 65 void __iomem *base; 66 unsigned long phys_base; 67 struct its_cmd_block *cmd_base; 68 struct its_cmd_block *cmd_write; 69 void *tables[GITS_BASER_NR_REGS]; 70 struct its_collection *collections; 71 struct list_head its_device_list; 72 u64 flags; 73 u32 ite_size; 74}; 75 76#define ITS_ITT_ALIGN SZ_256 77 78struct event_lpi_map { 79 unsigned long *lpi_map; 80 u16 *col_map; 81 irq_hw_number_t lpi_base; 82 int nr_lpis; 83}; 84 85/* 86 * The ITS view of a device - belongs to an ITS, a collection, owns an 87 * interrupt translation table, and a list of interrupts. 88 */ 89struct its_device { 90 struct list_head entry; 91 struct its_node *its; 92 struct event_lpi_map event_map; 93 void *itt; 94 u32 nr_ites; 95 u32 device_id; 96}; 97 98static LIST_HEAD(its_nodes); 99static DEFINE_SPINLOCK(its_lock); 100static struct device_node *gic_root_node; 101static struct rdists *gic_rdists; 102 103#define gic_data_rdist() (raw_cpu_ptr(gic_rdists->rdist)) 104#define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base) 105 106static struct its_collection *dev_event_to_col(struct its_device *its_dev, 107 u32 event) 108{ 109 struct its_node *its = its_dev->its; 110 111 return its->collections + its_dev->event_map.col_map[event]; 112} 113 114/* 115 * ITS command descriptors - parameters to be encoded in a command 116 * block. 117 */ 118struct its_cmd_desc { 119 union { 120 struct { 121 struct its_device *dev; 122 u32 event_id; 123 } its_inv_cmd; 124 125 struct { 126 struct its_device *dev; 127 u32 event_id; 128 } its_int_cmd; 129 130 struct { 131 struct its_device *dev; 132 int valid; 133 } its_mapd_cmd; 134 135 struct { 136 struct its_collection *col; 137 int valid; 138 } its_mapc_cmd; 139 140 struct { 141 struct its_device *dev; 142 u32 phys_id; 143 u32 event_id; 144 } its_mapvi_cmd; 145 146 struct { 147 struct its_device *dev; 148 struct its_collection *col; 149 u32 event_id; 150 } its_movi_cmd; 151 152 struct { 153 struct its_device *dev; 154 u32 event_id; 155 } its_discard_cmd; 156 157 struct { 158 struct its_collection *col; 159 } its_invall_cmd; 160 }; 161}; 162 163/* 164 * The ITS command block, which is what the ITS actually parses. 165 */ 166struct its_cmd_block { 167 u64 raw_cmd[4]; 168}; 169 170#define ITS_CMD_QUEUE_SZ SZ_64K 171#define ITS_CMD_QUEUE_NR_ENTRIES (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block)) 172 173typedef struct its_collection *(*its_cmd_builder_t)(struct its_cmd_block *, 174 struct its_cmd_desc *); 175 176static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr) 177{ 178 cmd->raw_cmd[0] &= ~0xffUL; 179 cmd->raw_cmd[0] |= cmd_nr; 180} 181 182static void its_encode_devid(struct its_cmd_block *cmd, u32 devid) 183{ 184 cmd->raw_cmd[0] &= BIT_ULL(32) - 1; 185 cmd->raw_cmd[0] |= ((u64)devid) << 32; 186} 187 188static void its_encode_event_id(struct its_cmd_block *cmd, u32 id) 189{ 190 cmd->raw_cmd[1] &= ~0xffffffffUL; 191 cmd->raw_cmd[1] |= id; 192} 193 194static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id) 195{ 196 cmd->raw_cmd[1] &= 0xffffffffUL; 197 cmd->raw_cmd[1] |= ((u64)phys_id) << 32; 198} 199 200static void its_encode_size(struct its_cmd_block *cmd, u8 size) 201{ 202 cmd->raw_cmd[1] &= ~0x1fUL; 203 cmd->raw_cmd[1] |= size & 0x1f; 204} 205 206static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr) 207{ 208 cmd->raw_cmd[2] &= ~0xffffffffffffUL; 209 cmd->raw_cmd[2] |= itt_addr & 0xffffffffff00UL; 210} 211 212static void its_encode_valid(struct its_cmd_block *cmd, int valid) 213{ 214 cmd->raw_cmd[2] &= ~(1UL << 63); 215 cmd->raw_cmd[2] |= ((u64)!!valid) << 63; 216} 217 218static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr) 219{ 220 cmd->raw_cmd[2] &= ~(0xffffffffUL << 16); 221 cmd->raw_cmd[2] |= (target_addr & (0xffffffffUL << 16)); 222} 223 224static void its_encode_collection(struct its_cmd_block *cmd, u16 col) 225{ 226 cmd->raw_cmd[2] &= ~0xffffUL; 227 cmd->raw_cmd[2] |= col; 228} 229 230static inline void its_fixup_cmd(struct its_cmd_block *cmd) 231{ 232 /* Let's fixup BE commands */ 233 cmd->raw_cmd[0] = cpu_to_le64(cmd->raw_cmd[0]); 234 cmd->raw_cmd[1] = cpu_to_le64(cmd->raw_cmd[1]); 235 cmd->raw_cmd[2] = cpu_to_le64(cmd->raw_cmd[2]); 236 cmd->raw_cmd[3] = cpu_to_le64(cmd->raw_cmd[3]); 237} 238 239static struct its_collection *its_build_mapd_cmd(struct its_cmd_block *cmd, 240 struct its_cmd_desc *desc) 241{ 242 unsigned long itt_addr; 243 u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites); 244 245 itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt); 246 itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN); 247 248 its_encode_cmd(cmd, GITS_CMD_MAPD); 249 its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id); 250 its_encode_size(cmd, size - 1); 251 its_encode_itt(cmd, itt_addr); 252 its_encode_valid(cmd, desc->its_mapd_cmd.valid); 253 254 its_fixup_cmd(cmd); 255 256 return NULL; 257} 258 259static struct its_collection *its_build_mapc_cmd(struct its_cmd_block *cmd, 260 struct its_cmd_desc *desc) 261{ 262 its_encode_cmd(cmd, GITS_CMD_MAPC); 263 its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id); 264 its_encode_target(cmd, desc->its_mapc_cmd.col->target_address); 265 its_encode_valid(cmd, desc->its_mapc_cmd.valid); 266 267 its_fixup_cmd(cmd); 268 269 return desc->its_mapc_cmd.col; 270} 271 272static struct its_collection *its_build_mapvi_cmd(struct its_cmd_block *cmd, 273 struct its_cmd_desc *desc) 274{ 275 struct its_collection *col; 276 277 col = dev_event_to_col(desc->its_mapvi_cmd.dev, 278 desc->its_mapvi_cmd.event_id); 279 280 its_encode_cmd(cmd, GITS_CMD_MAPVI); 281 its_encode_devid(cmd, desc->its_mapvi_cmd.dev->device_id); 282 its_encode_event_id(cmd, desc->its_mapvi_cmd.event_id); 283 its_encode_phys_id(cmd, desc->its_mapvi_cmd.phys_id); 284 its_encode_collection(cmd, col->col_id); 285 286 its_fixup_cmd(cmd); 287 288 return col; 289} 290 291static struct its_collection *its_build_movi_cmd(struct its_cmd_block *cmd, 292 struct its_cmd_desc *desc) 293{ 294 struct its_collection *col; 295 296 col = dev_event_to_col(desc->its_movi_cmd.dev, 297 desc->its_movi_cmd.event_id); 298 299 its_encode_cmd(cmd, GITS_CMD_MOVI); 300 its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id); 301 its_encode_event_id(cmd, desc->its_movi_cmd.event_id); 302 its_encode_collection(cmd, desc->its_movi_cmd.col->col_id); 303 304 its_fixup_cmd(cmd); 305 306 return col; 307} 308 309static struct its_collection *its_build_discard_cmd(struct its_cmd_block *cmd, 310 struct its_cmd_desc *desc) 311{ 312 struct its_collection *col; 313 314 col = dev_event_to_col(desc->its_discard_cmd.dev, 315 desc->its_discard_cmd.event_id); 316 317 its_encode_cmd(cmd, GITS_CMD_DISCARD); 318 its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id); 319 its_encode_event_id(cmd, desc->its_discard_cmd.event_id); 320 321 its_fixup_cmd(cmd); 322 323 return col; 324} 325 326static struct its_collection *its_build_inv_cmd(struct its_cmd_block *cmd, 327 struct its_cmd_desc *desc) 328{ 329 struct its_collection *col; 330 331 col = dev_event_to_col(desc->its_inv_cmd.dev, 332 desc->its_inv_cmd.event_id); 333 334 its_encode_cmd(cmd, GITS_CMD_INV); 335 its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id); 336 its_encode_event_id(cmd, desc->its_inv_cmd.event_id); 337 338 its_fixup_cmd(cmd); 339 340 return col; 341} 342 343static struct its_collection *its_build_invall_cmd(struct its_cmd_block *cmd, 344 struct its_cmd_desc *desc) 345{ 346 its_encode_cmd(cmd, GITS_CMD_INVALL); 347 its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id); 348 349 its_fixup_cmd(cmd); 350 351 return NULL; 352} 353 354static u64 its_cmd_ptr_to_offset(struct its_node *its, 355 struct its_cmd_block *ptr) 356{ 357 return (ptr - its->cmd_base) * sizeof(*ptr); 358} 359 360static int its_queue_full(struct its_node *its) 361{ 362 int widx; 363 int ridx; 364 365 widx = its->cmd_write - its->cmd_base; 366 ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block); 367 368 /* This is incredibly unlikely to happen, unless the ITS locks up. */ 369 if (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx) 370 return 1; 371 372 return 0; 373} 374 375static struct its_cmd_block *its_allocate_entry(struct its_node *its) 376{ 377 struct its_cmd_block *cmd; 378 u32 count = 1000000; /* 1s! */ 379 380 while (its_queue_full(its)) { 381 count--; 382 if (!count) { 383 pr_err_ratelimited("ITS queue not draining\n"); 384 return NULL; 385 } 386 cpu_relax(); 387 udelay(1); 388 } 389 390 cmd = its->cmd_write++; 391 392 /* Handle queue wrapping */ 393 if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES)) 394 its->cmd_write = its->cmd_base; 395 396 return cmd; 397} 398 399static struct its_cmd_block *its_post_commands(struct its_node *its) 400{ 401 u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write); 402 403 writel_relaxed(wr, its->base + GITS_CWRITER); 404 405 return its->cmd_write; 406} 407 408static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd) 409{ 410 /* 411 * Make sure the commands written to memory are observable by 412 * the ITS. 413 */ 414 if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING) 415 __flush_dcache_area(cmd, sizeof(*cmd)); 416 else 417 dsb(ishst); 418} 419 420static void its_wait_for_range_completion(struct its_node *its, 421 struct its_cmd_block *from, 422 struct its_cmd_block *to) 423{ 424 u64 rd_idx, from_idx, to_idx; 425 u32 count = 1000000; /* 1s! */ 426 427 from_idx = its_cmd_ptr_to_offset(its, from); 428 to_idx = its_cmd_ptr_to_offset(its, to); 429 430 while (1) { 431 rd_idx = readl_relaxed(its->base + GITS_CREADR); 432 if (rd_idx >= to_idx || rd_idx < from_idx) 433 break; 434 435 count--; 436 if (!count) { 437 pr_err_ratelimited("ITS queue timeout\n"); 438 return; 439 } 440 cpu_relax(); 441 udelay(1); 442 } 443} 444 445static void its_send_single_command(struct its_node *its, 446 its_cmd_builder_t builder, 447 struct its_cmd_desc *desc) 448{ 449 struct its_cmd_block *cmd, *sync_cmd, *next_cmd; 450 struct its_collection *sync_col; 451 unsigned long flags; 452 453 raw_spin_lock_irqsave(&its->lock, flags); 454 455 cmd = its_allocate_entry(its); 456 if (!cmd) { /* We're soooooo screewed... */ 457 pr_err_ratelimited("ITS can't allocate, dropping command\n"); 458 raw_spin_unlock_irqrestore(&its->lock, flags); 459 return; 460 } 461 sync_col = builder(cmd, desc); 462 its_flush_cmd(its, cmd); 463 464 if (sync_col) { 465 sync_cmd = its_allocate_entry(its); 466 if (!sync_cmd) { 467 pr_err_ratelimited("ITS can't SYNC, skipping\n"); 468 goto post; 469 } 470 its_encode_cmd(sync_cmd, GITS_CMD_SYNC); 471 its_encode_target(sync_cmd, sync_col->target_address); 472 its_fixup_cmd(sync_cmd); 473 its_flush_cmd(its, sync_cmd); 474 } 475 476post: 477 next_cmd = its_post_commands(its); 478 raw_spin_unlock_irqrestore(&its->lock, flags); 479 480 its_wait_for_range_completion(its, cmd, next_cmd); 481} 482 483static void its_send_inv(struct its_device *dev, u32 event_id) 484{ 485 struct its_cmd_desc desc; 486 487 desc.its_inv_cmd.dev = dev; 488 desc.its_inv_cmd.event_id = event_id; 489 490 its_send_single_command(dev->its, its_build_inv_cmd, &desc); 491} 492 493static void its_send_mapd(struct its_device *dev, int valid) 494{ 495 struct its_cmd_desc desc; 496 497 desc.its_mapd_cmd.dev = dev; 498 desc.its_mapd_cmd.valid = !!valid; 499 500 its_send_single_command(dev->its, its_build_mapd_cmd, &desc); 501} 502 503static void its_send_mapc(struct its_node *its, struct its_collection *col, 504 int valid) 505{ 506 struct its_cmd_desc desc; 507 508 desc.its_mapc_cmd.col = col; 509 desc.its_mapc_cmd.valid = !!valid; 510 511 its_send_single_command(its, its_build_mapc_cmd, &desc); 512} 513 514static void its_send_mapvi(struct its_device *dev, u32 irq_id, u32 id) 515{ 516 struct its_cmd_desc desc; 517 518 desc.its_mapvi_cmd.dev = dev; 519 desc.its_mapvi_cmd.phys_id = irq_id; 520 desc.its_mapvi_cmd.event_id = id; 521 522 its_send_single_command(dev->its, its_build_mapvi_cmd, &desc); 523} 524 525static void its_send_movi(struct its_device *dev, 526 struct its_collection *col, u32 id) 527{ 528 struct its_cmd_desc desc; 529 530 desc.its_movi_cmd.dev = dev; 531 desc.its_movi_cmd.col = col; 532 desc.its_movi_cmd.event_id = id; 533 534 its_send_single_command(dev->its, its_build_movi_cmd, &desc); 535} 536 537static void its_send_discard(struct its_device *dev, u32 id) 538{ 539 struct its_cmd_desc desc; 540 541 desc.its_discard_cmd.dev = dev; 542 desc.its_discard_cmd.event_id = id; 543 544 its_send_single_command(dev->its, its_build_discard_cmd, &desc); 545} 546 547static void its_send_invall(struct its_node *its, struct its_collection *col) 548{ 549 struct its_cmd_desc desc; 550 551 desc.its_invall_cmd.col = col; 552 553 its_send_single_command(its, its_build_invall_cmd, &desc); 554} 555 556/* 557 * irqchip functions - assumes MSI, mostly. 558 */ 559 560static inline u32 its_get_event_id(struct irq_data *d) 561{ 562 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 563 return d->hwirq - its_dev->event_map.lpi_base; 564} 565 566static void lpi_set_config(struct irq_data *d, bool enable) 567{ 568 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 569 irq_hw_number_t hwirq = d->hwirq; 570 u32 id = its_get_event_id(d); 571 u8 *cfg = page_address(gic_rdists->prop_page) + hwirq - 8192; 572 573 if (enable) 574 *cfg |= LPI_PROP_ENABLED; 575 else 576 *cfg &= ~LPI_PROP_ENABLED; 577 578 /* 579 * Make the above write visible to the redistributors. 580 * And yes, we're flushing exactly: One. Single. Byte. 581 * Humpf... 582 */ 583 if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING) 584 __flush_dcache_area(cfg, sizeof(*cfg)); 585 else 586 dsb(ishst); 587 its_send_inv(its_dev, id); 588} 589 590static void its_mask_irq(struct irq_data *d) 591{ 592 lpi_set_config(d, false); 593} 594 595static void its_unmask_irq(struct irq_data *d) 596{ 597 lpi_set_config(d, true); 598} 599 600static void its_eoi_irq(struct irq_data *d) 601{ 602 gic_write_eoir(d->hwirq); 603} 604 605static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val, 606 bool force) 607{ 608 unsigned int cpu = cpumask_any_and(mask_val, cpu_online_mask); 609 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 610 struct its_collection *target_col; 611 u32 id = its_get_event_id(d); 612 613 if (cpu >= nr_cpu_ids) 614 return -EINVAL; 615 616 target_col = &its_dev->its->collections[cpu]; 617 its_send_movi(its_dev, target_col, id); 618 its_dev->event_map.col_map[id] = cpu; 619 620 return IRQ_SET_MASK_OK_DONE; 621} 622 623static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) 624{ 625 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 626 struct its_node *its; 627 u64 addr; 628 629 its = its_dev->its; 630 addr = its->phys_base + GITS_TRANSLATER; 631 632 msg->address_lo = addr & ((1UL << 32) - 1); 633 msg->address_hi = addr >> 32; 634 msg->data = its_get_event_id(d); 635} 636 637static struct irq_chip its_irq_chip = { 638 .name = "ITS", 639 .irq_mask = its_mask_irq, 640 .irq_unmask = its_unmask_irq, 641 .irq_eoi = its_eoi_irq, 642 .irq_set_affinity = its_set_affinity, 643 .irq_compose_msi_msg = its_irq_compose_msi_msg, 644}; 645 646static void its_mask_msi_irq(struct irq_data *d) 647{ 648 pci_msi_mask_irq(d); 649 irq_chip_mask_parent(d); 650} 651 652static void its_unmask_msi_irq(struct irq_data *d) 653{ 654 pci_msi_unmask_irq(d); 655 irq_chip_unmask_parent(d); 656} 657 658static struct irq_chip its_msi_irq_chip = { 659 .name = "ITS-MSI", 660 .irq_unmask = its_unmask_msi_irq, 661 .irq_mask = its_mask_msi_irq, 662 .irq_eoi = irq_chip_eoi_parent, 663 .irq_write_msi_msg = pci_msi_domain_write_msg, 664}; 665 666/* 667 * How we allocate LPIs: 668 * 669 * The GIC has id_bits bits for interrupt identifiers. From there, we 670 * must subtract 8192 which are reserved for SGIs/PPIs/SPIs. Then, as 671 * we allocate LPIs by chunks of 32, we can shift the whole thing by 5 672 * bits to the right. 673 * 674 * This gives us (((1UL << id_bits) - 8192) >> 5) possible allocations. 675 */ 676#define IRQS_PER_CHUNK_SHIFT 5 677#define IRQS_PER_CHUNK (1 << IRQS_PER_CHUNK_SHIFT) 678 679static unsigned long *lpi_bitmap; 680static u32 lpi_chunks; 681static DEFINE_SPINLOCK(lpi_lock); 682 683static int its_lpi_to_chunk(int lpi) 684{ 685 return (lpi - 8192) >> IRQS_PER_CHUNK_SHIFT; 686} 687 688static int its_chunk_to_lpi(int chunk) 689{ 690 return (chunk << IRQS_PER_CHUNK_SHIFT) + 8192; 691} 692 693static int its_lpi_init(u32 id_bits) 694{ 695 lpi_chunks = its_lpi_to_chunk(1UL << id_bits); 696 697 lpi_bitmap = kzalloc(BITS_TO_LONGS(lpi_chunks) * sizeof(long), 698 GFP_KERNEL); 699 if (!lpi_bitmap) { 700 lpi_chunks = 0; 701 return -ENOMEM; 702 } 703 704 pr_info("ITS: Allocated %d chunks for LPIs\n", (int)lpi_chunks); 705 return 0; 706} 707 708static unsigned long *its_lpi_alloc_chunks(int nr_irqs, int *base, int *nr_ids) 709{ 710 unsigned long *bitmap = NULL; 711 int chunk_id; 712 int nr_chunks; 713 int i; 714 715 nr_chunks = DIV_ROUND_UP(nr_irqs, IRQS_PER_CHUNK); 716 717 spin_lock(&lpi_lock); 718 719 do { 720 chunk_id = bitmap_find_next_zero_area(lpi_bitmap, lpi_chunks, 721 0, nr_chunks, 0); 722 if (chunk_id < lpi_chunks) 723 break; 724 725 nr_chunks--; 726 } while (nr_chunks > 0); 727 728 if (!nr_chunks) 729 goto out; 730 731 bitmap = kzalloc(BITS_TO_LONGS(nr_chunks * IRQS_PER_CHUNK) * sizeof (long), 732 GFP_ATOMIC); 733 if (!bitmap) 734 goto out; 735 736 for (i = 0; i < nr_chunks; i++) 737 set_bit(chunk_id + i, lpi_bitmap); 738 739 *base = its_chunk_to_lpi(chunk_id); 740 *nr_ids = nr_chunks * IRQS_PER_CHUNK; 741 742out: 743 spin_unlock(&lpi_lock); 744 745 return bitmap; 746} 747 748static void its_lpi_free(struct event_lpi_map *map) 749{ 750 int base = map->lpi_base; 751 int nr_ids = map->nr_lpis; 752 int lpi; 753 754 spin_lock(&lpi_lock); 755 756 for (lpi = base; lpi < (base + nr_ids); lpi += IRQS_PER_CHUNK) { 757 int chunk = its_lpi_to_chunk(lpi); 758 BUG_ON(chunk > lpi_chunks); 759 if (test_bit(chunk, lpi_bitmap)) { 760 clear_bit(chunk, lpi_bitmap); 761 } else { 762 pr_err("Bad LPI chunk %d\n", chunk); 763 } 764 } 765 766 spin_unlock(&lpi_lock); 767 768 kfree(map->lpi_map); 769 kfree(map->col_map); 770} 771 772/* 773 * We allocate 64kB for PROPBASE. That gives us at most 64K LPIs to 774 * deal with (one configuration byte per interrupt). PENDBASE has to 775 * be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI). 776 */ 777#define LPI_PROPBASE_SZ SZ_64K 778#define LPI_PENDBASE_SZ (LPI_PROPBASE_SZ / 8 + SZ_1K) 779 780/* 781 * This is how many bits of ID we need, including the useless ones. 782 */ 783#define LPI_NRBITS ilog2(LPI_PROPBASE_SZ + SZ_8K) 784 785#define LPI_PROP_DEFAULT_PRIO 0xa0 786 787static int __init its_alloc_lpi_tables(void) 788{ 789 phys_addr_t paddr; 790 791 gic_rdists->prop_page = alloc_pages(GFP_NOWAIT, 792 get_order(LPI_PROPBASE_SZ)); 793 if (!gic_rdists->prop_page) { 794 pr_err("Failed to allocate PROPBASE\n"); 795 return -ENOMEM; 796 } 797 798 paddr = page_to_phys(gic_rdists->prop_page); 799 pr_info("GIC: using LPI property table @%pa\n", &paddr); 800 801 /* Priority 0xa0, Group-1, disabled */ 802 memset(page_address(gic_rdists->prop_page), 803 LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1, 804 LPI_PROPBASE_SZ); 805 806 /* Make sure the GIC will observe the written configuration */ 807 __flush_dcache_area(page_address(gic_rdists->prop_page), LPI_PROPBASE_SZ); 808 809 return 0; 810} 811 812static const char *its_base_type_string[] = { 813 [GITS_BASER_TYPE_DEVICE] = "Devices", 814 [GITS_BASER_TYPE_VCPU] = "Virtual CPUs", 815 [GITS_BASER_TYPE_CPU] = "Physical CPUs", 816 [GITS_BASER_TYPE_COLLECTION] = "Interrupt Collections", 817 [GITS_BASER_TYPE_RESERVED5] = "Reserved (5)", 818 [GITS_BASER_TYPE_RESERVED6] = "Reserved (6)", 819 [GITS_BASER_TYPE_RESERVED7] = "Reserved (7)", 820}; 821 822static void its_free_tables(struct its_node *its) 823{ 824 int i; 825 826 for (i = 0; i < GITS_BASER_NR_REGS; i++) { 827 if (its->tables[i]) { 828 free_page((unsigned long)its->tables[i]); 829 its->tables[i] = NULL; 830 } 831 } 832} 833 834static int its_alloc_tables(struct its_node *its) 835{ 836 int err; 837 int i; 838 int psz = SZ_64K; 839 u64 shr = GITS_BASER_InnerShareable; 840 u64 cache = GITS_BASER_WaWb; 841 842 for (i = 0; i < GITS_BASER_NR_REGS; i++) { 843 u64 val = readq_relaxed(its->base + GITS_BASER + i * 8); 844 u64 type = GITS_BASER_TYPE(val); 845 u64 entry_size = GITS_BASER_ENTRY_SIZE(val); 846 int order = get_order(psz); 847 int alloc_size; 848 u64 tmp; 849 void *base; 850 851 if (type == GITS_BASER_TYPE_NONE) 852 continue; 853 854 /* 855 * Allocate as many entries as required to fit the 856 * range of device IDs that the ITS can grok... The ID 857 * space being incredibly sparse, this results in a 858 * massive waste of memory. 859 * 860 * For other tables, only allocate a single page. 861 */ 862 if (type == GITS_BASER_TYPE_DEVICE) { 863 u64 typer = readq_relaxed(its->base + GITS_TYPER); 864 u32 ids = GITS_TYPER_DEVBITS(typer); 865 866 /* 867 * 'order' was initialized earlier to the default page 868 * granule of the the ITS. We can't have an allocation 869 * smaller than that. If the requested allocation 870 * is smaller, round up to the default page granule. 871 */ 872 order = max(get_order((1UL << ids) * entry_size), 873 order); 874 if (order >= MAX_ORDER) { 875 order = MAX_ORDER - 1; 876 pr_warn("%s: Device Table too large, reduce its page order to %u\n", 877 its->msi_chip.of_node->full_name, order); 878 } 879 } 880 881 alloc_size = (1 << order) * PAGE_SIZE; 882 base = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order); 883 if (!base) { 884 err = -ENOMEM; 885 goto out_free; 886 } 887 888 its->tables[i] = base; 889 890retry_baser: 891 val = (virt_to_phys(base) | 892 (type << GITS_BASER_TYPE_SHIFT) | 893 ((entry_size - 1) << GITS_BASER_ENTRY_SIZE_SHIFT) | 894 cache | 895 shr | 896 GITS_BASER_VALID); 897 898 switch (psz) { 899 case SZ_4K: 900 val |= GITS_BASER_PAGE_SIZE_4K; 901 break; 902 case SZ_16K: 903 val |= GITS_BASER_PAGE_SIZE_16K; 904 break; 905 case SZ_64K: 906 val |= GITS_BASER_PAGE_SIZE_64K; 907 break; 908 } 909 910 val |= (alloc_size / psz) - 1; 911 912 writeq_relaxed(val, its->base + GITS_BASER + i * 8); 913 tmp = readq_relaxed(its->base + GITS_BASER + i * 8); 914 915 if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) { 916 /* 917 * Shareability didn't stick. Just use 918 * whatever the read reported, which is likely 919 * to be the only thing this redistributor 920 * supports. If that's zero, make it 921 * non-cacheable as well. 922 */ 923 shr = tmp & GITS_BASER_SHAREABILITY_MASK; 924 if (!shr) { 925 cache = GITS_BASER_nC; 926 __flush_dcache_area(base, alloc_size); 927 } 928 goto retry_baser; 929 } 930 931 if ((val ^ tmp) & GITS_BASER_PAGE_SIZE_MASK) { 932 /* 933 * Page size didn't stick. Let's try a smaller 934 * size and retry. If we reach 4K, then 935 * something is horribly wrong... 936 */ 937 switch (psz) { 938 case SZ_16K: 939 psz = SZ_4K; 940 goto retry_baser; 941 case SZ_64K: 942 psz = SZ_16K; 943 goto retry_baser; 944 } 945 } 946 947 if (val != tmp) { 948 pr_err("ITS: %s: GITS_BASER%d doesn't stick: %lx %lx\n", 949 its->msi_chip.of_node->full_name, i, 950 (unsigned long) val, (unsigned long) tmp); 951 err = -ENXIO; 952 goto out_free; 953 } 954 955 pr_info("ITS: allocated %d %s @%lx (psz %dK, shr %d)\n", 956 (int)(alloc_size / entry_size), 957 its_base_type_string[type], 958 (unsigned long)virt_to_phys(base), 959 psz / SZ_1K, (int)shr >> GITS_BASER_SHAREABILITY_SHIFT); 960 } 961 962 return 0; 963 964out_free: 965 its_free_tables(its); 966 967 return err; 968} 969 970static int its_alloc_collections(struct its_node *its) 971{ 972 its->collections = kzalloc(nr_cpu_ids * sizeof(*its->collections), 973 GFP_KERNEL); 974 if (!its->collections) 975 return -ENOMEM; 976 977 return 0; 978} 979 980static void its_cpu_init_lpis(void) 981{ 982 void __iomem *rbase = gic_data_rdist_rd_base(); 983 struct page *pend_page; 984 u64 val, tmp; 985 986 /* If we didn't allocate the pending table yet, do it now */ 987 pend_page = gic_data_rdist()->pend_page; 988 if (!pend_page) { 989 phys_addr_t paddr; 990 /* 991 * The pending pages have to be at least 64kB aligned, 992 * hence the 'max(LPI_PENDBASE_SZ, SZ_64K)' below. 993 */ 994 pend_page = alloc_pages(GFP_NOWAIT | __GFP_ZERO, 995 get_order(max(LPI_PENDBASE_SZ, SZ_64K))); 996 if (!pend_page) { 997 pr_err("Failed to allocate PENDBASE for CPU%d\n", 998 smp_processor_id()); 999 return; 1000 } 1001 1002 /* Make sure the GIC will observe the zero-ed page */ 1003 __flush_dcache_area(page_address(pend_page), LPI_PENDBASE_SZ); 1004 1005 paddr = page_to_phys(pend_page); 1006 pr_info("CPU%d: using LPI pending table @%pa\n", 1007 smp_processor_id(), &paddr); 1008 gic_data_rdist()->pend_page = pend_page; 1009 } 1010 1011 /* Disable LPIs */ 1012 val = readl_relaxed(rbase + GICR_CTLR); 1013 val &= ~GICR_CTLR_ENABLE_LPIS; 1014 writel_relaxed(val, rbase + GICR_CTLR); 1015 1016 /* 1017 * Make sure any change to the table is observable by the GIC. 1018 */ 1019 dsb(sy); 1020 1021 /* set PROPBASE */ 1022 val = (page_to_phys(gic_rdists->prop_page) | 1023 GICR_PROPBASER_InnerShareable | 1024 GICR_PROPBASER_WaWb | 1025 ((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK)); 1026 1027 writeq_relaxed(val, rbase + GICR_PROPBASER); 1028 tmp = readq_relaxed(rbase + GICR_PROPBASER); 1029 1030 if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) { 1031 if (!(tmp & GICR_PROPBASER_SHAREABILITY_MASK)) { 1032 /* 1033 * The HW reports non-shareable, we must 1034 * remove the cacheability attributes as 1035 * well. 1036 */ 1037 val &= ~(GICR_PROPBASER_SHAREABILITY_MASK | 1038 GICR_PROPBASER_CACHEABILITY_MASK); 1039 val |= GICR_PROPBASER_nC; 1040 writeq_relaxed(val, rbase + GICR_PROPBASER); 1041 } 1042 pr_info_once("GIC: using cache flushing for LPI property table\n"); 1043 gic_rdists->flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING; 1044 } 1045 1046 /* set PENDBASE */ 1047 val = (page_to_phys(pend_page) | 1048 GICR_PENDBASER_InnerShareable | 1049 GICR_PENDBASER_WaWb); 1050 1051 writeq_relaxed(val, rbase + GICR_PENDBASER); 1052 tmp = readq_relaxed(rbase + GICR_PENDBASER); 1053 1054 if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) { 1055 /* 1056 * The HW reports non-shareable, we must remove the 1057 * cacheability attributes as well. 1058 */ 1059 val &= ~(GICR_PENDBASER_SHAREABILITY_MASK | 1060 GICR_PENDBASER_CACHEABILITY_MASK); 1061 val |= GICR_PENDBASER_nC; 1062 writeq_relaxed(val, rbase + GICR_PENDBASER); 1063 } 1064 1065 /* Enable LPIs */ 1066 val = readl_relaxed(rbase + GICR_CTLR); 1067 val |= GICR_CTLR_ENABLE_LPIS; 1068 writel_relaxed(val, rbase + GICR_CTLR); 1069 1070 /* Make sure the GIC has seen the above */ 1071 dsb(sy); 1072} 1073 1074static void its_cpu_init_collection(void) 1075{ 1076 struct its_node *its; 1077 int cpu; 1078 1079 spin_lock(&its_lock); 1080 cpu = smp_processor_id(); 1081 1082 list_for_each_entry(its, &its_nodes, entry) { 1083 u64 target; 1084 1085 /* 1086 * We now have to bind each collection to its target 1087 * redistributor. 1088 */ 1089 if (readq_relaxed(its->base + GITS_TYPER) & GITS_TYPER_PTA) { 1090 /* 1091 * This ITS wants the physical address of the 1092 * redistributor. 1093 */ 1094 target = gic_data_rdist()->phys_base; 1095 } else { 1096 /* 1097 * This ITS wants a linear CPU number. 1098 */ 1099 target = readq_relaxed(gic_data_rdist_rd_base() + GICR_TYPER); 1100 target = GICR_TYPER_CPU_NUMBER(target) << 16; 1101 } 1102 1103 /* Perform collection mapping */ 1104 its->collections[cpu].target_address = target; 1105 its->collections[cpu].col_id = cpu; 1106 1107 its_send_mapc(its, &its->collections[cpu], 1); 1108 its_send_invall(its, &its->collections[cpu]); 1109 } 1110 1111 spin_unlock(&its_lock); 1112} 1113 1114static struct its_device *its_find_device(struct its_node *its, u32 dev_id) 1115{ 1116 struct its_device *its_dev = NULL, *tmp; 1117 unsigned long flags; 1118 1119 raw_spin_lock_irqsave(&its->lock, flags); 1120 1121 list_for_each_entry(tmp, &its->its_device_list, entry) { 1122 if (tmp->device_id == dev_id) { 1123 its_dev = tmp; 1124 break; 1125 } 1126 } 1127 1128 raw_spin_unlock_irqrestore(&its->lock, flags); 1129 1130 return its_dev; 1131} 1132 1133static struct its_device *its_create_device(struct its_node *its, u32 dev_id, 1134 int nvecs) 1135{ 1136 struct its_device *dev; 1137 unsigned long *lpi_map; 1138 unsigned long flags; 1139 u16 *col_map = NULL; 1140 void *itt; 1141 int lpi_base; 1142 int nr_lpis; 1143 int nr_ites; 1144 int sz; 1145 1146 dev = kzalloc(sizeof(*dev), GFP_KERNEL); 1147 /* 1148 * At least one bit of EventID is being used, hence a minimum 1149 * of two entries. No, the architecture doesn't let you 1150 * express an ITT with a single entry. 1151 */ 1152 nr_ites = max(2UL, roundup_pow_of_two(nvecs)); 1153 sz = nr_ites * its->ite_size; 1154 sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1; 1155 itt = kzalloc(sz, GFP_KERNEL); 1156 lpi_map = its_lpi_alloc_chunks(nvecs, &lpi_base, &nr_lpis); 1157 if (lpi_map) 1158 col_map = kzalloc(sizeof(*col_map) * nr_lpis, GFP_KERNEL); 1159 1160 if (!dev || !itt || !lpi_map || !col_map) { 1161 kfree(dev); 1162 kfree(itt); 1163 kfree(lpi_map); 1164 kfree(col_map); 1165 return NULL; 1166 } 1167 1168 __flush_dcache_area(itt, sz); 1169 1170 dev->its = its; 1171 dev->itt = itt; 1172 dev->nr_ites = nr_ites; 1173 dev->event_map.lpi_map = lpi_map; 1174 dev->event_map.col_map = col_map; 1175 dev->event_map.lpi_base = lpi_base; 1176 dev->event_map.nr_lpis = nr_lpis; 1177 dev->device_id = dev_id; 1178 INIT_LIST_HEAD(&dev->entry); 1179 1180 raw_spin_lock_irqsave(&its->lock, flags); 1181 list_add(&dev->entry, &its->its_device_list); 1182 raw_spin_unlock_irqrestore(&its->lock, flags); 1183 1184 /* Map device to its ITT */ 1185 its_send_mapd(dev, 1); 1186 1187 return dev; 1188} 1189 1190static void its_free_device(struct its_device *its_dev) 1191{ 1192 unsigned long flags; 1193 1194 raw_spin_lock_irqsave(&its_dev->its->lock, flags); 1195 list_del(&its_dev->entry); 1196 raw_spin_unlock_irqrestore(&its_dev->its->lock, flags); 1197 kfree(its_dev->itt); 1198 kfree(its_dev); 1199} 1200 1201static int its_alloc_device_irq(struct its_device *dev, irq_hw_number_t *hwirq) 1202{ 1203 int idx; 1204 1205 idx = find_first_zero_bit(dev->event_map.lpi_map, 1206 dev->event_map.nr_lpis); 1207 if (idx == dev->event_map.nr_lpis) 1208 return -ENOSPC; 1209 1210 *hwirq = dev->event_map.lpi_base + idx; 1211 set_bit(idx, dev->event_map.lpi_map); 1212 1213 return 0; 1214} 1215 1216struct its_pci_alias { 1217 struct pci_dev *pdev; 1218 u32 dev_id; 1219 u32 count; 1220}; 1221 1222static int its_pci_msi_vec_count(struct pci_dev *pdev) 1223{ 1224 int msi, msix; 1225 1226 msi = max(pci_msi_vec_count(pdev), 0); 1227 msix = max(pci_msix_vec_count(pdev), 0); 1228 1229 return max(msi, msix); 1230} 1231 1232static int its_get_pci_alias(struct pci_dev *pdev, u16 alias, void *data) 1233{ 1234 struct its_pci_alias *dev_alias = data; 1235 1236 dev_alias->dev_id = alias; 1237 if (pdev != dev_alias->pdev) 1238 dev_alias->count += its_pci_msi_vec_count(dev_alias->pdev); 1239 1240 return 0; 1241} 1242 1243static int its_msi_prepare(struct irq_domain *domain, struct device *dev, 1244 int nvec, msi_alloc_info_t *info) 1245{ 1246 struct pci_dev *pdev; 1247 struct its_node *its; 1248 struct its_device *its_dev; 1249 struct its_pci_alias dev_alias; 1250 1251 if (!dev_is_pci(dev)) 1252 return -EINVAL; 1253 1254 pdev = to_pci_dev(dev); 1255 dev_alias.pdev = pdev; 1256 dev_alias.count = nvec; 1257 1258 pci_for_each_dma_alias(pdev, its_get_pci_alias, &dev_alias); 1259 its = domain->parent->host_data; 1260 1261 its_dev = its_find_device(its, dev_alias.dev_id); 1262 if (its_dev) { 1263 /* 1264 * We already have seen this ID, probably through 1265 * another alias (PCI bridge of some sort). No need to 1266 * create the device. 1267 */ 1268 dev_dbg(dev, "Reusing ITT for devID %x\n", dev_alias.dev_id); 1269 goto out; 1270 } 1271 1272 its_dev = its_create_device(its, dev_alias.dev_id, dev_alias.count); 1273 if (!its_dev) 1274 return -ENOMEM; 1275 1276 dev_dbg(&pdev->dev, "ITT %d entries, %d bits\n", 1277 dev_alias.count, ilog2(dev_alias.count)); 1278out: 1279 info->scratchpad[0].ptr = its_dev; 1280 info->scratchpad[1].ptr = dev; 1281 return 0; 1282} 1283 1284static struct msi_domain_ops its_pci_msi_ops = { 1285 .msi_prepare = its_msi_prepare, 1286}; 1287 1288static struct msi_domain_info its_pci_msi_domain_info = { 1289 .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | 1290 MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX), 1291 .ops = &its_pci_msi_ops, 1292 .chip = &its_msi_irq_chip, 1293}; 1294 1295static int its_irq_gic_domain_alloc(struct irq_domain *domain, 1296 unsigned int virq, 1297 irq_hw_number_t hwirq) 1298{ 1299 struct of_phandle_args args; 1300 1301 args.np = domain->parent->of_node; 1302 args.args_count = 3; 1303 args.args[0] = GIC_IRQ_TYPE_LPI; 1304 args.args[1] = hwirq; 1305 args.args[2] = IRQ_TYPE_EDGE_RISING; 1306 1307 return irq_domain_alloc_irqs_parent(domain, virq, 1, &args); 1308} 1309 1310static int its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 1311 unsigned int nr_irqs, void *args) 1312{ 1313 msi_alloc_info_t *info = args; 1314 struct its_device *its_dev = info->scratchpad[0].ptr; 1315 irq_hw_number_t hwirq; 1316 int err; 1317 int i; 1318 1319 for (i = 0; i < nr_irqs; i++) { 1320 err = its_alloc_device_irq(its_dev, &hwirq); 1321 if (err) 1322 return err; 1323 1324 err = its_irq_gic_domain_alloc(domain, virq + i, hwirq); 1325 if (err) 1326 return err; 1327 1328 irq_domain_set_hwirq_and_chip(domain, virq + i, 1329 hwirq, &its_irq_chip, its_dev); 1330 dev_dbg(info->scratchpad[1].ptr, "ID:%d pID:%d vID:%d\n", 1331 (int)(hwirq - its_dev->event_map.lpi_base), 1332 (int)hwirq, virq + i); 1333 } 1334 1335 return 0; 1336} 1337 1338static void its_irq_domain_activate(struct irq_domain *domain, 1339 struct irq_data *d) 1340{ 1341 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1342 u32 event = its_get_event_id(d); 1343 1344 /* Bind the LPI to the first possible CPU */ 1345 its_dev->event_map.col_map[event] = cpumask_first(cpu_online_mask); 1346 1347 /* Map the GIC IRQ and event to the device */ 1348 its_send_mapvi(its_dev, d->hwirq, event); 1349} 1350 1351static void its_irq_domain_deactivate(struct irq_domain *domain, 1352 struct irq_data *d) 1353{ 1354 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1355 u32 event = its_get_event_id(d); 1356 1357 /* Stop the delivery of interrupts */ 1358 its_send_discard(its_dev, event); 1359} 1360 1361static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq, 1362 unsigned int nr_irqs) 1363{ 1364 struct irq_data *d = irq_domain_get_irq_data(domain, virq); 1365 struct its_device *its_dev = irq_data_get_irq_chip_data(d); 1366 int i; 1367 1368 for (i = 0; i < nr_irqs; i++) { 1369 struct irq_data *data = irq_domain_get_irq_data(domain, 1370 virq + i); 1371 u32 event = its_get_event_id(data); 1372 1373 /* Mark interrupt index as unused */ 1374 clear_bit(event, its_dev->event_map.lpi_map); 1375 1376 /* Nuke the entry in the domain */ 1377 irq_domain_reset_irq_data(data); 1378 } 1379 1380 /* If all interrupts have been freed, start mopping the floor */ 1381 if (bitmap_empty(its_dev->event_map.lpi_map, 1382 its_dev->event_map.nr_lpis)) { 1383 its_lpi_free(&its_dev->event_map); 1384 1385 /* Unmap device/itt */ 1386 its_send_mapd(its_dev, 0); 1387 its_free_device(its_dev); 1388 } 1389 1390 irq_domain_free_irqs_parent(domain, virq, nr_irqs); 1391} 1392 1393static const struct irq_domain_ops its_domain_ops = { 1394 .alloc = its_irq_domain_alloc, 1395 .free = its_irq_domain_free, 1396 .activate = its_irq_domain_activate, 1397 .deactivate = its_irq_domain_deactivate, 1398}; 1399 1400static int its_force_quiescent(void __iomem *base) 1401{ 1402 u32 count = 1000000; /* 1s */ 1403 u32 val; 1404 1405 val = readl_relaxed(base + GITS_CTLR); 1406 if (val & GITS_CTLR_QUIESCENT) 1407 return 0; 1408 1409 /* Disable the generation of all interrupts to this ITS */ 1410 val &= ~GITS_CTLR_ENABLE; 1411 writel_relaxed(val, base + GITS_CTLR); 1412 1413 /* Poll GITS_CTLR and wait until ITS becomes quiescent */ 1414 while (1) { 1415 val = readl_relaxed(base + GITS_CTLR); 1416 if (val & GITS_CTLR_QUIESCENT) 1417 return 0; 1418 1419 count--; 1420 if (!count) 1421 return -EBUSY; 1422 1423 cpu_relax(); 1424 udelay(1); 1425 } 1426} 1427 1428static int its_probe(struct device_node *node, struct irq_domain *parent) 1429{ 1430 struct resource res; 1431 struct its_node *its; 1432 void __iomem *its_base; 1433 u32 val; 1434 u64 baser, tmp; 1435 int err; 1436 1437 err = of_address_to_resource(node, 0, &res); 1438 if (err) { 1439 pr_warn("%s: no regs?\n", node->full_name); 1440 return -ENXIO; 1441 } 1442 1443 its_base = ioremap(res.start, resource_size(&res)); 1444 if (!its_base) { 1445 pr_warn("%s: unable to map registers\n", node->full_name); 1446 return -ENOMEM; 1447 } 1448 1449 val = readl_relaxed(its_base + GITS_PIDR2) & GIC_PIDR2_ARCH_MASK; 1450 if (val != 0x30 && val != 0x40) { 1451 pr_warn("%s: no ITS detected, giving up\n", node->full_name); 1452 err = -ENODEV; 1453 goto out_unmap; 1454 } 1455 1456 err = its_force_quiescent(its_base); 1457 if (err) { 1458 pr_warn("%s: failed to quiesce, giving up\n", 1459 node->full_name); 1460 goto out_unmap; 1461 } 1462 1463 pr_info("ITS: %s\n", node->full_name); 1464 1465 its = kzalloc(sizeof(*its), GFP_KERNEL); 1466 if (!its) { 1467 err = -ENOMEM; 1468 goto out_unmap; 1469 } 1470 1471 raw_spin_lock_init(&its->lock); 1472 INIT_LIST_HEAD(&its->entry); 1473 INIT_LIST_HEAD(&its->its_device_list); 1474 its->base = its_base; 1475 its->phys_base = res.start; 1476 its->msi_chip.of_node = node; 1477 its->ite_size = ((readl_relaxed(its_base + GITS_TYPER) >> 4) & 0xf) + 1; 1478 1479 its->cmd_base = kzalloc(ITS_CMD_QUEUE_SZ, GFP_KERNEL); 1480 if (!its->cmd_base) { 1481 err = -ENOMEM; 1482 goto out_free_its; 1483 } 1484 its->cmd_write = its->cmd_base; 1485 1486 err = its_alloc_tables(its); 1487 if (err) 1488 goto out_free_cmd; 1489 1490 err = its_alloc_collections(its); 1491 if (err) 1492 goto out_free_tables; 1493 1494 baser = (virt_to_phys(its->cmd_base) | 1495 GITS_CBASER_WaWb | 1496 GITS_CBASER_InnerShareable | 1497 (ITS_CMD_QUEUE_SZ / SZ_4K - 1) | 1498 GITS_CBASER_VALID); 1499 1500 writeq_relaxed(baser, its->base + GITS_CBASER); 1501 tmp = readq_relaxed(its->base + GITS_CBASER); 1502 1503 if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) { 1504 if (!(tmp & GITS_CBASER_SHAREABILITY_MASK)) { 1505 /* 1506 * The HW reports non-shareable, we must 1507 * remove the cacheability attributes as 1508 * well. 1509 */ 1510 baser &= ~(GITS_CBASER_SHAREABILITY_MASK | 1511 GITS_CBASER_CACHEABILITY_MASK); 1512 baser |= GITS_CBASER_nC; 1513 writeq_relaxed(baser, its->base + GITS_CBASER); 1514 } 1515 pr_info("ITS: using cache flushing for cmd queue\n"); 1516 its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING; 1517 } 1518 1519 writeq_relaxed(0, its->base + GITS_CWRITER); 1520 writel_relaxed(GITS_CTLR_ENABLE, its->base + GITS_CTLR); 1521 1522 if (of_property_read_bool(its->msi_chip.of_node, "msi-controller")) { 1523 its->domain = irq_domain_add_tree(NULL, &its_domain_ops, its); 1524 if (!its->domain) { 1525 err = -ENOMEM; 1526 goto out_free_tables; 1527 } 1528 1529 its->domain->parent = parent; 1530 1531 its->msi_chip.domain = pci_msi_create_irq_domain(node, 1532 &its_pci_msi_domain_info, 1533 its->domain); 1534 if (!its->msi_chip.domain) { 1535 err = -ENOMEM; 1536 goto out_free_domains; 1537 } 1538 1539 err = of_pci_msi_chip_add(&its->msi_chip); 1540 if (err) 1541 goto out_free_domains; 1542 } 1543 1544 spin_lock(&its_lock); 1545 list_add(&its->entry, &its_nodes); 1546 spin_unlock(&its_lock); 1547 1548 return 0; 1549 1550out_free_domains: 1551 if (its->msi_chip.domain) 1552 irq_domain_remove(its->msi_chip.domain); 1553 if (its->domain) 1554 irq_domain_remove(its->domain); 1555out_free_tables: 1556 its_free_tables(its); 1557out_free_cmd: 1558 kfree(its->cmd_base); 1559out_free_its: 1560 kfree(its); 1561out_unmap: 1562 iounmap(its_base); 1563 pr_err("ITS: failed probing %s (%d)\n", node->full_name, err); 1564 return err; 1565} 1566 1567static bool gic_rdists_supports_plpis(void) 1568{ 1569 return !!(readl_relaxed(gic_data_rdist_rd_base() + GICR_TYPER) & GICR_TYPER_PLPIS); 1570} 1571 1572int its_cpu_init(void) 1573{ 1574 if (!list_empty(&its_nodes)) { 1575 if (!gic_rdists_supports_plpis()) { 1576 pr_info("CPU%d: LPIs not supported\n", smp_processor_id()); 1577 return -ENXIO; 1578 } 1579 its_cpu_init_lpis(); 1580 its_cpu_init_collection(); 1581 } 1582 1583 return 0; 1584} 1585 1586static struct of_device_id its_device_id[] = { 1587 { .compatible = "arm,gic-v3-its", }, 1588 {}, 1589}; 1590 1591int its_init(struct device_node *node, struct rdists *rdists, 1592 struct irq_domain *parent_domain) 1593{ 1594 struct device_node *np; 1595 1596 for (np = of_find_matching_node(node, its_device_id); np; 1597 np = of_find_matching_node(np, its_device_id)) { 1598 its_probe(np, parent_domain); 1599 } 1600 1601 if (list_empty(&its_nodes)) { 1602 pr_warn("ITS: No ITS available, not enabling LPIs\n"); 1603 return -ENXIO; 1604 } 1605 1606 gic_rdists = rdists; 1607 gic_root_node = node; 1608 1609 its_alloc_lpi_tables(); 1610 its_lpi_init(rdists->id_bits); 1611 1612 return 0; 1613} 1614