1/* 2 * spu_switch.c 3 * 4 * (C) Copyright IBM Corp. 2005 5 * 6 * Author: Mark Nutter <mnutter@us.ibm.com> 7 * 8 * Host-side part of SPU context switch sequence outlined in 9 * Synergistic Processor Element, Book IV. 10 * 11 * A fully premptive switch of an SPE is very expensive in terms 12 * of time and system resources. SPE Book IV indicates that SPE 13 * allocation should follow a "serially reusable device" model, 14 * in which the SPE is assigned a task until it completes. When 15 * this is not possible, this sequence may be used to premptively 16 * save, and then later (optionally) restore the context of a 17 * program executing on an SPE. 18 * 19 * 20 * This program is free software; you can redistribute it and/or modify 21 * it under the terms of the GNU General Public License as published by 22 * the Free Software Foundation; either version 2, or (at your option) 23 * any later version. 24 * 25 * This program is distributed in the hope that it will be useful, 26 * but WITHOUT ANY WARRANTY; without even the implied warranty of 27 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 28 * GNU General Public License for more details. 29 * 30 * You should have received a copy of the GNU General Public License 31 * along with this program; if not, write to the Free Software 32 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 33 */ 34 35#include <linux/export.h> 36#include <linux/errno.h> 37#include <linux/hardirq.h> 38#include <linux/sched.h> 39#include <linux/kernel.h> 40#include <linux/mm.h> 41#include <linux/vmalloc.h> 42#include <linux/smp.h> 43#include <linux/stddef.h> 44#include <linux/unistd.h> 45 46#include <asm/io.h> 47#include <asm/spu.h> 48#include <asm/spu_priv1.h> 49#include <asm/spu_csa.h> 50#include <asm/mmu_context.h> 51 52#include "spufs.h" 53 54#include "spu_save_dump.h" 55#include "spu_restore_dump.h" 56 57#if 0 58#define POLL_WHILE_TRUE(_c) { \ 59 do { \ 60 } while (_c); \ 61 } 62#else 63#define RELAX_SPIN_COUNT 1000 64#define POLL_WHILE_TRUE(_c) { \ 65 do { \ 66 int _i; \ 67 for (_i=0; _i<RELAX_SPIN_COUNT && (_c); _i++) { \ 68 cpu_relax(); \ 69 } \ 70 if (unlikely(_c)) yield(); \ 71 else break; \ 72 } while (_c); \ 73 } 74#endif /* debug */ 75 76#define POLL_WHILE_FALSE(_c) POLL_WHILE_TRUE(!(_c)) 77 78static inline void acquire_spu_lock(struct spu *spu) 79{ 80 /* Save, Step 1: 81 * Restore, Step 1: 82 * Acquire SPU-specific mutual exclusion lock. 83 * TBD. 84 */ 85} 86 87static inline void release_spu_lock(struct spu *spu) 88{ 89 /* Restore, Step 76: 90 * Release SPU-specific mutual exclusion lock. 91 * TBD. 92 */ 93} 94 95static inline int check_spu_isolate(struct spu_state *csa, struct spu *spu) 96{ 97 struct spu_problem __iomem *prob = spu->problem; 98 u32 isolate_state; 99 100 /* Save, Step 2: 101 * Save, Step 6: 102 * If SPU_Status[E,L,IS] any field is '1', this 103 * SPU is in isolate state and cannot be context 104 * saved at this time. 105 */ 106 isolate_state = SPU_STATUS_ISOLATED_STATE | 107 SPU_STATUS_ISOLATED_LOAD_STATUS | SPU_STATUS_ISOLATED_EXIT_STATUS; 108 return (in_be32(&prob->spu_status_R) & isolate_state) ? 1 : 0; 109} 110 111static inline void disable_interrupts(struct spu_state *csa, struct spu *spu) 112{ 113 /* Save, Step 3: 114 * Restore, Step 2: 115 * Save INT_Mask_class0 in CSA. 116 * Write INT_MASK_class0 with value of 0. 117 * Save INT_Mask_class1 in CSA. 118 * Write INT_MASK_class1 with value of 0. 119 * Save INT_Mask_class2 in CSA. 120 * Write INT_MASK_class2 with value of 0. 121 * Synchronize all three interrupts to be sure 122 * we no longer execute a handler on another CPU. 123 */ 124 spin_lock_irq(&spu->register_lock); 125 if (csa) { 126 csa->priv1.int_mask_class0_RW = spu_int_mask_get(spu, 0); 127 csa->priv1.int_mask_class1_RW = spu_int_mask_get(spu, 1); 128 csa->priv1.int_mask_class2_RW = spu_int_mask_get(spu, 2); 129 } 130 spu_int_mask_set(spu, 0, 0ul); 131 spu_int_mask_set(spu, 1, 0ul); 132 spu_int_mask_set(spu, 2, 0ul); 133 eieio(); 134 spin_unlock_irq(&spu->register_lock); 135 136 /* 137 * This flag needs to be set before calling synchronize_irq so 138 * that the update will be visible to the relevant handlers 139 * via a simple load. 140 */ 141 set_bit(SPU_CONTEXT_SWITCH_PENDING, &spu->flags); 142 clear_bit(SPU_CONTEXT_FAULT_PENDING, &spu->flags); 143 synchronize_irq(spu->irqs[0]); 144 synchronize_irq(spu->irqs[1]); 145 synchronize_irq(spu->irqs[2]); 146} 147 148static inline void set_watchdog_timer(struct spu_state *csa, struct spu *spu) 149{ 150 /* Save, Step 4: 151 * Restore, Step 25. 152 * Set a software watchdog timer, which specifies the 153 * maximum allowable time for a context save sequence. 154 * 155 * For present, this implementation will not set a global 156 * watchdog timer, as virtualization & variable system load 157 * may cause unpredictable execution times. 158 */ 159} 160 161static inline void inhibit_user_access(struct spu_state *csa, struct spu *spu) 162{ 163 /* Save, Step 5: 164 * Restore, Step 3: 165 * Inhibit user-space access (if provided) to this 166 * SPU by unmapping the virtual pages assigned to 167 * the SPU memory-mapped I/O (MMIO) for problem 168 * state. TBD. 169 */ 170} 171 172static inline void set_switch_pending(struct spu_state *csa, struct spu *spu) 173{ 174 /* Save, Step 7: 175 * Restore, Step 5: 176 * Set a software context switch pending flag. 177 * Done above in Step 3 - disable_interrupts(). 178 */ 179} 180 181static inline void save_mfc_cntl(struct spu_state *csa, struct spu *spu) 182{ 183 struct spu_priv2 __iomem *priv2 = spu->priv2; 184 185 /* Save, Step 8: 186 * Suspend DMA and save MFC_CNTL. 187 */ 188 switch (in_be64(&priv2->mfc_control_RW) & 189 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) { 190 case MFC_CNTL_SUSPEND_IN_PROGRESS: 191 POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) & 192 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) == 193 MFC_CNTL_SUSPEND_COMPLETE); 194 /* fall through */ 195 case MFC_CNTL_SUSPEND_COMPLETE: 196 if (csa) 197 csa->priv2.mfc_control_RW = 198 in_be64(&priv2->mfc_control_RW) | 199 MFC_CNTL_SUSPEND_DMA_QUEUE; 200 break; 201 case MFC_CNTL_NORMAL_DMA_QUEUE_OPERATION: 202 out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE); 203 POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) & 204 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) == 205 MFC_CNTL_SUSPEND_COMPLETE); 206 if (csa) 207 csa->priv2.mfc_control_RW = 208 in_be64(&priv2->mfc_control_RW) & 209 ~MFC_CNTL_SUSPEND_DMA_QUEUE & 210 ~MFC_CNTL_SUSPEND_MASK; 211 break; 212 } 213} 214 215static inline void save_spu_runcntl(struct spu_state *csa, struct spu *spu) 216{ 217 struct spu_problem __iomem *prob = spu->problem; 218 219 /* Save, Step 9: 220 * Save SPU_Runcntl in the CSA. This value contains 221 * the "Application Desired State". 222 */ 223 csa->prob.spu_runcntl_RW = in_be32(&prob->spu_runcntl_RW); 224} 225 226static inline void save_mfc_sr1(struct spu_state *csa, struct spu *spu) 227{ 228 /* Save, Step 10: 229 * Save MFC_SR1 in the CSA. 230 */ 231 csa->priv1.mfc_sr1_RW = spu_mfc_sr1_get(spu); 232} 233 234static inline void save_spu_status(struct spu_state *csa, struct spu *spu) 235{ 236 struct spu_problem __iomem *prob = spu->problem; 237 238 /* Save, Step 11: 239 * Read SPU_Status[R], and save to CSA. 240 */ 241 if ((in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) == 0) { 242 csa->prob.spu_status_R = in_be32(&prob->spu_status_R); 243 } else { 244 u32 stopped; 245 246 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 247 eieio(); 248 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 249 SPU_STATUS_RUNNING); 250 stopped = 251 SPU_STATUS_INVALID_INSTR | SPU_STATUS_SINGLE_STEP | 252 SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP; 253 if ((in_be32(&prob->spu_status_R) & stopped) == 0) 254 csa->prob.spu_status_R = SPU_STATUS_RUNNING; 255 else 256 csa->prob.spu_status_R = in_be32(&prob->spu_status_R); 257 } 258} 259 260static inline void save_mfc_stopped_status(struct spu_state *csa, 261 struct spu *spu) 262{ 263 struct spu_priv2 __iomem *priv2 = spu->priv2; 264 const u64 mask = MFC_CNTL_DECREMENTER_RUNNING | 265 MFC_CNTL_DMA_QUEUES_EMPTY; 266 267 /* Save, Step 12: 268 * Read MFC_CNTL[Ds]. Update saved copy of 269 * CSA.MFC_CNTL[Ds]. 270 * 271 * update: do the same with MFC_CNTL[Q]. 272 */ 273 csa->priv2.mfc_control_RW &= ~mask; 274 csa->priv2.mfc_control_RW |= in_be64(&priv2->mfc_control_RW) & mask; 275} 276 277static inline void halt_mfc_decr(struct spu_state *csa, struct spu *spu) 278{ 279 struct spu_priv2 __iomem *priv2 = spu->priv2; 280 281 /* Save, Step 13: 282 * Write MFC_CNTL[Dh] set to a '1' to halt 283 * the decrementer. 284 */ 285 out_be64(&priv2->mfc_control_RW, 286 MFC_CNTL_DECREMENTER_HALTED | MFC_CNTL_SUSPEND_MASK); 287 eieio(); 288} 289 290static inline void save_timebase(struct spu_state *csa, struct spu *spu) 291{ 292 /* Save, Step 14: 293 * Read PPE Timebase High and Timebase low registers 294 * and save in CSA. TBD. 295 */ 296 csa->suspend_time = get_cycles(); 297} 298 299static inline void remove_other_spu_access(struct spu_state *csa, 300 struct spu *spu) 301{ 302 /* Save, Step 15: 303 * Remove other SPU access to this SPU by unmapping 304 * this SPU's pages from their address space. TBD. 305 */ 306} 307 308static inline void do_mfc_mssync(struct spu_state *csa, struct spu *spu) 309{ 310 struct spu_problem __iomem *prob = spu->problem; 311 312 /* Save, Step 16: 313 * Restore, Step 11. 314 * Write SPU_MSSync register. Poll SPU_MSSync[P] 315 * for a value of 0. 316 */ 317 out_be64(&prob->spc_mssync_RW, 1UL); 318 POLL_WHILE_TRUE(in_be64(&prob->spc_mssync_RW) & MS_SYNC_PENDING); 319} 320 321static inline void issue_mfc_tlbie(struct spu_state *csa, struct spu *spu) 322{ 323 /* Save, Step 17: 324 * Restore, Step 12. 325 * Restore, Step 48. 326 * Write TLB_Invalidate_Entry[IS,VPN,L,Lp]=0 register. 327 * Then issue a PPE sync instruction. 328 */ 329 spu_tlb_invalidate(spu); 330 mb(); 331} 332 333static inline void handle_pending_interrupts(struct spu_state *csa, 334 struct spu *spu) 335{ 336 /* Save, Step 18: 337 * Handle any pending interrupts from this SPU 338 * here. This is OS or hypervisor specific. One 339 * option is to re-enable interrupts to handle any 340 * pending interrupts, with the interrupt handlers 341 * recognizing the software Context Switch Pending 342 * flag, to ensure the SPU execution or MFC command 343 * queue is not restarted. TBD. 344 */ 345} 346 347static inline void save_mfc_queues(struct spu_state *csa, struct spu *spu) 348{ 349 struct spu_priv2 __iomem *priv2 = spu->priv2; 350 int i; 351 352 /* Save, Step 19: 353 * If MFC_Cntl[Se]=0 then save 354 * MFC command queues. 355 */ 356 if ((in_be64(&priv2->mfc_control_RW) & MFC_CNTL_DMA_QUEUES_EMPTY) == 0) { 357 for (i = 0; i < 8; i++) { 358 csa->priv2.puq[i].mfc_cq_data0_RW = 359 in_be64(&priv2->puq[i].mfc_cq_data0_RW); 360 csa->priv2.puq[i].mfc_cq_data1_RW = 361 in_be64(&priv2->puq[i].mfc_cq_data1_RW); 362 csa->priv2.puq[i].mfc_cq_data2_RW = 363 in_be64(&priv2->puq[i].mfc_cq_data2_RW); 364 csa->priv2.puq[i].mfc_cq_data3_RW = 365 in_be64(&priv2->puq[i].mfc_cq_data3_RW); 366 } 367 for (i = 0; i < 16; i++) { 368 csa->priv2.spuq[i].mfc_cq_data0_RW = 369 in_be64(&priv2->spuq[i].mfc_cq_data0_RW); 370 csa->priv2.spuq[i].mfc_cq_data1_RW = 371 in_be64(&priv2->spuq[i].mfc_cq_data1_RW); 372 csa->priv2.spuq[i].mfc_cq_data2_RW = 373 in_be64(&priv2->spuq[i].mfc_cq_data2_RW); 374 csa->priv2.spuq[i].mfc_cq_data3_RW = 375 in_be64(&priv2->spuq[i].mfc_cq_data3_RW); 376 } 377 } 378} 379 380static inline void save_ppu_querymask(struct spu_state *csa, struct spu *spu) 381{ 382 struct spu_problem __iomem *prob = spu->problem; 383 384 /* Save, Step 20: 385 * Save the PPU_QueryMask register 386 * in the CSA. 387 */ 388 csa->prob.dma_querymask_RW = in_be32(&prob->dma_querymask_RW); 389} 390 391static inline void save_ppu_querytype(struct spu_state *csa, struct spu *spu) 392{ 393 struct spu_problem __iomem *prob = spu->problem; 394 395 /* Save, Step 21: 396 * Save the PPU_QueryType register 397 * in the CSA. 398 */ 399 csa->prob.dma_querytype_RW = in_be32(&prob->dma_querytype_RW); 400} 401 402static inline void save_ppu_tagstatus(struct spu_state *csa, struct spu *spu) 403{ 404 struct spu_problem __iomem *prob = spu->problem; 405 406 /* Save the Prxy_TagStatus register in the CSA. 407 * 408 * It is unnecessary to restore dma_tagstatus_R, however, 409 * dma_tagstatus_R in the CSA is accessed via backing_ops, so 410 * we must save it. 411 */ 412 csa->prob.dma_tagstatus_R = in_be32(&prob->dma_tagstatus_R); 413} 414 415static inline void save_mfc_csr_tsq(struct spu_state *csa, struct spu *spu) 416{ 417 struct spu_priv2 __iomem *priv2 = spu->priv2; 418 419 /* Save, Step 22: 420 * Save the MFC_CSR_TSQ register 421 * in the LSCSA. 422 */ 423 csa->priv2.spu_tag_status_query_RW = 424 in_be64(&priv2->spu_tag_status_query_RW); 425} 426 427static inline void save_mfc_csr_cmd(struct spu_state *csa, struct spu *spu) 428{ 429 struct spu_priv2 __iomem *priv2 = spu->priv2; 430 431 /* Save, Step 23: 432 * Save the MFC_CSR_CMD1 and MFC_CSR_CMD2 433 * registers in the CSA. 434 */ 435 csa->priv2.spu_cmd_buf1_RW = in_be64(&priv2->spu_cmd_buf1_RW); 436 csa->priv2.spu_cmd_buf2_RW = in_be64(&priv2->spu_cmd_buf2_RW); 437} 438 439static inline void save_mfc_csr_ato(struct spu_state *csa, struct spu *spu) 440{ 441 struct spu_priv2 __iomem *priv2 = spu->priv2; 442 443 /* Save, Step 24: 444 * Save the MFC_CSR_ATO register in 445 * the CSA. 446 */ 447 csa->priv2.spu_atomic_status_RW = in_be64(&priv2->spu_atomic_status_RW); 448} 449 450static inline void save_mfc_tclass_id(struct spu_state *csa, struct spu *spu) 451{ 452 /* Save, Step 25: 453 * Save the MFC_TCLASS_ID register in 454 * the CSA. 455 */ 456 csa->priv1.mfc_tclass_id_RW = spu_mfc_tclass_id_get(spu); 457} 458 459static inline void set_mfc_tclass_id(struct spu_state *csa, struct spu *spu) 460{ 461 /* Save, Step 26: 462 * Restore, Step 23. 463 * Write the MFC_TCLASS_ID register with 464 * the value 0x10000000. 465 */ 466 spu_mfc_tclass_id_set(spu, 0x10000000); 467 eieio(); 468} 469 470static inline void purge_mfc_queue(struct spu_state *csa, struct spu *spu) 471{ 472 struct spu_priv2 __iomem *priv2 = spu->priv2; 473 474 /* Save, Step 27: 475 * Restore, Step 14. 476 * Write MFC_CNTL[Pc]=1 (purge queue). 477 */ 478 out_be64(&priv2->mfc_control_RW, 479 MFC_CNTL_PURGE_DMA_REQUEST | 480 MFC_CNTL_SUSPEND_MASK); 481 eieio(); 482} 483 484static inline void wait_purge_complete(struct spu_state *csa, struct spu *spu) 485{ 486 struct spu_priv2 __iomem *priv2 = spu->priv2; 487 488 /* Save, Step 28: 489 * Poll MFC_CNTL[Ps] until value '11' is read 490 * (purge complete). 491 */ 492 POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) & 493 MFC_CNTL_PURGE_DMA_STATUS_MASK) == 494 MFC_CNTL_PURGE_DMA_COMPLETE); 495} 496 497static inline void setup_mfc_sr1(struct spu_state *csa, struct spu *spu) 498{ 499 /* Save, Step 30: 500 * Restore, Step 18: 501 * Write MFC_SR1 with MFC_SR1[D=0,S=1] and 502 * MFC_SR1[TL,R,Pr,T] set correctly for the 503 * OS specific environment. 504 * 505 * Implementation note: The SPU-side code 506 * for save/restore is privileged, so the 507 * MFC_SR1[Pr] bit is not set. 508 * 509 */ 510 spu_mfc_sr1_set(spu, (MFC_STATE1_MASTER_RUN_CONTROL_MASK | 511 MFC_STATE1_RELOCATE_MASK | 512 MFC_STATE1_BUS_TLBIE_MASK)); 513} 514 515static inline void save_spu_npc(struct spu_state *csa, struct spu *spu) 516{ 517 struct spu_problem __iomem *prob = spu->problem; 518 519 /* Save, Step 31: 520 * Save SPU_NPC in the CSA. 521 */ 522 csa->prob.spu_npc_RW = in_be32(&prob->spu_npc_RW); 523} 524 525static inline void save_spu_privcntl(struct spu_state *csa, struct spu *spu) 526{ 527 struct spu_priv2 __iomem *priv2 = spu->priv2; 528 529 /* Save, Step 32: 530 * Save SPU_PrivCntl in the CSA. 531 */ 532 csa->priv2.spu_privcntl_RW = in_be64(&priv2->spu_privcntl_RW); 533} 534 535static inline void reset_spu_privcntl(struct spu_state *csa, struct spu *spu) 536{ 537 struct spu_priv2 __iomem *priv2 = spu->priv2; 538 539 /* Save, Step 33: 540 * Restore, Step 16: 541 * Write SPU_PrivCntl[S,Le,A] fields reset to 0. 542 */ 543 out_be64(&priv2->spu_privcntl_RW, 0UL); 544 eieio(); 545} 546 547static inline void save_spu_lslr(struct spu_state *csa, struct spu *spu) 548{ 549 struct spu_priv2 __iomem *priv2 = spu->priv2; 550 551 /* Save, Step 34: 552 * Save SPU_LSLR in the CSA. 553 */ 554 csa->priv2.spu_lslr_RW = in_be64(&priv2->spu_lslr_RW); 555} 556 557static inline void reset_spu_lslr(struct spu_state *csa, struct spu *spu) 558{ 559 struct spu_priv2 __iomem *priv2 = spu->priv2; 560 561 /* Save, Step 35: 562 * Restore, Step 17. 563 * Reset SPU_LSLR. 564 */ 565 out_be64(&priv2->spu_lslr_RW, LS_ADDR_MASK); 566 eieio(); 567} 568 569static inline void save_spu_cfg(struct spu_state *csa, struct spu *spu) 570{ 571 struct spu_priv2 __iomem *priv2 = spu->priv2; 572 573 /* Save, Step 36: 574 * Save SPU_Cfg in the CSA. 575 */ 576 csa->priv2.spu_cfg_RW = in_be64(&priv2->spu_cfg_RW); 577} 578 579static inline void save_pm_trace(struct spu_state *csa, struct spu *spu) 580{ 581 /* Save, Step 37: 582 * Save PM_Trace_Tag_Wait_Mask in the CSA. 583 * Not performed by this implementation. 584 */ 585} 586 587static inline void save_mfc_rag(struct spu_state *csa, struct spu *spu) 588{ 589 /* Save, Step 38: 590 * Save RA_GROUP_ID register and the 591 * RA_ENABLE reigster in the CSA. 592 */ 593 csa->priv1.resource_allocation_groupID_RW = 594 spu_resource_allocation_groupID_get(spu); 595 csa->priv1.resource_allocation_enable_RW = 596 spu_resource_allocation_enable_get(spu); 597} 598 599static inline void save_ppu_mb_stat(struct spu_state *csa, struct spu *spu) 600{ 601 struct spu_problem __iomem *prob = spu->problem; 602 603 /* Save, Step 39: 604 * Save MB_Stat register in the CSA. 605 */ 606 csa->prob.mb_stat_R = in_be32(&prob->mb_stat_R); 607} 608 609static inline void save_ppu_mb(struct spu_state *csa, struct spu *spu) 610{ 611 struct spu_problem __iomem *prob = spu->problem; 612 613 /* Save, Step 40: 614 * Save the PPU_MB register in the CSA. 615 */ 616 csa->prob.pu_mb_R = in_be32(&prob->pu_mb_R); 617} 618 619static inline void save_ppuint_mb(struct spu_state *csa, struct spu *spu) 620{ 621 struct spu_priv2 __iomem *priv2 = spu->priv2; 622 623 /* Save, Step 41: 624 * Save the PPUINT_MB register in the CSA. 625 */ 626 csa->priv2.puint_mb_R = in_be64(&priv2->puint_mb_R); 627} 628 629static inline void save_ch_part1(struct spu_state *csa, struct spu *spu) 630{ 631 struct spu_priv2 __iomem *priv2 = spu->priv2; 632 u64 idx, ch_indices[] = { 0UL, 3UL, 4UL, 24UL, 25UL, 27UL }; 633 int i; 634 635 /* Save, Step 42: 636 */ 637 638 /* Save CH 1, without channel count */ 639 out_be64(&priv2->spu_chnlcntptr_RW, 1); 640 csa->spu_chnldata_RW[1] = in_be64(&priv2->spu_chnldata_RW); 641 642 /* Save the following CH: [0,3,4,24,25,27] */ 643 for (i = 0; i < ARRAY_SIZE(ch_indices); i++) { 644 idx = ch_indices[i]; 645 out_be64(&priv2->spu_chnlcntptr_RW, idx); 646 eieio(); 647 csa->spu_chnldata_RW[idx] = in_be64(&priv2->spu_chnldata_RW); 648 csa->spu_chnlcnt_RW[idx] = in_be64(&priv2->spu_chnlcnt_RW); 649 out_be64(&priv2->spu_chnldata_RW, 0UL); 650 out_be64(&priv2->spu_chnlcnt_RW, 0UL); 651 eieio(); 652 } 653} 654 655static inline void save_spu_mb(struct spu_state *csa, struct spu *spu) 656{ 657 struct spu_priv2 __iomem *priv2 = spu->priv2; 658 int i; 659 660 /* Save, Step 43: 661 * Save SPU Read Mailbox Channel. 662 */ 663 out_be64(&priv2->spu_chnlcntptr_RW, 29UL); 664 eieio(); 665 csa->spu_chnlcnt_RW[29] = in_be64(&priv2->spu_chnlcnt_RW); 666 for (i = 0; i < 4; i++) { 667 csa->spu_mailbox_data[i] = in_be64(&priv2->spu_chnldata_RW); 668 } 669 out_be64(&priv2->spu_chnlcnt_RW, 0UL); 670 eieio(); 671} 672 673static inline void save_mfc_cmd(struct spu_state *csa, struct spu *spu) 674{ 675 struct spu_priv2 __iomem *priv2 = spu->priv2; 676 677 /* Save, Step 44: 678 * Save MFC_CMD Channel. 679 */ 680 out_be64(&priv2->spu_chnlcntptr_RW, 21UL); 681 eieio(); 682 csa->spu_chnlcnt_RW[21] = in_be64(&priv2->spu_chnlcnt_RW); 683 eieio(); 684} 685 686static inline void reset_ch(struct spu_state *csa, struct spu *spu) 687{ 688 struct spu_priv2 __iomem *priv2 = spu->priv2; 689 u64 ch_indices[4] = { 21UL, 23UL, 28UL, 30UL }; 690 u64 ch_counts[4] = { 16UL, 1UL, 1UL, 1UL }; 691 u64 idx; 692 int i; 693 694 /* Save, Step 45: 695 * Reset the following CH: [21, 23, 28, 30] 696 */ 697 for (i = 0; i < 4; i++) { 698 idx = ch_indices[i]; 699 out_be64(&priv2->spu_chnlcntptr_RW, idx); 700 eieio(); 701 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]); 702 eieio(); 703 } 704} 705 706static inline void resume_mfc_queue(struct spu_state *csa, struct spu *spu) 707{ 708 struct spu_priv2 __iomem *priv2 = spu->priv2; 709 710 /* Save, Step 46: 711 * Restore, Step 25. 712 * Write MFC_CNTL[Sc]=0 (resume queue processing). 713 */ 714 out_be64(&priv2->mfc_control_RW, MFC_CNTL_RESUME_DMA_QUEUE); 715} 716 717static inline void setup_mfc_slbs(struct spu_state *csa, struct spu *spu, 718 unsigned int *code, int code_size) 719{ 720 /* Save, Step 47: 721 * Restore, Step 30. 722 * If MFC_SR1[R]=1, write 0 to SLB_Invalidate_All 723 * register, then initialize SLB_VSID and SLB_ESID 724 * to provide access to SPU context save code and 725 * LSCSA. 726 * 727 * This implementation places both the context 728 * switch code and LSCSA in kernel address space. 729 * 730 * Further this implementation assumes that the 731 * MFC_SR1[R]=1 (in other words, assume that 732 * translation is desired by OS environment). 733 */ 734 spu_invalidate_slbs(spu); 735 spu_setup_kernel_slbs(spu, csa->lscsa, code, code_size); 736} 737 738static inline void set_switch_active(struct spu_state *csa, struct spu *spu) 739{ 740 /* Save, Step 48: 741 * Restore, Step 23. 742 * Change the software context switch pending flag 743 * to context switch active. This implementation does 744 * not uses a switch active flag. 745 * 746 * Now that we have saved the mfc in the csa, we can add in the 747 * restart command if an exception occurred. 748 */ 749 if (test_bit(SPU_CONTEXT_FAULT_PENDING, &spu->flags)) 750 csa->priv2.mfc_control_RW |= MFC_CNTL_RESTART_DMA_COMMAND; 751 clear_bit(SPU_CONTEXT_SWITCH_PENDING, &spu->flags); 752 mb(); 753} 754 755static inline void enable_interrupts(struct spu_state *csa, struct spu *spu) 756{ 757 unsigned long class1_mask = CLASS1_ENABLE_SEGMENT_FAULT_INTR | 758 CLASS1_ENABLE_STORAGE_FAULT_INTR; 759 760 /* Save, Step 49: 761 * Restore, Step 22: 762 * Reset and then enable interrupts, as 763 * needed by OS. 764 * 765 * This implementation enables only class1 766 * (translation) interrupts. 767 */ 768 spin_lock_irq(&spu->register_lock); 769 spu_int_stat_clear(spu, 0, CLASS0_INTR_MASK); 770 spu_int_stat_clear(spu, 1, CLASS1_INTR_MASK); 771 spu_int_stat_clear(spu, 2, CLASS2_INTR_MASK); 772 spu_int_mask_set(spu, 0, 0ul); 773 spu_int_mask_set(spu, 1, class1_mask); 774 spu_int_mask_set(spu, 2, 0ul); 775 spin_unlock_irq(&spu->register_lock); 776} 777 778static inline int send_mfc_dma(struct spu *spu, unsigned long ea, 779 unsigned int ls_offset, unsigned int size, 780 unsigned int tag, unsigned int rclass, 781 unsigned int cmd) 782{ 783 struct spu_problem __iomem *prob = spu->problem; 784 union mfc_tag_size_class_cmd command; 785 unsigned int transfer_size; 786 volatile unsigned int status = 0x0; 787 788 while (size > 0) { 789 transfer_size = 790 (size > MFC_MAX_DMA_SIZE) ? MFC_MAX_DMA_SIZE : size; 791 command.u.mfc_size = transfer_size; 792 command.u.mfc_tag = tag; 793 command.u.mfc_rclassid = rclass; 794 command.u.mfc_cmd = cmd; 795 do { 796 out_be32(&prob->mfc_lsa_W, ls_offset); 797 out_be64(&prob->mfc_ea_W, ea); 798 out_be64(&prob->mfc_union_W.all64, command.all64); 799 status = 800 in_be32(&prob->mfc_union_W.by32.mfc_class_cmd32); 801 if (unlikely(status & 0x2)) { 802 cpu_relax(); 803 } 804 } while (status & 0x3); 805 size -= transfer_size; 806 ea += transfer_size; 807 ls_offset += transfer_size; 808 } 809 return 0; 810} 811 812static inline void save_ls_16kb(struct spu_state *csa, struct spu *spu) 813{ 814 unsigned long addr = (unsigned long)&csa->lscsa->ls[0]; 815 unsigned int ls_offset = 0x0; 816 unsigned int size = 16384; 817 unsigned int tag = 0; 818 unsigned int rclass = 0; 819 unsigned int cmd = MFC_PUT_CMD; 820 821 /* Save, Step 50: 822 * Issue a DMA command to copy the first 16K bytes 823 * of local storage to the CSA. 824 */ 825 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 826} 827 828static inline void set_spu_npc(struct spu_state *csa, struct spu *spu) 829{ 830 struct spu_problem __iomem *prob = spu->problem; 831 832 /* Save, Step 51: 833 * Restore, Step 31. 834 * Write SPU_NPC[IE]=0 and SPU_NPC[LSA] to entry 835 * point address of context save code in local 836 * storage. 837 * 838 * This implementation uses SPU-side save/restore 839 * programs with entry points at LSA of 0. 840 */ 841 out_be32(&prob->spu_npc_RW, 0); 842 eieio(); 843} 844 845static inline void set_signot1(struct spu_state *csa, struct spu *spu) 846{ 847 struct spu_problem __iomem *prob = spu->problem; 848 union { 849 u64 ull; 850 u32 ui[2]; 851 } addr64; 852 853 /* Save, Step 52: 854 * Restore, Step 32: 855 * Write SPU_Sig_Notify_1 register with upper 32-bits 856 * of the CSA.LSCSA effective address. 857 */ 858 addr64.ull = (u64) csa->lscsa; 859 out_be32(&prob->signal_notify1, addr64.ui[0]); 860 eieio(); 861} 862 863static inline void set_signot2(struct spu_state *csa, struct spu *spu) 864{ 865 struct spu_problem __iomem *prob = spu->problem; 866 union { 867 u64 ull; 868 u32 ui[2]; 869 } addr64; 870 871 /* Save, Step 53: 872 * Restore, Step 33: 873 * Write SPU_Sig_Notify_2 register with lower 32-bits 874 * of the CSA.LSCSA effective address. 875 */ 876 addr64.ull = (u64) csa->lscsa; 877 out_be32(&prob->signal_notify2, addr64.ui[1]); 878 eieio(); 879} 880 881static inline void send_save_code(struct spu_state *csa, struct spu *spu) 882{ 883 unsigned long addr = (unsigned long)&spu_save_code[0]; 884 unsigned int ls_offset = 0x0; 885 unsigned int size = sizeof(spu_save_code); 886 unsigned int tag = 0; 887 unsigned int rclass = 0; 888 unsigned int cmd = MFC_GETFS_CMD; 889 890 /* Save, Step 54: 891 * Issue a DMA command to copy context save code 892 * to local storage and start SPU. 893 */ 894 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 895} 896 897static inline void set_ppu_querymask(struct spu_state *csa, struct spu *spu) 898{ 899 struct spu_problem __iomem *prob = spu->problem; 900 901 /* Save, Step 55: 902 * Restore, Step 38. 903 * Write PPU_QueryMask=1 (enable Tag Group 0) 904 * and issue eieio instruction. 905 */ 906 out_be32(&prob->dma_querymask_RW, MFC_TAGID_TO_TAGMASK(0)); 907 eieio(); 908} 909 910static inline void wait_tag_complete(struct spu_state *csa, struct spu *spu) 911{ 912 struct spu_problem __iomem *prob = spu->problem; 913 u32 mask = MFC_TAGID_TO_TAGMASK(0); 914 unsigned long flags; 915 916 /* Save, Step 56: 917 * Restore, Step 39. 918 * Restore, Step 39. 919 * Restore, Step 46. 920 * Poll PPU_TagStatus[gn] until 01 (Tag group 0 complete) 921 * or write PPU_QueryType[TS]=01 and wait for Tag Group 922 * Complete Interrupt. Write INT_Stat_Class0 or 923 * INT_Stat_Class2 with value of 'handled'. 924 */ 925 POLL_WHILE_FALSE(in_be32(&prob->dma_tagstatus_R) & mask); 926 927 local_irq_save(flags); 928 spu_int_stat_clear(spu, 0, CLASS0_INTR_MASK); 929 spu_int_stat_clear(spu, 2, CLASS2_INTR_MASK); 930 local_irq_restore(flags); 931} 932 933static inline void wait_spu_stopped(struct spu_state *csa, struct spu *spu) 934{ 935 struct spu_problem __iomem *prob = spu->problem; 936 unsigned long flags; 937 938 /* Save, Step 57: 939 * Restore, Step 40. 940 * Poll until SPU_Status[R]=0 or wait for SPU Class 0 941 * or SPU Class 2 interrupt. Write INT_Stat_class0 942 * or INT_Stat_class2 with value of handled. 943 */ 944 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING); 945 946 local_irq_save(flags); 947 spu_int_stat_clear(spu, 0, CLASS0_INTR_MASK); 948 spu_int_stat_clear(spu, 2, CLASS2_INTR_MASK); 949 local_irq_restore(flags); 950} 951 952static inline int check_save_status(struct spu_state *csa, struct spu *spu) 953{ 954 struct spu_problem __iomem *prob = spu->problem; 955 u32 complete; 956 957 /* Save, Step 54: 958 * If SPU_Status[P]=1 and SPU_Status[SC] = "success", 959 * context save succeeded, otherwise context save 960 * failed. 961 */ 962 complete = ((SPU_SAVE_COMPLETE << SPU_STOP_STATUS_SHIFT) | 963 SPU_STATUS_STOPPED_BY_STOP); 964 return (in_be32(&prob->spu_status_R) != complete) ? 1 : 0; 965} 966 967static inline void terminate_spu_app(struct spu_state *csa, struct spu *spu) 968{ 969 /* Restore, Step 4: 970 * If required, notify the "using application" that 971 * the SPU task has been terminated. TBD. 972 */ 973} 974 975static inline void suspend_mfc_and_halt_decr(struct spu_state *csa, 976 struct spu *spu) 977{ 978 struct spu_priv2 __iomem *priv2 = spu->priv2; 979 980 /* Restore, Step 7: 981 * Write MFC_Cntl[Dh,Sc,Sm]='1','1','0' to suspend 982 * the queue and halt the decrementer. 983 */ 984 out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE | 985 MFC_CNTL_DECREMENTER_HALTED); 986 eieio(); 987} 988 989static inline void wait_suspend_mfc_complete(struct spu_state *csa, 990 struct spu *spu) 991{ 992 struct spu_priv2 __iomem *priv2 = spu->priv2; 993 994 /* Restore, Step 8: 995 * Restore, Step 47. 996 * Poll MFC_CNTL[Ss] until 11 is returned. 997 */ 998 POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) & 999 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) == 1000 MFC_CNTL_SUSPEND_COMPLETE); 1001} 1002 1003static inline int suspend_spe(struct spu_state *csa, struct spu *spu) 1004{ 1005 struct spu_problem __iomem *prob = spu->problem; 1006 1007 /* Restore, Step 9: 1008 * If SPU_Status[R]=1, stop SPU execution 1009 * and wait for stop to complete. 1010 * 1011 * Returns 1 if SPU_Status[R]=1 on entry. 1012 * 0 otherwise 1013 */ 1014 if (in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) { 1015 if (in_be32(&prob->spu_status_R) & 1016 SPU_STATUS_ISOLATED_EXIT_STATUS) { 1017 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1018 SPU_STATUS_RUNNING); 1019 } 1020 if ((in_be32(&prob->spu_status_R) & 1021 SPU_STATUS_ISOLATED_LOAD_STATUS) 1022 || (in_be32(&prob->spu_status_R) & 1023 SPU_STATUS_ISOLATED_STATE)) { 1024 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 1025 eieio(); 1026 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1027 SPU_STATUS_RUNNING); 1028 out_be32(&prob->spu_runcntl_RW, 0x2); 1029 eieio(); 1030 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1031 SPU_STATUS_RUNNING); 1032 } 1033 if (in_be32(&prob->spu_status_R) & 1034 SPU_STATUS_WAITING_FOR_CHANNEL) { 1035 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 1036 eieio(); 1037 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1038 SPU_STATUS_RUNNING); 1039 } 1040 return 1; 1041 } 1042 return 0; 1043} 1044 1045static inline void clear_spu_status(struct spu_state *csa, struct spu *spu) 1046{ 1047 struct spu_problem __iomem *prob = spu->problem; 1048 1049 /* Restore, Step 10: 1050 * If SPU_Status[R]=0 and SPU_Status[E,L,IS]=1, 1051 * release SPU from isolate state. 1052 */ 1053 if (!(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING)) { 1054 if (in_be32(&prob->spu_status_R) & 1055 SPU_STATUS_ISOLATED_EXIT_STATUS) { 1056 spu_mfc_sr1_set(spu, 1057 MFC_STATE1_MASTER_RUN_CONTROL_MASK); 1058 eieio(); 1059 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1060 eieio(); 1061 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1062 SPU_STATUS_RUNNING); 1063 } 1064 if ((in_be32(&prob->spu_status_R) & 1065 SPU_STATUS_ISOLATED_LOAD_STATUS) 1066 || (in_be32(&prob->spu_status_R) & 1067 SPU_STATUS_ISOLATED_STATE)) { 1068 spu_mfc_sr1_set(spu, 1069 MFC_STATE1_MASTER_RUN_CONTROL_MASK); 1070 eieio(); 1071 out_be32(&prob->spu_runcntl_RW, 0x2); 1072 eieio(); 1073 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1074 SPU_STATUS_RUNNING); 1075 } 1076 } 1077} 1078 1079static inline void reset_ch_part1(struct spu_state *csa, struct spu *spu) 1080{ 1081 struct spu_priv2 __iomem *priv2 = spu->priv2; 1082 u64 ch_indices[] = { 0UL, 3UL, 4UL, 24UL, 25UL, 27UL }; 1083 u64 idx; 1084 int i; 1085 1086 /* Restore, Step 20: 1087 */ 1088 1089 /* Reset CH 1 */ 1090 out_be64(&priv2->spu_chnlcntptr_RW, 1); 1091 out_be64(&priv2->spu_chnldata_RW, 0UL); 1092 1093 /* Reset the following CH: [0,3,4,24,25,27] */ 1094 for (i = 0; i < ARRAY_SIZE(ch_indices); i++) { 1095 idx = ch_indices[i]; 1096 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1097 eieio(); 1098 out_be64(&priv2->spu_chnldata_RW, 0UL); 1099 out_be64(&priv2->spu_chnlcnt_RW, 0UL); 1100 eieio(); 1101 } 1102} 1103 1104static inline void reset_ch_part2(struct spu_state *csa, struct spu *spu) 1105{ 1106 struct spu_priv2 __iomem *priv2 = spu->priv2; 1107 u64 ch_indices[5] = { 21UL, 23UL, 28UL, 29UL, 30UL }; 1108 u64 ch_counts[5] = { 16UL, 1UL, 1UL, 0UL, 1UL }; 1109 u64 idx; 1110 int i; 1111 1112 /* Restore, Step 21: 1113 * Reset the following CH: [21, 23, 28, 29, 30] 1114 */ 1115 for (i = 0; i < 5; i++) { 1116 idx = ch_indices[i]; 1117 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1118 eieio(); 1119 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]); 1120 eieio(); 1121 } 1122} 1123 1124static inline void setup_spu_status_part1(struct spu_state *csa, 1125 struct spu *spu) 1126{ 1127 u32 status_P = SPU_STATUS_STOPPED_BY_STOP; 1128 u32 status_I = SPU_STATUS_INVALID_INSTR; 1129 u32 status_H = SPU_STATUS_STOPPED_BY_HALT; 1130 u32 status_S = SPU_STATUS_SINGLE_STEP; 1131 u32 status_S_I = SPU_STATUS_SINGLE_STEP | SPU_STATUS_INVALID_INSTR; 1132 u32 status_S_P = SPU_STATUS_SINGLE_STEP | SPU_STATUS_STOPPED_BY_STOP; 1133 u32 status_P_H = SPU_STATUS_STOPPED_BY_HALT |SPU_STATUS_STOPPED_BY_STOP; 1134 u32 status_P_I = SPU_STATUS_STOPPED_BY_STOP |SPU_STATUS_INVALID_INSTR; 1135 u32 status_code; 1136 1137 /* Restore, Step 27: 1138 * If the CSA.SPU_Status[I,S,H,P]=1 then add the correct 1139 * instruction sequence to the end of the SPU based restore 1140 * code (after the "context restored" stop and signal) to 1141 * restore the correct SPU status. 1142 * 1143 * NOTE: Rather than modifying the SPU executable, we 1144 * instead add a new 'stopped_status' field to the 1145 * LSCSA. The SPU-side restore reads this field and 1146 * takes the appropriate action when exiting. 1147 */ 1148 1149 status_code = 1150 (csa->prob.spu_status_R >> SPU_STOP_STATUS_SHIFT) & 0xFFFF; 1151 if ((csa->prob.spu_status_R & status_P_I) == status_P_I) { 1152 1153 /* SPU_Status[P,I]=1 - Illegal Instruction followed 1154 * by Stop and Signal instruction, followed by 'br -4'. 1155 * 1156 */ 1157 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P_I; 1158 csa->lscsa->stopped_status.slot[1] = status_code; 1159 1160 } else if ((csa->prob.spu_status_R & status_P_H) == status_P_H) { 1161 1162 /* SPU_Status[P,H]=1 - Halt Conditional, followed 1163 * by Stop and Signal instruction, followed by 1164 * 'br -4'. 1165 */ 1166 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P_H; 1167 csa->lscsa->stopped_status.slot[1] = status_code; 1168 1169 } else if ((csa->prob.spu_status_R & status_S_P) == status_S_P) { 1170 1171 /* SPU_Status[S,P]=1 - Stop and Signal instruction 1172 * followed by 'br -4'. 1173 */ 1174 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S_P; 1175 csa->lscsa->stopped_status.slot[1] = status_code; 1176 1177 } else if ((csa->prob.spu_status_R & status_S_I) == status_S_I) { 1178 1179 /* SPU_Status[S,I]=1 - Illegal instruction followed 1180 * by 'br -4'. 1181 */ 1182 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S_I; 1183 csa->lscsa->stopped_status.slot[1] = status_code; 1184 1185 } else if ((csa->prob.spu_status_R & status_P) == status_P) { 1186 1187 /* SPU_Status[P]=1 - Stop and Signal instruction 1188 * followed by 'br -4'. 1189 */ 1190 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P; 1191 csa->lscsa->stopped_status.slot[1] = status_code; 1192 1193 } else if ((csa->prob.spu_status_R & status_H) == status_H) { 1194 1195 /* SPU_Status[H]=1 - Halt Conditional, followed 1196 * by 'br -4'. 1197 */ 1198 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_H; 1199 1200 } else if ((csa->prob.spu_status_R & status_S) == status_S) { 1201 1202 /* SPU_Status[S]=1 - Two nop instructions. 1203 */ 1204 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S; 1205 1206 } else if ((csa->prob.spu_status_R & status_I) == status_I) { 1207 1208 /* SPU_Status[I]=1 - Illegal instruction followed 1209 * by 'br -4'. 1210 */ 1211 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_I; 1212 1213 } 1214} 1215 1216static inline void setup_spu_status_part2(struct spu_state *csa, 1217 struct spu *spu) 1218{ 1219 u32 mask; 1220 1221 /* Restore, Step 28: 1222 * If the CSA.SPU_Status[I,S,H,P,R]=0 then 1223 * add a 'br *' instruction to the end of 1224 * the SPU based restore code. 1225 * 1226 * NOTE: Rather than modifying the SPU executable, we 1227 * instead add a new 'stopped_status' field to the 1228 * LSCSA. The SPU-side restore reads this field and 1229 * takes the appropriate action when exiting. 1230 */ 1231 mask = SPU_STATUS_INVALID_INSTR | 1232 SPU_STATUS_SINGLE_STEP | 1233 SPU_STATUS_STOPPED_BY_HALT | 1234 SPU_STATUS_STOPPED_BY_STOP | SPU_STATUS_RUNNING; 1235 if (!(csa->prob.spu_status_R & mask)) { 1236 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_R; 1237 } 1238} 1239 1240static inline void restore_mfc_rag(struct spu_state *csa, struct spu *spu) 1241{ 1242 /* Restore, Step 29: 1243 * Restore RA_GROUP_ID register and the 1244 * RA_ENABLE reigster from the CSA. 1245 */ 1246 spu_resource_allocation_groupID_set(spu, 1247 csa->priv1.resource_allocation_groupID_RW); 1248 spu_resource_allocation_enable_set(spu, 1249 csa->priv1.resource_allocation_enable_RW); 1250} 1251 1252static inline void send_restore_code(struct spu_state *csa, struct spu *spu) 1253{ 1254 unsigned long addr = (unsigned long)&spu_restore_code[0]; 1255 unsigned int ls_offset = 0x0; 1256 unsigned int size = sizeof(spu_restore_code); 1257 unsigned int tag = 0; 1258 unsigned int rclass = 0; 1259 unsigned int cmd = MFC_GETFS_CMD; 1260 1261 /* Restore, Step 37: 1262 * Issue MFC DMA command to copy context 1263 * restore code to local storage. 1264 */ 1265 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 1266} 1267 1268static inline void setup_decr(struct spu_state *csa, struct spu *spu) 1269{ 1270 /* Restore, Step 34: 1271 * If CSA.MFC_CNTL[Ds]=1 (decrementer was 1272 * running) then adjust decrementer, set 1273 * decrementer running status in LSCSA, 1274 * and set decrementer "wrapped" status 1275 * in LSCSA. 1276 */ 1277 if (csa->priv2.mfc_control_RW & MFC_CNTL_DECREMENTER_RUNNING) { 1278 cycles_t resume_time = get_cycles(); 1279 cycles_t delta_time = resume_time - csa->suspend_time; 1280 1281 csa->lscsa->decr_status.slot[0] = SPU_DECR_STATUS_RUNNING; 1282 if (csa->lscsa->decr.slot[0] < delta_time) { 1283 csa->lscsa->decr_status.slot[0] |= 1284 SPU_DECR_STATUS_WRAPPED; 1285 } 1286 1287 csa->lscsa->decr.slot[0] -= delta_time; 1288 } else { 1289 csa->lscsa->decr_status.slot[0] = 0; 1290 } 1291} 1292 1293static inline void setup_ppu_mb(struct spu_state *csa, struct spu *spu) 1294{ 1295 /* Restore, Step 35: 1296 * Copy the CSA.PU_MB data into the LSCSA. 1297 */ 1298 csa->lscsa->ppu_mb.slot[0] = csa->prob.pu_mb_R; 1299} 1300 1301static inline void setup_ppuint_mb(struct spu_state *csa, struct spu *spu) 1302{ 1303 /* Restore, Step 36: 1304 * Copy the CSA.PUINT_MB data into the LSCSA. 1305 */ 1306 csa->lscsa->ppuint_mb.slot[0] = csa->priv2.puint_mb_R; 1307} 1308 1309static inline int check_restore_status(struct spu_state *csa, struct spu *spu) 1310{ 1311 struct spu_problem __iomem *prob = spu->problem; 1312 u32 complete; 1313 1314 /* Restore, Step 40: 1315 * If SPU_Status[P]=1 and SPU_Status[SC] = "success", 1316 * context restore succeeded, otherwise context restore 1317 * failed. 1318 */ 1319 complete = ((SPU_RESTORE_COMPLETE << SPU_STOP_STATUS_SHIFT) | 1320 SPU_STATUS_STOPPED_BY_STOP); 1321 return (in_be32(&prob->spu_status_R) != complete) ? 1 : 0; 1322} 1323 1324static inline void restore_spu_privcntl(struct spu_state *csa, struct spu *spu) 1325{ 1326 struct spu_priv2 __iomem *priv2 = spu->priv2; 1327 1328 /* Restore, Step 41: 1329 * Restore SPU_PrivCntl from the CSA. 1330 */ 1331 out_be64(&priv2->spu_privcntl_RW, csa->priv2.spu_privcntl_RW); 1332 eieio(); 1333} 1334 1335static inline void restore_status_part1(struct spu_state *csa, struct spu *spu) 1336{ 1337 struct spu_problem __iomem *prob = spu->problem; 1338 u32 mask; 1339 1340 /* Restore, Step 42: 1341 * If any CSA.SPU_Status[I,S,H,P]=1, then 1342 * restore the error or single step state. 1343 */ 1344 mask = SPU_STATUS_INVALID_INSTR | 1345 SPU_STATUS_SINGLE_STEP | 1346 SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP; 1347 if (csa->prob.spu_status_R & mask) { 1348 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1349 eieio(); 1350 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1351 SPU_STATUS_RUNNING); 1352 } 1353} 1354 1355static inline void restore_status_part2(struct spu_state *csa, struct spu *spu) 1356{ 1357 struct spu_problem __iomem *prob = spu->problem; 1358 u32 mask; 1359 1360 /* Restore, Step 43: 1361 * If all CSA.SPU_Status[I,S,H,P,R]=0 then write 1362 * SPU_RunCntl[R0R1]='01', wait for SPU_Status[R]=1, 1363 * then write '00' to SPU_RunCntl[R0R1] and wait 1364 * for SPU_Status[R]=0. 1365 */ 1366 mask = SPU_STATUS_INVALID_INSTR | 1367 SPU_STATUS_SINGLE_STEP | 1368 SPU_STATUS_STOPPED_BY_HALT | 1369 SPU_STATUS_STOPPED_BY_STOP | SPU_STATUS_RUNNING; 1370 if (!(csa->prob.spu_status_R & mask)) { 1371 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1372 eieio(); 1373 POLL_WHILE_FALSE(in_be32(&prob->spu_status_R) & 1374 SPU_STATUS_RUNNING); 1375 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 1376 eieio(); 1377 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1378 SPU_STATUS_RUNNING); 1379 } 1380} 1381 1382static inline void restore_ls_16kb(struct spu_state *csa, struct spu *spu) 1383{ 1384 unsigned long addr = (unsigned long)&csa->lscsa->ls[0]; 1385 unsigned int ls_offset = 0x0; 1386 unsigned int size = 16384; 1387 unsigned int tag = 0; 1388 unsigned int rclass = 0; 1389 unsigned int cmd = MFC_GET_CMD; 1390 1391 /* Restore, Step 44: 1392 * Issue a DMA command to restore the first 1393 * 16kb of local storage from CSA. 1394 */ 1395 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 1396} 1397 1398static inline void suspend_mfc(struct spu_state *csa, struct spu *spu) 1399{ 1400 struct spu_priv2 __iomem *priv2 = spu->priv2; 1401 1402 /* Restore, Step 47. 1403 * Write MFC_Cntl[Sc,Sm]='1','0' to suspend 1404 * the queue. 1405 */ 1406 out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE); 1407 eieio(); 1408} 1409 1410static inline void clear_interrupts(struct spu_state *csa, struct spu *spu) 1411{ 1412 /* Restore, Step 49: 1413 * Write INT_MASK_class0 with value of 0. 1414 * Write INT_MASK_class1 with value of 0. 1415 * Write INT_MASK_class2 with value of 0. 1416 * Write INT_STAT_class0 with value of -1. 1417 * Write INT_STAT_class1 with value of -1. 1418 * Write INT_STAT_class2 with value of -1. 1419 */ 1420 spin_lock_irq(&spu->register_lock); 1421 spu_int_mask_set(spu, 0, 0ul); 1422 spu_int_mask_set(spu, 1, 0ul); 1423 spu_int_mask_set(spu, 2, 0ul); 1424 spu_int_stat_clear(spu, 0, CLASS0_INTR_MASK); 1425 spu_int_stat_clear(spu, 1, CLASS1_INTR_MASK); 1426 spu_int_stat_clear(spu, 2, CLASS2_INTR_MASK); 1427 spin_unlock_irq(&spu->register_lock); 1428} 1429 1430static inline void restore_mfc_queues(struct spu_state *csa, struct spu *spu) 1431{ 1432 struct spu_priv2 __iomem *priv2 = spu->priv2; 1433 int i; 1434 1435 /* Restore, Step 50: 1436 * If MFC_Cntl[Se]!=0 then restore 1437 * MFC command queues. 1438 */ 1439 if ((csa->priv2.mfc_control_RW & MFC_CNTL_DMA_QUEUES_EMPTY_MASK) == 0) { 1440 for (i = 0; i < 8; i++) { 1441 out_be64(&priv2->puq[i].mfc_cq_data0_RW, 1442 csa->priv2.puq[i].mfc_cq_data0_RW); 1443 out_be64(&priv2->puq[i].mfc_cq_data1_RW, 1444 csa->priv2.puq[i].mfc_cq_data1_RW); 1445 out_be64(&priv2->puq[i].mfc_cq_data2_RW, 1446 csa->priv2.puq[i].mfc_cq_data2_RW); 1447 out_be64(&priv2->puq[i].mfc_cq_data3_RW, 1448 csa->priv2.puq[i].mfc_cq_data3_RW); 1449 } 1450 for (i = 0; i < 16; i++) { 1451 out_be64(&priv2->spuq[i].mfc_cq_data0_RW, 1452 csa->priv2.spuq[i].mfc_cq_data0_RW); 1453 out_be64(&priv2->spuq[i].mfc_cq_data1_RW, 1454 csa->priv2.spuq[i].mfc_cq_data1_RW); 1455 out_be64(&priv2->spuq[i].mfc_cq_data2_RW, 1456 csa->priv2.spuq[i].mfc_cq_data2_RW); 1457 out_be64(&priv2->spuq[i].mfc_cq_data3_RW, 1458 csa->priv2.spuq[i].mfc_cq_data3_RW); 1459 } 1460 } 1461 eieio(); 1462} 1463 1464static inline void restore_ppu_querymask(struct spu_state *csa, struct spu *spu) 1465{ 1466 struct spu_problem __iomem *prob = spu->problem; 1467 1468 /* Restore, Step 51: 1469 * Restore the PPU_QueryMask register from CSA. 1470 */ 1471 out_be32(&prob->dma_querymask_RW, csa->prob.dma_querymask_RW); 1472 eieio(); 1473} 1474 1475static inline void restore_ppu_querytype(struct spu_state *csa, struct spu *spu) 1476{ 1477 struct spu_problem __iomem *prob = spu->problem; 1478 1479 /* Restore, Step 52: 1480 * Restore the PPU_QueryType register from CSA. 1481 */ 1482 out_be32(&prob->dma_querytype_RW, csa->prob.dma_querytype_RW); 1483 eieio(); 1484} 1485 1486static inline void restore_mfc_csr_tsq(struct spu_state *csa, struct spu *spu) 1487{ 1488 struct spu_priv2 __iomem *priv2 = spu->priv2; 1489 1490 /* Restore, Step 53: 1491 * Restore the MFC_CSR_TSQ register from CSA. 1492 */ 1493 out_be64(&priv2->spu_tag_status_query_RW, 1494 csa->priv2.spu_tag_status_query_RW); 1495 eieio(); 1496} 1497 1498static inline void restore_mfc_csr_cmd(struct spu_state *csa, struct spu *spu) 1499{ 1500 struct spu_priv2 __iomem *priv2 = spu->priv2; 1501 1502 /* Restore, Step 54: 1503 * Restore the MFC_CSR_CMD1 and MFC_CSR_CMD2 1504 * registers from CSA. 1505 */ 1506 out_be64(&priv2->spu_cmd_buf1_RW, csa->priv2.spu_cmd_buf1_RW); 1507 out_be64(&priv2->spu_cmd_buf2_RW, csa->priv2.spu_cmd_buf2_RW); 1508 eieio(); 1509} 1510 1511static inline void restore_mfc_csr_ato(struct spu_state *csa, struct spu *spu) 1512{ 1513 struct spu_priv2 __iomem *priv2 = spu->priv2; 1514 1515 /* Restore, Step 55: 1516 * Restore the MFC_CSR_ATO register from CSA. 1517 */ 1518 out_be64(&priv2->spu_atomic_status_RW, csa->priv2.spu_atomic_status_RW); 1519} 1520 1521static inline void restore_mfc_tclass_id(struct spu_state *csa, struct spu *spu) 1522{ 1523 /* Restore, Step 56: 1524 * Restore the MFC_TCLASS_ID register from CSA. 1525 */ 1526 spu_mfc_tclass_id_set(spu, csa->priv1.mfc_tclass_id_RW); 1527 eieio(); 1528} 1529 1530static inline void set_llr_event(struct spu_state *csa, struct spu *spu) 1531{ 1532 u64 ch0_cnt, ch0_data; 1533 u64 ch1_data; 1534 1535 /* Restore, Step 57: 1536 * Set the Lock Line Reservation Lost Event by: 1537 * 1. OR CSA.SPU_Event_Status with bit 21 (Lr) set to 1. 1538 * 2. If CSA.SPU_Channel_0_Count=0 and 1539 * CSA.SPU_Wr_Event_Mask[Lr]=1 and 1540 * CSA.SPU_Event_Status[Lr]=0 then set 1541 * CSA.SPU_Event_Status_Count=1. 1542 */ 1543 ch0_cnt = csa->spu_chnlcnt_RW[0]; 1544 ch0_data = csa->spu_chnldata_RW[0]; 1545 ch1_data = csa->spu_chnldata_RW[1]; 1546 csa->spu_chnldata_RW[0] |= MFC_LLR_LOST_EVENT; 1547 if ((ch0_cnt == 0) && !(ch0_data & MFC_LLR_LOST_EVENT) && 1548 (ch1_data & MFC_LLR_LOST_EVENT)) { 1549 csa->spu_chnlcnt_RW[0] = 1; 1550 } 1551} 1552 1553static inline void restore_decr_wrapped(struct spu_state *csa, struct spu *spu) 1554{ 1555 /* Restore, Step 58: 1556 * If the status of the CSA software decrementer 1557 * "wrapped" flag is set, OR in a '1' to 1558 * CSA.SPU_Event_Status[Tm]. 1559 */ 1560 if (!(csa->lscsa->decr_status.slot[0] & SPU_DECR_STATUS_WRAPPED)) 1561 return; 1562 1563 if ((csa->spu_chnlcnt_RW[0] == 0) && 1564 (csa->spu_chnldata_RW[1] & 0x20) && 1565 !(csa->spu_chnldata_RW[0] & 0x20)) 1566 csa->spu_chnlcnt_RW[0] = 1; 1567 1568 csa->spu_chnldata_RW[0] |= 0x20; 1569} 1570 1571static inline void restore_ch_part1(struct spu_state *csa, struct spu *spu) 1572{ 1573 struct spu_priv2 __iomem *priv2 = spu->priv2; 1574 u64 idx, ch_indices[] = { 0UL, 3UL, 4UL, 24UL, 25UL, 27UL }; 1575 int i; 1576 1577 /* Restore, Step 59: 1578 * Restore the following CH: [0,3,4,24,25,27] 1579 */ 1580 for (i = 0; i < ARRAY_SIZE(ch_indices); i++) { 1581 idx = ch_indices[i]; 1582 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1583 eieio(); 1584 out_be64(&priv2->spu_chnldata_RW, csa->spu_chnldata_RW[idx]); 1585 out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[idx]); 1586 eieio(); 1587 } 1588} 1589 1590static inline void restore_ch_part2(struct spu_state *csa, struct spu *spu) 1591{ 1592 struct spu_priv2 __iomem *priv2 = spu->priv2; 1593 u64 ch_indices[3] = { 9UL, 21UL, 23UL }; 1594 u64 ch_counts[3] = { 1UL, 16UL, 1UL }; 1595 u64 idx; 1596 int i; 1597 1598 /* Restore, Step 60: 1599 * Restore the following CH: [9,21,23]. 1600 */ 1601 ch_counts[0] = 1UL; 1602 ch_counts[1] = csa->spu_chnlcnt_RW[21]; 1603 ch_counts[2] = 1UL; 1604 for (i = 0; i < 3; i++) { 1605 idx = ch_indices[i]; 1606 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1607 eieio(); 1608 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]); 1609 eieio(); 1610 } 1611} 1612 1613static inline void restore_spu_lslr(struct spu_state *csa, struct spu *spu) 1614{ 1615 struct spu_priv2 __iomem *priv2 = spu->priv2; 1616 1617 /* Restore, Step 61: 1618 * Restore the SPU_LSLR register from CSA. 1619 */ 1620 out_be64(&priv2->spu_lslr_RW, csa->priv2.spu_lslr_RW); 1621 eieio(); 1622} 1623 1624static inline void restore_spu_cfg(struct spu_state *csa, struct spu *spu) 1625{ 1626 struct spu_priv2 __iomem *priv2 = spu->priv2; 1627 1628 /* Restore, Step 62: 1629 * Restore the SPU_Cfg register from CSA. 1630 */ 1631 out_be64(&priv2->spu_cfg_RW, csa->priv2.spu_cfg_RW); 1632 eieio(); 1633} 1634 1635static inline void restore_pm_trace(struct spu_state *csa, struct spu *spu) 1636{ 1637 /* Restore, Step 63: 1638 * Restore PM_Trace_Tag_Wait_Mask from CSA. 1639 * Not performed by this implementation. 1640 */ 1641} 1642 1643static inline void restore_spu_npc(struct spu_state *csa, struct spu *spu) 1644{ 1645 struct spu_problem __iomem *prob = spu->problem; 1646 1647 /* Restore, Step 64: 1648 * Restore SPU_NPC from CSA. 1649 */ 1650 out_be32(&prob->spu_npc_RW, csa->prob.spu_npc_RW); 1651 eieio(); 1652} 1653 1654static inline void restore_spu_mb(struct spu_state *csa, struct spu *spu) 1655{ 1656 struct spu_priv2 __iomem *priv2 = spu->priv2; 1657 int i; 1658 1659 /* Restore, Step 65: 1660 * Restore MFC_RdSPU_MB from CSA. 1661 */ 1662 out_be64(&priv2->spu_chnlcntptr_RW, 29UL); 1663 eieio(); 1664 out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[29]); 1665 for (i = 0; i < 4; i++) { 1666 out_be64(&priv2->spu_chnldata_RW, csa->spu_mailbox_data[i]); 1667 } 1668 eieio(); 1669} 1670 1671static inline void check_ppu_mb_stat(struct spu_state *csa, struct spu *spu) 1672{ 1673 struct spu_problem __iomem *prob = spu->problem; 1674 u32 dummy = 0; 1675 1676 /* Restore, Step 66: 1677 * If CSA.MB_Stat[P]=0 (mailbox empty) then 1678 * read from the PPU_MB register. 1679 */ 1680 if ((csa->prob.mb_stat_R & 0xFF) == 0) { 1681 dummy = in_be32(&prob->pu_mb_R); 1682 eieio(); 1683 } 1684} 1685 1686static inline void check_ppuint_mb_stat(struct spu_state *csa, struct spu *spu) 1687{ 1688 struct spu_priv2 __iomem *priv2 = spu->priv2; 1689 u64 dummy = 0UL; 1690 1691 /* Restore, Step 66: 1692 * If CSA.MB_Stat[I]=0 (mailbox empty) then 1693 * read from the PPUINT_MB register. 1694 */ 1695 if ((csa->prob.mb_stat_R & 0xFF0000) == 0) { 1696 dummy = in_be64(&priv2->puint_mb_R); 1697 eieio(); 1698 spu_int_stat_clear(spu, 2, CLASS2_ENABLE_MAILBOX_INTR); 1699 eieio(); 1700 } 1701} 1702 1703static inline void restore_mfc_sr1(struct spu_state *csa, struct spu *spu) 1704{ 1705 /* Restore, Step 69: 1706 * Restore the MFC_SR1 register from CSA. 1707 */ 1708 spu_mfc_sr1_set(spu, csa->priv1.mfc_sr1_RW); 1709 eieio(); 1710} 1711 1712static inline void set_int_route(struct spu_state *csa, struct spu *spu) 1713{ 1714 struct spu_context *ctx = spu->ctx; 1715 1716 spu_cpu_affinity_set(spu, ctx->last_ran); 1717} 1718 1719static inline void restore_other_spu_access(struct spu_state *csa, 1720 struct spu *spu) 1721{ 1722 /* Restore, Step 70: 1723 * Restore other SPU mappings to this SPU. TBD. 1724 */ 1725} 1726 1727static inline void restore_spu_runcntl(struct spu_state *csa, struct spu *spu) 1728{ 1729 struct spu_problem __iomem *prob = spu->problem; 1730 1731 /* Restore, Step 71: 1732 * If CSA.SPU_Status[R]=1 then write 1733 * SPU_RunCntl[R0R1]='01'. 1734 */ 1735 if (csa->prob.spu_status_R & SPU_STATUS_RUNNING) { 1736 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1737 eieio(); 1738 } 1739} 1740 1741static inline void restore_mfc_cntl(struct spu_state *csa, struct spu *spu) 1742{ 1743 struct spu_priv2 __iomem *priv2 = spu->priv2; 1744 1745 /* Restore, Step 72: 1746 * Restore the MFC_CNTL register for the CSA. 1747 */ 1748 out_be64(&priv2->mfc_control_RW, csa->priv2.mfc_control_RW); 1749 eieio(); 1750 1751 /* 1752 * The queue is put back into the same state that was evident prior to 1753 * the context switch. The suspend flag is added to the saved state in 1754 * the csa, if the operational state was suspending or suspended. In 1755 * this case, the code that suspended the mfc is responsible for 1756 * continuing it. Note that SPE faults do not change the operational 1757 * state of the spu. 1758 */ 1759} 1760 1761static inline void enable_user_access(struct spu_state *csa, struct spu *spu) 1762{ 1763 /* Restore, Step 73: 1764 * Enable user-space access (if provided) to this 1765 * SPU by mapping the virtual pages assigned to 1766 * the SPU memory-mapped I/O (MMIO) for problem 1767 * state. TBD. 1768 */ 1769} 1770 1771static inline void reset_switch_active(struct spu_state *csa, struct spu *spu) 1772{ 1773 /* Restore, Step 74: 1774 * Reset the "context switch active" flag. 1775 * Not performed by this implementation. 1776 */ 1777} 1778 1779static inline void reenable_interrupts(struct spu_state *csa, struct spu *spu) 1780{ 1781 /* Restore, Step 75: 1782 * Re-enable SPU interrupts. 1783 */ 1784 spin_lock_irq(&spu->register_lock); 1785 spu_int_mask_set(spu, 0, csa->priv1.int_mask_class0_RW); 1786 spu_int_mask_set(spu, 1, csa->priv1.int_mask_class1_RW); 1787 spu_int_mask_set(spu, 2, csa->priv1.int_mask_class2_RW); 1788 spin_unlock_irq(&spu->register_lock); 1789} 1790 1791static int quiece_spu(struct spu_state *prev, struct spu *spu) 1792{ 1793 /* 1794 * Combined steps 2-18 of SPU context save sequence, which 1795 * quiesce the SPU state (disable SPU execution, MFC command 1796 * queues, decrementer, SPU interrupts, etc.). 1797 * 1798 * Returns 0 on success. 1799 * 2 if failed step 2. 1800 * 6 if failed step 6. 1801 */ 1802 1803 if (check_spu_isolate(prev, spu)) { /* Step 2. */ 1804 return 2; 1805 } 1806 disable_interrupts(prev, spu); /* Step 3. */ 1807 set_watchdog_timer(prev, spu); /* Step 4. */ 1808 inhibit_user_access(prev, spu); /* Step 5. */ 1809 if (check_spu_isolate(prev, spu)) { /* Step 6. */ 1810 return 6; 1811 } 1812 set_switch_pending(prev, spu); /* Step 7. */ 1813 save_mfc_cntl(prev, spu); /* Step 8. */ 1814 save_spu_runcntl(prev, spu); /* Step 9. */ 1815 save_mfc_sr1(prev, spu); /* Step 10. */ 1816 save_spu_status(prev, spu); /* Step 11. */ 1817 save_mfc_stopped_status(prev, spu); /* Step 12. */ 1818 halt_mfc_decr(prev, spu); /* Step 13. */ 1819 save_timebase(prev, spu); /* Step 14. */ 1820 remove_other_spu_access(prev, spu); /* Step 15. */ 1821 do_mfc_mssync(prev, spu); /* Step 16. */ 1822 issue_mfc_tlbie(prev, spu); /* Step 17. */ 1823 handle_pending_interrupts(prev, spu); /* Step 18. */ 1824 1825 return 0; 1826} 1827 1828static void save_csa(struct spu_state *prev, struct spu *spu) 1829{ 1830 /* 1831 * Combine steps 19-44 of SPU context save sequence, which 1832 * save regions of the privileged & problem state areas. 1833 */ 1834 1835 save_mfc_queues(prev, spu); /* Step 19. */ 1836 save_ppu_querymask(prev, spu); /* Step 20. */ 1837 save_ppu_querytype(prev, spu); /* Step 21. */ 1838 save_ppu_tagstatus(prev, spu); /* NEW. */ 1839 save_mfc_csr_tsq(prev, spu); /* Step 22. */ 1840 save_mfc_csr_cmd(prev, spu); /* Step 23. */ 1841 save_mfc_csr_ato(prev, spu); /* Step 24. */ 1842 save_mfc_tclass_id(prev, spu); /* Step 25. */ 1843 set_mfc_tclass_id(prev, spu); /* Step 26. */ 1844 save_mfc_cmd(prev, spu); /* Step 26a - moved from 44. */ 1845 purge_mfc_queue(prev, spu); /* Step 27. */ 1846 wait_purge_complete(prev, spu); /* Step 28. */ 1847 setup_mfc_sr1(prev, spu); /* Step 30. */ 1848 save_spu_npc(prev, spu); /* Step 31. */ 1849 save_spu_privcntl(prev, spu); /* Step 32. */ 1850 reset_spu_privcntl(prev, spu); /* Step 33. */ 1851 save_spu_lslr(prev, spu); /* Step 34. */ 1852 reset_spu_lslr(prev, spu); /* Step 35. */ 1853 save_spu_cfg(prev, spu); /* Step 36. */ 1854 save_pm_trace(prev, spu); /* Step 37. */ 1855 save_mfc_rag(prev, spu); /* Step 38. */ 1856 save_ppu_mb_stat(prev, spu); /* Step 39. */ 1857 save_ppu_mb(prev, spu); /* Step 40. */ 1858 save_ppuint_mb(prev, spu); /* Step 41. */ 1859 save_ch_part1(prev, spu); /* Step 42. */ 1860 save_spu_mb(prev, spu); /* Step 43. */ 1861 reset_ch(prev, spu); /* Step 45. */ 1862} 1863 1864static void save_lscsa(struct spu_state *prev, struct spu *spu) 1865{ 1866 /* 1867 * Perform steps 46-57 of SPU context save sequence, 1868 * which save regions of the local store and register 1869 * file. 1870 */ 1871 1872 resume_mfc_queue(prev, spu); /* Step 46. */ 1873 /* Step 47. */ 1874 setup_mfc_slbs(prev, spu, spu_save_code, sizeof(spu_save_code)); 1875 set_switch_active(prev, spu); /* Step 48. */ 1876 enable_interrupts(prev, spu); /* Step 49. */ 1877 save_ls_16kb(prev, spu); /* Step 50. */ 1878 set_spu_npc(prev, spu); /* Step 51. */ 1879 set_signot1(prev, spu); /* Step 52. */ 1880 set_signot2(prev, spu); /* Step 53. */ 1881 send_save_code(prev, spu); /* Step 54. */ 1882 set_ppu_querymask(prev, spu); /* Step 55. */ 1883 wait_tag_complete(prev, spu); /* Step 56. */ 1884 wait_spu_stopped(prev, spu); /* Step 57. */ 1885} 1886 1887static void force_spu_isolate_exit(struct spu *spu) 1888{ 1889 struct spu_problem __iomem *prob = spu->problem; 1890 struct spu_priv2 __iomem *priv2 = spu->priv2; 1891 1892 /* Stop SPE execution and wait for completion. */ 1893 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 1894 iobarrier_rw(); 1895 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING); 1896 1897 /* Restart SPE master runcntl. */ 1898 spu_mfc_sr1_set(spu, MFC_STATE1_MASTER_RUN_CONTROL_MASK); 1899 iobarrier_w(); 1900 1901 /* Initiate isolate exit request and wait for completion. */ 1902 out_be64(&priv2->spu_privcntl_RW, 4LL); 1903 iobarrier_w(); 1904 out_be32(&prob->spu_runcntl_RW, 2); 1905 iobarrier_rw(); 1906 POLL_WHILE_FALSE((in_be32(&prob->spu_status_R) 1907 & SPU_STATUS_STOPPED_BY_STOP)); 1908 1909 /* Reset load request to normal. */ 1910 out_be64(&priv2->spu_privcntl_RW, SPU_PRIVCNT_LOAD_REQUEST_NORMAL); 1911 iobarrier_w(); 1912} 1913 1914/** 1915 * stop_spu_isolate 1916 * Check SPU run-control state and force isolated 1917 * exit function as necessary. 1918 */ 1919static void stop_spu_isolate(struct spu *spu) 1920{ 1921 struct spu_problem __iomem *prob = spu->problem; 1922 1923 if (in_be32(&prob->spu_status_R) & SPU_STATUS_ISOLATED_STATE) { 1924 /* The SPU is in isolated state; the only way 1925 * to get it out is to perform an isolated 1926 * exit (clean) operation. 1927 */ 1928 force_spu_isolate_exit(spu); 1929 } 1930} 1931 1932static void harvest(struct spu_state *prev, struct spu *spu) 1933{ 1934 /* 1935 * Perform steps 2-25 of SPU context restore sequence, 1936 * which resets an SPU either after a failed save, or 1937 * when using SPU for first time. 1938 */ 1939 1940 disable_interrupts(prev, spu); /* Step 2. */ 1941 inhibit_user_access(prev, spu); /* Step 3. */ 1942 terminate_spu_app(prev, spu); /* Step 4. */ 1943 set_switch_pending(prev, spu); /* Step 5. */ 1944 stop_spu_isolate(spu); /* NEW. */ 1945 remove_other_spu_access(prev, spu); /* Step 6. */ 1946 suspend_mfc_and_halt_decr(prev, spu); /* Step 7. */ 1947 wait_suspend_mfc_complete(prev, spu); /* Step 8. */ 1948 if (!suspend_spe(prev, spu)) /* Step 9. */ 1949 clear_spu_status(prev, spu); /* Step 10. */ 1950 do_mfc_mssync(prev, spu); /* Step 11. */ 1951 issue_mfc_tlbie(prev, spu); /* Step 12. */ 1952 handle_pending_interrupts(prev, spu); /* Step 13. */ 1953 purge_mfc_queue(prev, spu); /* Step 14. */ 1954 wait_purge_complete(prev, spu); /* Step 15. */ 1955 reset_spu_privcntl(prev, spu); /* Step 16. */ 1956 reset_spu_lslr(prev, spu); /* Step 17. */ 1957 setup_mfc_sr1(prev, spu); /* Step 18. */ 1958 spu_invalidate_slbs(spu); /* Step 19. */ 1959 reset_ch_part1(prev, spu); /* Step 20. */ 1960 reset_ch_part2(prev, spu); /* Step 21. */ 1961 enable_interrupts(prev, spu); /* Step 22. */ 1962 set_switch_active(prev, spu); /* Step 23. */ 1963 set_mfc_tclass_id(prev, spu); /* Step 24. */ 1964 resume_mfc_queue(prev, spu); /* Step 25. */ 1965} 1966 1967static void restore_lscsa(struct spu_state *next, struct spu *spu) 1968{ 1969 /* 1970 * Perform steps 26-40 of SPU context restore sequence, 1971 * which restores regions of the local store and register 1972 * file. 1973 */ 1974 1975 set_watchdog_timer(next, spu); /* Step 26. */ 1976 setup_spu_status_part1(next, spu); /* Step 27. */ 1977 setup_spu_status_part2(next, spu); /* Step 28. */ 1978 restore_mfc_rag(next, spu); /* Step 29. */ 1979 /* Step 30. */ 1980 setup_mfc_slbs(next, spu, spu_restore_code, sizeof(spu_restore_code)); 1981 set_spu_npc(next, spu); /* Step 31. */ 1982 set_signot1(next, spu); /* Step 32. */ 1983 set_signot2(next, spu); /* Step 33. */ 1984 setup_decr(next, spu); /* Step 34. */ 1985 setup_ppu_mb(next, spu); /* Step 35. */ 1986 setup_ppuint_mb(next, spu); /* Step 36. */ 1987 send_restore_code(next, spu); /* Step 37. */ 1988 set_ppu_querymask(next, spu); /* Step 38. */ 1989 wait_tag_complete(next, spu); /* Step 39. */ 1990 wait_spu_stopped(next, spu); /* Step 40. */ 1991} 1992 1993static void restore_csa(struct spu_state *next, struct spu *spu) 1994{ 1995 /* 1996 * Combine steps 41-76 of SPU context restore sequence, which 1997 * restore regions of the privileged & problem state areas. 1998 */ 1999 2000 restore_spu_privcntl(next, spu); /* Step 41. */ 2001 restore_status_part1(next, spu); /* Step 42. */ 2002 restore_status_part2(next, spu); /* Step 43. */ 2003 restore_ls_16kb(next, spu); /* Step 44. */ 2004 wait_tag_complete(next, spu); /* Step 45. */ 2005 suspend_mfc(next, spu); /* Step 46. */ 2006 wait_suspend_mfc_complete(next, spu); /* Step 47. */ 2007 issue_mfc_tlbie(next, spu); /* Step 48. */ 2008 clear_interrupts(next, spu); /* Step 49. */ 2009 restore_mfc_queues(next, spu); /* Step 50. */ 2010 restore_ppu_querymask(next, spu); /* Step 51. */ 2011 restore_ppu_querytype(next, spu); /* Step 52. */ 2012 restore_mfc_csr_tsq(next, spu); /* Step 53. */ 2013 restore_mfc_csr_cmd(next, spu); /* Step 54. */ 2014 restore_mfc_csr_ato(next, spu); /* Step 55. */ 2015 restore_mfc_tclass_id(next, spu); /* Step 56. */ 2016 set_llr_event(next, spu); /* Step 57. */ 2017 restore_decr_wrapped(next, spu); /* Step 58. */ 2018 restore_ch_part1(next, spu); /* Step 59. */ 2019 restore_ch_part2(next, spu); /* Step 60. */ 2020 restore_spu_lslr(next, spu); /* Step 61. */ 2021 restore_spu_cfg(next, spu); /* Step 62. */ 2022 restore_pm_trace(next, spu); /* Step 63. */ 2023 restore_spu_npc(next, spu); /* Step 64. */ 2024 restore_spu_mb(next, spu); /* Step 65. */ 2025 check_ppu_mb_stat(next, spu); /* Step 66. */ 2026 check_ppuint_mb_stat(next, spu); /* Step 67. */ 2027 spu_invalidate_slbs(spu); /* Modified Step 68. */ 2028 restore_mfc_sr1(next, spu); /* Step 69. */ 2029 set_int_route(next, spu); /* NEW */ 2030 restore_other_spu_access(next, spu); /* Step 70. */ 2031 restore_spu_runcntl(next, spu); /* Step 71. */ 2032 restore_mfc_cntl(next, spu); /* Step 72. */ 2033 enable_user_access(next, spu); /* Step 73. */ 2034 reset_switch_active(next, spu); /* Step 74. */ 2035 reenable_interrupts(next, spu); /* Step 75. */ 2036} 2037 2038static int __do_spu_save(struct spu_state *prev, struct spu *spu) 2039{ 2040 int rc; 2041 2042 /* 2043 * SPU context save can be broken into three phases: 2044 * 2045 * (a) quiesce [steps 2-16]. 2046 * (b) save of CSA, performed by PPE [steps 17-42] 2047 * (c) save of LSCSA, mostly performed by SPU [steps 43-52]. 2048 * 2049 * Returns 0 on success. 2050 * 2,6 if failed to quiece SPU 2051 * 53 if SPU-side of save failed. 2052 */ 2053 2054 rc = quiece_spu(prev, spu); /* Steps 2-16. */ 2055 switch (rc) { 2056 default: 2057 case 2: 2058 case 6: 2059 harvest(prev, spu); 2060 return rc; 2061 break; 2062 case 0: 2063 break; 2064 } 2065 save_csa(prev, spu); /* Steps 17-43. */ 2066 save_lscsa(prev, spu); /* Steps 44-53. */ 2067 return check_save_status(prev, spu); /* Step 54. */ 2068} 2069 2070static int __do_spu_restore(struct spu_state *next, struct spu *spu) 2071{ 2072 int rc; 2073 2074 /* 2075 * SPU context restore can be broken into three phases: 2076 * 2077 * (a) harvest (or reset) SPU [steps 2-24]. 2078 * (b) restore LSCSA [steps 25-40], mostly performed by SPU. 2079 * (c) restore CSA [steps 41-76], performed by PPE. 2080 * 2081 * The 'harvest' step is not performed here, but rather 2082 * as needed below. 2083 */ 2084 2085 restore_lscsa(next, spu); /* Steps 24-39. */ 2086 rc = check_restore_status(next, spu); /* Step 40. */ 2087 switch (rc) { 2088 default: 2089 /* Failed. Return now. */ 2090 return rc; 2091 break; 2092 case 0: 2093 /* Fall through to next step. */ 2094 break; 2095 } 2096 restore_csa(next, spu); 2097 2098 return 0; 2099} 2100 2101/** 2102 * spu_save - SPU context save, with locking. 2103 * @prev: pointer to SPU context save area, to be saved. 2104 * @spu: pointer to SPU iomem structure. 2105 * 2106 * Acquire locks, perform the save operation then return. 2107 */ 2108int spu_save(struct spu_state *prev, struct spu *spu) 2109{ 2110 int rc; 2111 2112 acquire_spu_lock(spu); /* Step 1. */ 2113 rc = __do_spu_save(prev, spu); /* Steps 2-53. */ 2114 release_spu_lock(spu); 2115 if (rc != 0 && rc != 2 && rc != 6) { 2116 panic("%s failed on SPU[%d], rc=%d.\n", 2117 __func__, spu->number, rc); 2118 } 2119 return 0; 2120} 2121EXPORT_SYMBOL_GPL(spu_save); 2122 2123/** 2124 * spu_restore - SPU context restore, with harvest and locking. 2125 * @new: pointer to SPU context save area, to be restored. 2126 * @spu: pointer to SPU iomem structure. 2127 * 2128 * Perform harvest + restore, as we may not be coming 2129 * from a previous successful save operation, and the 2130 * hardware state is unknown. 2131 */ 2132int spu_restore(struct spu_state *new, struct spu *spu) 2133{ 2134 int rc; 2135 2136 acquire_spu_lock(spu); 2137 harvest(NULL, spu); 2138 spu->slb_replace = 0; 2139 rc = __do_spu_restore(new, spu); 2140 release_spu_lock(spu); 2141 if (rc) { 2142 panic("%s failed on SPU[%d] rc=%d.\n", 2143 __func__, spu->number, rc); 2144 } 2145 return rc; 2146} 2147EXPORT_SYMBOL_GPL(spu_restore); 2148 2149static void init_prob(struct spu_state *csa) 2150{ 2151 csa->spu_chnlcnt_RW[9] = 1; 2152 csa->spu_chnlcnt_RW[21] = 16; 2153 csa->spu_chnlcnt_RW[23] = 1; 2154 csa->spu_chnlcnt_RW[28] = 1; 2155 csa->spu_chnlcnt_RW[30] = 1; 2156 csa->prob.spu_runcntl_RW = SPU_RUNCNTL_STOP; 2157 csa->prob.mb_stat_R = 0x000400; 2158} 2159 2160static void init_priv1(struct spu_state *csa) 2161{ 2162 /* Enable decode, relocate, tlbie response, master runcntl. */ 2163 csa->priv1.mfc_sr1_RW = MFC_STATE1_LOCAL_STORAGE_DECODE_MASK | 2164 MFC_STATE1_MASTER_RUN_CONTROL_MASK | 2165 MFC_STATE1_PROBLEM_STATE_MASK | 2166 MFC_STATE1_RELOCATE_MASK | MFC_STATE1_BUS_TLBIE_MASK; 2167 2168 /* Enable OS-specific set of interrupts. */ 2169 csa->priv1.int_mask_class0_RW = CLASS0_ENABLE_DMA_ALIGNMENT_INTR | 2170 CLASS0_ENABLE_INVALID_DMA_COMMAND_INTR | 2171 CLASS0_ENABLE_SPU_ERROR_INTR; 2172 csa->priv1.int_mask_class1_RW = CLASS1_ENABLE_SEGMENT_FAULT_INTR | 2173 CLASS1_ENABLE_STORAGE_FAULT_INTR; 2174 csa->priv1.int_mask_class2_RW = CLASS2_ENABLE_SPU_STOP_INTR | 2175 CLASS2_ENABLE_SPU_HALT_INTR | 2176 CLASS2_ENABLE_SPU_DMA_TAG_GROUP_COMPLETE_INTR; 2177} 2178 2179static void init_priv2(struct spu_state *csa) 2180{ 2181 csa->priv2.spu_lslr_RW = LS_ADDR_MASK; 2182 csa->priv2.mfc_control_RW = MFC_CNTL_RESUME_DMA_QUEUE | 2183 MFC_CNTL_NORMAL_DMA_QUEUE_OPERATION | 2184 MFC_CNTL_DMA_QUEUES_EMPTY_MASK; 2185} 2186 2187/** 2188 * spu_alloc_csa - allocate and initialize an SPU context save area. 2189 * 2190 * Allocate and initialize the contents of an SPU context save area. 2191 * This includes enabling address translation, interrupt masks, etc., 2192 * as appropriate for the given OS environment. 2193 * 2194 * Note that storage for the 'lscsa' is allocated separately, 2195 * as it is by far the largest of the context save regions, 2196 * and may need to be pinned or otherwise specially aligned. 2197 */ 2198int spu_init_csa(struct spu_state *csa) 2199{ 2200 int rc; 2201 2202 if (!csa) 2203 return -EINVAL; 2204 memset(csa, 0, sizeof(struct spu_state)); 2205 2206 rc = spu_alloc_lscsa(csa); 2207 if (rc) 2208 return rc; 2209 2210 spin_lock_init(&csa->register_lock); 2211 2212 init_prob(csa); 2213 init_priv1(csa); 2214 init_priv2(csa); 2215 2216 return 0; 2217} 2218 2219void spu_fini_csa(struct spu_state *csa) 2220{ 2221 spu_free_lscsa(csa); 2222} 2223