1/* 2 * spu_switch.c 3 * 4 * (C) Copyright IBM Corp. 2005 5 * 6 * Author: Mark Nutter <mnutter@us.ibm.com> 7 * 8 * Host-side part of SPU context switch sequence outlined in 9 * Synergistic Processor Element, Book IV. 10 * 11 * A fully premptive switch of an SPE is very expensive in terms 12 * of time and system resources. SPE Book IV indicates that SPE 13 * allocation should follow a "serially reusable device" model, 14 * in which the SPE is assigned a task until it completes. When 15 * this is not possible, this sequence may be used to premptively 16 * save, and then later (optionally) restore the context of a 17 * program executing on an SPE. 18 * 19 * 20 * This program is free software; you can redistribute it and/or modify 21 * it under the terms of the GNU General Public License as published by 22 * the Free Software Foundation; either version 2, or (at your option) 23 * any later version. 24 * 25 * This program is distributed in the hope that it will be useful, 26 * but WITHOUT ANY WARRANTY; without even the implied warranty of 27 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 28 * GNU General Public License for more details. 29 * 30 * You should have received a copy of the GNU General Public License 31 * along with this program; if not, write to the Free Software 32 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 33 */ 34 35#include <linux/module.h> 36#include <linux/errno.h> 37#include <linux/sched.h> 38#include <linux/kernel.h> 39#include <linux/mm.h> 40#include <linux/vmalloc.h> 41#include <linux/smp.h> 42#include <linux/stddef.h> 43#include <linux/unistd.h> 44 45#include <asm/io.h> 46#include <asm/spu.h> 47#include <asm/spu_priv1.h> 48#include <asm/spu_csa.h> 49#include <asm/mmu_context.h> 50 51#include "spu_save_dump.h" 52#include "spu_restore_dump.h" 53 54#define RELAX_SPIN_COUNT 1000 55#define POLL_WHILE_TRUE(_c) { \ 56 do { \ 57 int _i; \ 58 for (_i=0; _i<RELAX_SPIN_COUNT && (_c); _i++) { \ 59 cpu_relax(); \ 60 } \ 61 if (unlikely(_c)) yield(); \ 62 else break; \ 63 } while (_c); \ 64 } 65 66#define POLL_WHILE_FALSE(_c) POLL_WHILE_TRUE(!(_c)) 67 68static inline void acquire_spu_lock(struct spu *spu) 69{ 70 /* Save, Step 1: 71 * Restore, Step 1: 72 * Acquire SPU-specific mutual exclusion lock. 73 * TBD. 74 */ 75} 76 77static inline void release_spu_lock(struct spu *spu) 78{ 79 /* Restore, Step 76: 80 * Release SPU-specific mutual exclusion lock. 81 * TBD. 82 */ 83} 84 85static inline int check_spu_isolate(struct spu_state *csa, struct spu *spu) 86{ 87 struct spu_problem __iomem *prob = spu->problem; 88 u32 isolate_state; 89 90 /* Save, Step 2: 91 * Save, Step 6: 92 * If SPU_Status[E,L,IS] any field is '1', this 93 * SPU is in isolate state and cannot be context 94 * saved at this time. 95 */ 96 isolate_state = SPU_STATUS_ISOLATED_STATE | 97 SPU_STATUS_ISOLATED_LOAD_STATUS | SPU_STATUS_ISOLATED_EXIT_STATUS; 98 return (in_be32(&prob->spu_status_R) & isolate_state) ? 1 : 0; 99} 100 101static inline void disable_interrupts(struct spu_state *csa, struct spu *spu) 102{ 103 /* Save, Step 3: 104 * Restore, Step 2: 105 * Save INT_Mask_class0 in CSA. 106 * Write INT_MASK_class0 with value of 0. 107 * Save INT_Mask_class1 in CSA. 108 * Write INT_MASK_class1 with value of 0. 109 * Save INT_Mask_class2 in CSA. 110 * Write INT_MASK_class2 with value of 0. 111 */ 112 spin_lock_irq(&spu->register_lock); 113 if (csa) { 114 csa->priv1.int_mask_class0_RW = spu_int_mask_get(spu, 0); 115 csa->priv1.int_mask_class1_RW = spu_int_mask_get(spu, 1); 116 csa->priv1.int_mask_class2_RW = spu_int_mask_get(spu, 2); 117 } 118 spu_int_mask_set(spu, 0, 0ul); 119 spu_int_mask_set(spu, 1, 0ul); 120 spu_int_mask_set(spu, 2, 0ul); 121 eieio(); 122 spin_unlock_irq(&spu->register_lock); 123} 124 125static inline void set_watchdog_timer(struct spu_state *csa, struct spu *spu) 126{ 127 /* Save, Step 4: 128 * Restore, Step 25. 129 * Set a software watchdog timer, which specifies the 130 * maximum allowable time for a context save sequence. 131 * 132 * For present, this implementation will not set a global 133 * watchdog timer, as virtualization & variable system load 134 * may cause unpredictable execution times. 135 */ 136} 137 138static inline void inhibit_user_access(struct spu_state *csa, struct spu *spu) 139{ 140 /* Save, Step 5: 141 * Restore, Step 3: 142 * Inhibit user-space access (if provided) to this 143 * SPU by unmapping the virtual pages assigned to 144 * the SPU memory-mapped I/O (MMIO) for problem 145 * state. TBD. 146 */ 147} 148 149static inline void set_switch_pending(struct spu_state *csa, struct spu *spu) 150{ 151 /* Save, Step 7: 152 * Restore, Step 5: 153 * Set a software context switch pending flag. 154 */ 155 set_bit(SPU_CONTEXT_SWITCH_PENDING, &spu->flags); 156 mb(); 157} 158 159static inline void save_mfc_cntl(struct spu_state *csa, struct spu *spu) 160{ 161 struct spu_priv2 __iomem *priv2 = spu->priv2; 162 163 /* Save, Step 8: 164 * Suspend DMA and save MFC_CNTL. 165 */ 166 switch (in_be64(&priv2->mfc_control_RW) & 167 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) { 168 case MFC_CNTL_SUSPEND_IN_PROGRESS: 169 POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) & 170 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) == 171 MFC_CNTL_SUSPEND_COMPLETE); 172 /* fall through */ 173 case MFC_CNTL_SUSPEND_COMPLETE: 174 if (csa) { 175 csa->priv2.mfc_control_RW = 176 in_be64(&priv2->mfc_control_RW) | 177 MFC_CNTL_SUSPEND_DMA_QUEUE; 178 } 179 break; 180 case MFC_CNTL_NORMAL_DMA_QUEUE_OPERATION: 181 out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE); 182 POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) & 183 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) == 184 MFC_CNTL_SUSPEND_COMPLETE); 185 if (csa) { 186 csa->priv2.mfc_control_RW = 187 in_be64(&priv2->mfc_control_RW) & 188 ~MFC_CNTL_SUSPEND_DMA_QUEUE; 189 } 190 break; 191 } 192} 193 194static inline void save_spu_runcntl(struct spu_state *csa, struct spu *spu) 195{ 196 struct spu_problem __iomem *prob = spu->problem; 197 198 /* Save, Step 9: 199 * Save SPU_Runcntl in the CSA. This value contains 200 * the "Application Desired State". 201 */ 202 csa->prob.spu_runcntl_RW = in_be32(&prob->spu_runcntl_RW); 203} 204 205static inline void save_mfc_sr1(struct spu_state *csa, struct spu *spu) 206{ 207 /* Save, Step 10: 208 * Save MFC_SR1 in the CSA. 209 */ 210 csa->priv1.mfc_sr1_RW = spu_mfc_sr1_get(spu); 211} 212 213static inline void save_spu_status(struct spu_state *csa, struct spu *spu) 214{ 215 struct spu_problem __iomem *prob = spu->problem; 216 217 /* Save, Step 11: 218 * Read SPU_Status[R], and save to CSA. 219 */ 220 if ((in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) == 0) { 221 csa->prob.spu_status_R = in_be32(&prob->spu_status_R); 222 } else { 223 u32 stopped; 224 225 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 226 eieio(); 227 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 228 SPU_STATUS_RUNNING); 229 stopped = 230 SPU_STATUS_INVALID_INSTR | SPU_STATUS_SINGLE_STEP | 231 SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP; 232 if ((in_be32(&prob->spu_status_R) & stopped) == 0) 233 csa->prob.spu_status_R = SPU_STATUS_RUNNING; 234 else 235 csa->prob.spu_status_R = in_be32(&prob->spu_status_R); 236 } 237} 238 239static inline void save_mfc_decr(struct spu_state *csa, struct spu *spu) 240{ 241 struct spu_priv2 __iomem *priv2 = spu->priv2; 242 243 /* Save, Step 12: 244 * Read MFC_CNTL[Ds]. Update saved copy of 245 * CSA.MFC_CNTL[Ds]. 246 */ 247 if (in_be64(&priv2->mfc_control_RW) & MFC_CNTL_DECREMENTER_RUNNING) { 248 csa->priv2.mfc_control_RW |= MFC_CNTL_DECREMENTER_RUNNING; 249 csa->suspend_time = get_cycles(); 250 out_be64(&priv2->spu_chnlcntptr_RW, 7ULL); 251 eieio(); 252 csa->spu_chnldata_RW[7] = in_be64(&priv2->spu_chnldata_RW); 253 eieio(); 254 } else { 255 csa->priv2.mfc_control_RW &= ~MFC_CNTL_DECREMENTER_RUNNING; 256 } 257} 258 259static inline void halt_mfc_decr(struct spu_state *csa, struct spu *spu) 260{ 261 struct spu_priv2 __iomem *priv2 = spu->priv2; 262 263 /* Save, Step 13: 264 * Write MFC_CNTL[Dh] set to a '1' to halt 265 * the decrementer. 266 */ 267 out_be64(&priv2->mfc_control_RW, MFC_CNTL_DECREMENTER_HALTED); 268 eieio(); 269} 270 271static inline void save_timebase(struct spu_state *csa, struct spu *spu) 272{ 273 /* Save, Step 14: 274 * Read PPE Timebase High and Timebase low registers 275 * and save in CSA. TBD. 276 */ 277 csa->suspend_time = get_cycles(); 278} 279 280static inline void remove_other_spu_access(struct spu_state *csa, 281 struct spu *spu) 282{ 283 /* Save, Step 15: 284 * Remove other SPU access to this SPU by unmapping 285 * this SPU's pages from their address space. TBD. 286 */ 287} 288 289static inline void do_mfc_mssync(struct spu_state *csa, struct spu *spu) 290{ 291 struct spu_problem __iomem *prob = spu->problem; 292 293 /* Save, Step 16: 294 * Restore, Step 11. 295 * Write SPU_MSSync register. Poll SPU_MSSync[P] 296 * for a value of 0. 297 */ 298 out_be64(&prob->spc_mssync_RW, 1UL); 299 POLL_WHILE_TRUE(in_be64(&prob->spc_mssync_RW) & MS_SYNC_PENDING); 300} 301 302static inline void issue_mfc_tlbie(struct spu_state *csa, struct spu *spu) 303{ 304 /* Save, Step 17: 305 * Restore, Step 12. 306 * Restore, Step 48. 307 * Write TLB_Invalidate_Entry[IS,VPN,L,Lp]=0 register. 308 * Then issue a PPE sync instruction. 309 */ 310 spu_tlb_invalidate(spu); 311 mb(); 312} 313 314static inline void handle_pending_interrupts(struct spu_state *csa, 315 struct spu *spu) 316{ 317 /* Save, Step 18: 318 * Handle any pending interrupts from this SPU 319 * here. This is OS or hypervisor specific. One 320 * option is to re-enable interrupts to handle any 321 * pending interrupts, with the interrupt handlers 322 * recognizing the software Context Switch Pending 323 * flag, to ensure the SPU execution or MFC command 324 * queue is not restarted. TBD. 325 */ 326} 327 328static inline void save_mfc_queues(struct spu_state *csa, struct spu *spu) 329{ 330 struct spu_priv2 __iomem *priv2 = spu->priv2; 331 int i; 332 333 /* Save, Step 19: 334 * If MFC_Cntl[Se]=0 then save 335 * MFC command queues. 336 */ 337 if ((in_be64(&priv2->mfc_control_RW) & MFC_CNTL_DMA_QUEUES_EMPTY) == 0) { 338 for (i = 0; i < 8; i++) { 339 csa->priv2.puq[i].mfc_cq_data0_RW = 340 in_be64(&priv2->puq[i].mfc_cq_data0_RW); 341 csa->priv2.puq[i].mfc_cq_data1_RW = 342 in_be64(&priv2->puq[i].mfc_cq_data1_RW); 343 csa->priv2.puq[i].mfc_cq_data2_RW = 344 in_be64(&priv2->puq[i].mfc_cq_data2_RW); 345 csa->priv2.puq[i].mfc_cq_data3_RW = 346 in_be64(&priv2->puq[i].mfc_cq_data3_RW); 347 } 348 for (i = 0; i < 16; i++) { 349 csa->priv2.spuq[i].mfc_cq_data0_RW = 350 in_be64(&priv2->spuq[i].mfc_cq_data0_RW); 351 csa->priv2.spuq[i].mfc_cq_data1_RW = 352 in_be64(&priv2->spuq[i].mfc_cq_data1_RW); 353 csa->priv2.spuq[i].mfc_cq_data2_RW = 354 in_be64(&priv2->spuq[i].mfc_cq_data2_RW); 355 csa->priv2.spuq[i].mfc_cq_data3_RW = 356 in_be64(&priv2->spuq[i].mfc_cq_data3_RW); 357 } 358 } 359} 360 361static inline void save_ppu_querymask(struct spu_state *csa, struct spu *spu) 362{ 363 struct spu_problem __iomem *prob = spu->problem; 364 365 /* Save, Step 20: 366 * Save the PPU_QueryMask register 367 * in the CSA. 368 */ 369 csa->prob.dma_querymask_RW = in_be32(&prob->dma_querymask_RW); 370} 371 372static inline void save_ppu_querytype(struct spu_state *csa, struct spu *spu) 373{ 374 struct spu_problem __iomem *prob = spu->problem; 375 376 /* Save, Step 21: 377 * Save the PPU_QueryType register 378 * in the CSA. 379 */ 380 csa->prob.dma_querytype_RW = in_be32(&prob->dma_querytype_RW); 381} 382 383static inline void save_mfc_csr_tsq(struct spu_state *csa, struct spu *spu) 384{ 385 struct spu_priv2 __iomem *priv2 = spu->priv2; 386 387 /* Save, Step 22: 388 * Save the MFC_CSR_TSQ register 389 * in the LSCSA. 390 */ 391 csa->priv2.spu_tag_status_query_RW = 392 in_be64(&priv2->spu_tag_status_query_RW); 393} 394 395static inline void save_mfc_csr_cmd(struct spu_state *csa, struct spu *spu) 396{ 397 struct spu_priv2 __iomem *priv2 = spu->priv2; 398 399 /* Save, Step 23: 400 * Save the MFC_CSR_CMD1 and MFC_CSR_CMD2 401 * registers in the CSA. 402 */ 403 csa->priv2.spu_cmd_buf1_RW = in_be64(&priv2->spu_cmd_buf1_RW); 404 csa->priv2.spu_cmd_buf2_RW = in_be64(&priv2->spu_cmd_buf2_RW); 405} 406 407static inline void save_mfc_csr_ato(struct spu_state *csa, struct spu *spu) 408{ 409 struct spu_priv2 __iomem *priv2 = spu->priv2; 410 411 /* Save, Step 24: 412 * Save the MFC_CSR_ATO register in 413 * the CSA. 414 */ 415 csa->priv2.spu_atomic_status_RW = in_be64(&priv2->spu_atomic_status_RW); 416} 417 418static inline void save_mfc_tclass_id(struct spu_state *csa, struct spu *spu) 419{ 420 /* Save, Step 25: 421 * Save the MFC_TCLASS_ID register in 422 * the CSA. 423 */ 424 csa->priv1.mfc_tclass_id_RW = spu_mfc_tclass_id_get(spu); 425} 426 427static inline void set_mfc_tclass_id(struct spu_state *csa, struct spu *spu) 428{ 429 /* Save, Step 26: 430 * Restore, Step 23. 431 * Write the MFC_TCLASS_ID register with 432 * the value 0x10000000. 433 */ 434 spu_mfc_tclass_id_set(spu, 0x10000000); 435 eieio(); 436} 437 438static inline void purge_mfc_queue(struct spu_state *csa, struct spu *spu) 439{ 440 struct spu_priv2 __iomem *priv2 = spu->priv2; 441 442 /* Save, Step 27: 443 * Restore, Step 14. 444 * Write MFC_CNTL[Pc]=1 (purge queue). 445 */ 446 out_be64(&priv2->mfc_control_RW, MFC_CNTL_PURGE_DMA_REQUEST); 447 eieio(); 448} 449 450static inline void wait_purge_complete(struct spu_state *csa, struct spu *spu) 451{ 452 struct spu_priv2 __iomem *priv2 = spu->priv2; 453 454 /* Save, Step 28: 455 * Poll MFC_CNTL[Ps] until value '11' is read 456 * (purge complete). 457 */ 458 POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) & 459 MFC_CNTL_PURGE_DMA_STATUS_MASK) == 460 MFC_CNTL_PURGE_DMA_COMPLETE); 461} 462 463static inline void setup_mfc_sr1(struct spu_state *csa, struct spu *spu) 464{ 465 /* Save, Step 30: 466 * Restore, Step 18: 467 * Write MFC_SR1 with MFC_SR1[D=0,S=1] and 468 * MFC_SR1[TL,R,Pr,T] set correctly for the 469 * OS specific environment. 470 * 471 * Implementation note: The SPU-side code 472 * for save/restore is privileged, so the 473 * MFC_SR1[Pr] bit is not set. 474 * 475 */ 476 spu_mfc_sr1_set(spu, (MFC_STATE1_MASTER_RUN_CONTROL_MASK | 477 MFC_STATE1_RELOCATE_MASK | 478 MFC_STATE1_BUS_TLBIE_MASK)); 479} 480 481static inline void save_spu_npc(struct spu_state *csa, struct spu *spu) 482{ 483 struct spu_problem __iomem *prob = spu->problem; 484 485 /* Save, Step 31: 486 * Save SPU_NPC in the CSA. 487 */ 488 csa->prob.spu_npc_RW = in_be32(&prob->spu_npc_RW); 489} 490 491static inline void save_spu_privcntl(struct spu_state *csa, struct spu *spu) 492{ 493 struct spu_priv2 __iomem *priv2 = spu->priv2; 494 495 /* Save, Step 32: 496 * Save SPU_PrivCntl in the CSA. 497 */ 498 csa->priv2.spu_privcntl_RW = in_be64(&priv2->spu_privcntl_RW); 499} 500 501static inline void reset_spu_privcntl(struct spu_state *csa, struct spu *spu) 502{ 503 struct spu_priv2 __iomem *priv2 = spu->priv2; 504 505 /* Save, Step 33: 506 * Restore, Step 16: 507 * Write SPU_PrivCntl[S,Le,A] fields reset to 0. 508 */ 509 out_be64(&priv2->spu_privcntl_RW, 0UL); 510 eieio(); 511} 512 513static inline void save_spu_lslr(struct spu_state *csa, struct spu *spu) 514{ 515 struct spu_priv2 __iomem *priv2 = spu->priv2; 516 517 /* Save, Step 34: 518 * Save SPU_LSLR in the CSA. 519 */ 520 csa->priv2.spu_lslr_RW = in_be64(&priv2->spu_lslr_RW); 521} 522 523static inline void reset_spu_lslr(struct spu_state *csa, struct spu *spu) 524{ 525 struct spu_priv2 __iomem *priv2 = spu->priv2; 526 527 /* Save, Step 35: 528 * Restore, Step 17. 529 * Reset SPU_LSLR. 530 */ 531 out_be64(&priv2->spu_lslr_RW, LS_ADDR_MASK); 532 eieio(); 533} 534 535static inline void save_spu_cfg(struct spu_state *csa, struct spu *spu) 536{ 537 struct spu_priv2 __iomem *priv2 = spu->priv2; 538 539 /* Save, Step 36: 540 * Save SPU_Cfg in the CSA. 541 */ 542 csa->priv2.spu_cfg_RW = in_be64(&priv2->spu_cfg_RW); 543} 544 545static inline void save_pm_trace(struct spu_state *csa, struct spu *spu) 546{ 547 /* Save, Step 37: 548 * Save PM_Trace_Tag_Wait_Mask in the CSA. 549 * Not performed by this implementation. 550 */ 551} 552 553static inline void save_mfc_rag(struct spu_state *csa, struct spu *spu) 554{ 555 /* Save, Step 38: 556 * Save RA_GROUP_ID register and the 557 * RA_ENABLE reigster in the CSA. 558 */ 559 csa->priv1.resource_allocation_groupID_RW = 560 spu_resource_allocation_groupID_get(spu); 561 csa->priv1.resource_allocation_enable_RW = 562 spu_resource_allocation_enable_get(spu); 563} 564 565static inline void save_ppu_mb_stat(struct spu_state *csa, struct spu *spu) 566{ 567 struct spu_problem __iomem *prob = spu->problem; 568 569 /* Save, Step 39: 570 * Save MB_Stat register in the CSA. 571 */ 572 csa->prob.mb_stat_R = in_be32(&prob->mb_stat_R); 573} 574 575static inline void save_ppu_mb(struct spu_state *csa, struct spu *spu) 576{ 577 struct spu_problem __iomem *prob = spu->problem; 578 579 /* Save, Step 40: 580 * Save the PPU_MB register in the CSA. 581 */ 582 csa->prob.pu_mb_R = in_be32(&prob->pu_mb_R); 583} 584 585static inline void save_ppuint_mb(struct spu_state *csa, struct spu *spu) 586{ 587 struct spu_priv2 __iomem *priv2 = spu->priv2; 588 589 /* Save, Step 41: 590 * Save the PPUINT_MB register in the CSA. 591 */ 592 csa->priv2.puint_mb_R = in_be64(&priv2->puint_mb_R); 593} 594 595static inline void save_ch_part1(struct spu_state *csa, struct spu *spu) 596{ 597 struct spu_priv2 __iomem *priv2 = spu->priv2; 598 u64 idx, ch_indices[7] = { 0UL, 3UL, 4UL, 24UL, 25UL, 27UL }; 599 int i; 600 601 /* Save, Step 42: 602 */ 603 604 /* Save CH 1, without channel count */ 605 out_be64(&priv2->spu_chnlcntptr_RW, 1); 606 csa->spu_chnldata_RW[1] = in_be64(&priv2->spu_chnldata_RW); 607 608 /* Save the following CH: [0,3,4,24,25,27] */ 609 for (i = 0; i < 7; i++) { 610 idx = ch_indices[i]; 611 out_be64(&priv2->spu_chnlcntptr_RW, idx); 612 eieio(); 613 csa->spu_chnldata_RW[idx] = in_be64(&priv2->spu_chnldata_RW); 614 csa->spu_chnlcnt_RW[idx] = in_be64(&priv2->spu_chnlcnt_RW); 615 out_be64(&priv2->spu_chnldata_RW, 0UL); 616 out_be64(&priv2->spu_chnlcnt_RW, 0UL); 617 eieio(); 618 } 619} 620 621static inline void save_spu_mb(struct spu_state *csa, struct spu *spu) 622{ 623 struct spu_priv2 __iomem *priv2 = spu->priv2; 624 int i; 625 626 /* Save, Step 43: 627 * Save SPU Read Mailbox Channel. 628 */ 629 out_be64(&priv2->spu_chnlcntptr_RW, 29UL); 630 eieio(); 631 csa->spu_chnlcnt_RW[29] = in_be64(&priv2->spu_chnlcnt_RW); 632 for (i = 0; i < 4; i++) { 633 csa->spu_mailbox_data[i] = in_be64(&priv2->spu_chnldata_RW); 634 } 635 out_be64(&priv2->spu_chnlcnt_RW, 0UL); 636 eieio(); 637} 638 639static inline void save_mfc_cmd(struct spu_state *csa, struct spu *spu) 640{ 641 struct spu_priv2 __iomem *priv2 = spu->priv2; 642 643 /* Save, Step 44: 644 * Save MFC_CMD Channel. 645 */ 646 out_be64(&priv2->spu_chnlcntptr_RW, 21UL); 647 eieio(); 648 csa->spu_chnlcnt_RW[21] = in_be64(&priv2->spu_chnlcnt_RW); 649 eieio(); 650} 651 652static inline void reset_ch(struct spu_state *csa, struct spu *spu) 653{ 654 struct spu_priv2 __iomem *priv2 = spu->priv2; 655 u64 ch_indices[4] = { 21UL, 23UL, 28UL, 30UL }; 656 u64 ch_counts[4] = { 16UL, 1UL, 1UL, 1UL }; 657 u64 idx; 658 int i; 659 660 /* Save, Step 45: 661 * Reset the following CH: [21, 23, 28, 30] 662 */ 663 for (i = 0; i < 4; i++) { 664 idx = ch_indices[i]; 665 out_be64(&priv2->spu_chnlcntptr_RW, idx); 666 eieio(); 667 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]); 668 eieio(); 669 } 670} 671 672static inline void resume_mfc_queue(struct spu_state *csa, struct spu *spu) 673{ 674 struct spu_priv2 __iomem *priv2 = spu->priv2; 675 676 /* Save, Step 46: 677 * Restore, Step 25. 678 * Write MFC_CNTL[Sc]=0 (resume queue processing). 679 */ 680 out_be64(&priv2->mfc_control_RW, MFC_CNTL_RESUME_DMA_QUEUE); 681} 682 683static inline void get_kernel_slb(u64 ea, u64 slb[2]) 684{ 685 u64 llp; 686 687 if (REGION_ID(ea) == KERNEL_REGION_ID) 688 llp = mmu_psize_defs[mmu_linear_psize].sllp; 689 else 690 llp = mmu_psize_defs[mmu_virtual_psize].sllp; 691 slb[0] = (get_kernel_vsid(ea) << SLB_VSID_SHIFT) | 692 SLB_VSID_KERNEL | llp; 693 slb[1] = (ea & ESID_MASK) | SLB_ESID_V; 694} 695 696static inline void load_mfc_slb(struct spu *spu, u64 slb[2], int slbe) 697{ 698 struct spu_priv2 __iomem *priv2 = spu->priv2; 699 700 out_be64(&priv2->slb_index_W, slbe); 701 eieio(); 702 out_be64(&priv2->slb_vsid_RW, slb[0]); 703 out_be64(&priv2->slb_esid_RW, slb[1]); 704 eieio(); 705} 706 707static inline void setup_mfc_slbs(struct spu_state *csa, struct spu *spu) 708{ 709 u64 code_slb[2]; 710 u64 lscsa_slb[2]; 711 712 /* Save, Step 47: 713 * Restore, Step 30. 714 * If MFC_SR1[R]=1, write 0 to SLB_Invalidate_All 715 * register, then initialize SLB_VSID and SLB_ESID 716 * to provide access to SPU context save code and 717 * LSCSA. 718 * 719 * This implementation places both the context 720 * switch code and LSCSA in kernel address space. 721 * 722 * Further this implementation assumes that the 723 * MFC_SR1[R]=1 (in other words, assume that 724 * translation is desired by OS environment). 725 */ 726 spu_invalidate_slbs(spu); 727 get_kernel_slb((unsigned long)&spu_save_code[0], code_slb); 728 get_kernel_slb((unsigned long)csa->lscsa, lscsa_slb); 729 load_mfc_slb(spu, code_slb, 0); 730 if ((lscsa_slb[0] != code_slb[0]) || (lscsa_slb[1] != code_slb[1])) 731 load_mfc_slb(spu, lscsa_slb, 1); 732} 733 734static inline void set_switch_active(struct spu_state *csa, struct spu *spu) 735{ 736 /* Save, Step 48: 737 * Restore, Step 23. 738 * Change the software context switch pending flag 739 * to context switch active. 740 */ 741 set_bit(SPU_CONTEXT_SWITCH_ACTIVE, &spu->flags); 742 clear_bit(SPU_CONTEXT_SWITCH_PENDING, &spu->flags); 743 mb(); 744} 745 746static inline void enable_interrupts(struct spu_state *csa, struct spu *spu) 747{ 748 unsigned long class1_mask = CLASS1_ENABLE_SEGMENT_FAULT_INTR | 749 CLASS1_ENABLE_STORAGE_FAULT_INTR; 750 751 /* Save, Step 49: 752 * Restore, Step 22: 753 * Reset and then enable interrupts, as 754 * needed by OS. 755 * 756 * This implementation enables only class1 757 * (translation) interrupts. 758 */ 759 spin_lock_irq(&spu->register_lock); 760 spu_int_stat_clear(spu, 0, ~0ul); 761 spu_int_stat_clear(spu, 1, ~0ul); 762 spu_int_stat_clear(spu, 2, ~0ul); 763 spu_int_mask_set(spu, 0, 0ul); 764 spu_int_mask_set(spu, 1, class1_mask); 765 spu_int_mask_set(spu, 2, 0ul); 766 spin_unlock_irq(&spu->register_lock); 767} 768 769static inline int send_mfc_dma(struct spu *spu, unsigned long ea, 770 unsigned int ls_offset, unsigned int size, 771 unsigned int tag, unsigned int rclass, 772 unsigned int cmd) 773{ 774 struct spu_problem __iomem *prob = spu->problem; 775 union mfc_tag_size_class_cmd command; 776 unsigned int transfer_size; 777 volatile unsigned int status = 0x0; 778 779 while (size > 0) { 780 transfer_size = 781 (size > MFC_MAX_DMA_SIZE) ? MFC_MAX_DMA_SIZE : size; 782 command.u.mfc_size = transfer_size; 783 command.u.mfc_tag = tag; 784 command.u.mfc_rclassid = rclass; 785 command.u.mfc_cmd = cmd; 786 do { 787 out_be32(&prob->mfc_lsa_W, ls_offset); 788 out_be64(&prob->mfc_ea_W, ea); 789 out_be64(&prob->mfc_union_W.all64, command.all64); 790 status = 791 in_be32(&prob->mfc_union_W.by32.mfc_class_cmd32); 792 if (unlikely(status & 0x2)) { 793 cpu_relax(); 794 } 795 } while (status & 0x3); 796 size -= transfer_size; 797 ea += transfer_size; 798 ls_offset += transfer_size; 799 } 800 return 0; 801} 802 803static inline void save_ls_16kb(struct spu_state *csa, struct spu *spu) 804{ 805 unsigned long addr = (unsigned long)&csa->lscsa->ls[0]; 806 unsigned int ls_offset = 0x0; 807 unsigned int size = 16384; 808 unsigned int tag = 0; 809 unsigned int rclass = 0; 810 unsigned int cmd = MFC_PUT_CMD; 811 812 /* Save, Step 50: 813 * Issue a DMA command to copy the first 16K bytes 814 * of local storage to the CSA. 815 */ 816 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 817} 818 819static inline void set_spu_npc(struct spu_state *csa, struct spu *spu) 820{ 821 struct spu_problem __iomem *prob = spu->problem; 822 823 /* Save, Step 51: 824 * Restore, Step 31. 825 * Write SPU_NPC[IE]=0 and SPU_NPC[LSA] to entry 826 * point address of context save code in local 827 * storage. 828 * 829 * This implementation uses SPU-side save/restore 830 * programs with entry points at LSA of 0. 831 */ 832 out_be32(&prob->spu_npc_RW, 0); 833 eieio(); 834} 835 836static inline void set_signot1(struct spu_state *csa, struct spu *spu) 837{ 838 struct spu_problem __iomem *prob = spu->problem; 839 union { 840 u64 ull; 841 u32 ui[2]; 842 } addr64; 843 844 /* Save, Step 52: 845 * Restore, Step 32: 846 * Write SPU_Sig_Notify_1 register with upper 32-bits 847 * of the CSA.LSCSA effective address. 848 */ 849 addr64.ull = (u64) csa->lscsa; 850 out_be32(&prob->signal_notify1, addr64.ui[0]); 851 eieio(); 852} 853 854static inline void set_signot2(struct spu_state *csa, struct spu *spu) 855{ 856 struct spu_problem __iomem *prob = spu->problem; 857 union { 858 u64 ull; 859 u32 ui[2]; 860 } addr64; 861 862 /* Save, Step 53: 863 * Restore, Step 33: 864 * Write SPU_Sig_Notify_2 register with lower 32-bits 865 * of the CSA.LSCSA effective address. 866 */ 867 addr64.ull = (u64) csa->lscsa; 868 out_be32(&prob->signal_notify2, addr64.ui[1]); 869 eieio(); 870} 871 872static inline void send_save_code(struct spu_state *csa, struct spu *spu) 873{ 874 unsigned long addr = (unsigned long)&spu_save_code[0]; 875 unsigned int ls_offset = 0x0; 876 unsigned int size = sizeof(spu_save_code); 877 unsigned int tag = 0; 878 unsigned int rclass = 0; 879 unsigned int cmd = MFC_GETFS_CMD; 880 881 /* Save, Step 54: 882 * Issue a DMA command to copy context save code 883 * to local storage and start SPU. 884 */ 885 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 886} 887 888static inline void set_ppu_querymask(struct spu_state *csa, struct spu *spu) 889{ 890 struct spu_problem __iomem *prob = spu->problem; 891 892 /* Save, Step 55: 893 * Restore, Step 38. 894 * Write PPU_QueryMask=1 (enable Tag Group 0) 895 * and issue eieio instruction. 896 */ 897 out_be32(&prob->dma_querymask_RW, MFC_TAGID_TO_TAGMASK(0)); 898 eieio(); 899} 900 901static inline void wait_tag_complete(struct spu_state *csa, struct spu *spu) 902{ 903 struct spu_problem __iomem *prob = spu->problem; 904 u32 mask = MFC_TAGID_TO_TAGMASK(0); 905 unsigned long flags; 906 907 /* Save, Step 56: 908 * Restore, Step 39. 909 * Restore, Step 39. 910 * Restore, Step 46. 911 * Poll PPU_TagStatus[gn] until 01 (Tag group 0 complete) 912 * or write PPU_QueryType[TS]=01 and wait for Tag Group 913 * Complete Interrupt. Write INT_Stat_Class0 or 914 * INT_Stat_Class2 with value of 'handled'. 915 */ 916 POLL_WHILE_FALSE(in_be32(&prob->dma_tagstatus_R) & mask); 917 918 local_irq_save(flags); 919 spu_int_stat_clear(spu, 0, ~(0ul)); 920 spu_int_stat_clear(spu, 2, ~(0ul)); 921 local_irq_restore(flags); 922} 923 924static inline void wait_spu_stopped(struct spu_state *csa, struct spu *spu) 925{ 926 struct spu_problem __iomem *prob = spu->problem; 927 unsigned long flags; 928 929 /* Save, Step 57: 930 * Restore, Step 40. 931 * Poll until SPU_Status[R]=0 or wait for SPU Class 0 932 * or SPU Class 2 interrupt. Write INT_Stat_class0 933 * or INT_Stat_class2 with value of handled. 934 */ 935 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING); 936 937 local_irq_save(flags); 938 spu_int_stat_clear(spu, 0, ~(0ul)); 939 spu_int_stat_clear(spu, 2, ~(0ul)); 940 local_irq_restore(flags); 941} 942 943static inline int check_save_status(struct spu_state *csa, struct spu *spu) 944{ 945 struct spu_problem __iomem *prob = spu->problem; 946 u32 complete; 947 948 /* Save, Step 54: 949 * If SPU_Status[P]=1 and SPU_Status[SC] = "success", 950 * context save succeeded, otherwise context save 951 * failed. 952 */ 953 complete = ((SPU_SAVE_COMPLETE << SPU_STOP_STATUS_SHIFT) | 954 SPU_STATUS_STOPPED_BY_STOP); 955 return (in_be32(&prob->spu_status_R) != complete) ? 1 : 0; 956} 957 958static inline void terminate_spu_app(struct spu_state *csa, struct spu *spu) 959{ 960 /* Restore, Step 4: 961 * If required, notify the "using application" that 962 * the SPU task has been terminated. TBD. 963 */ 964} 965 966static inline void suspend_mfc(struct spu_state *csa, struct spu *spu) 967{ 968 struct spu_priv2 __iomem *priv2 = spu->priv2; 969 970 /* Restore, Step 7: 971 * Restore, Step 47. 972 * Write MFC_Cntl[Dh,Sc]='1','1' to suspend 973 * the queue and halt the decrementer. 974 */ 975 out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE | 976 MFC_CNTL_DECREMENTER_HALTED); 977 eieio(); 978} 979 980static inline void wait_suspend_mfc_complete(struct spu_state *csa, 981 struct spu *spu) 982{ 983 struct spu_priv2 __iomem *priv2 = spu->priv2; 984 985 /* Restore, Step 8: 986 * Restore, Step 47. 987 * Poll MFC_CNTL[Ss] until 11 is returned. 988 */ 989 POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) & 990 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) == 991 MFC_CNTL_SUSPEND_COMPLETE); 992} 993 994static inline int suspend_spe(struct spu_state *csa, struct spu *spu) 995{ 996 struct spu_problem __iomem *prob = spu->problem; 997 998 /* Restore, Step 9: 999 * If SPU_Status[R]=1, stop SPU execution 1000 * and wait for stop to complete. 1001 * 1002 * Returns 1 if SPU_Status[R]=1 on entry. 1003 * 0 otherwise 1004 */ 1005 if (in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) { 1006 if (in_be32(&prob->spu_status_R) & 1007 SPU_STATUS_ISOLATED_EXIT_STATUS) { 1008 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1009 SPU_STATUS_RUNNING); 1010 } 1011 if ((in_be32(&prob->spu_status_R) & 1012 SPU_STATUS_ISOLATED_LOAD_STATUS) 1013 || (in_be32(&prob->spu_status_R) & 1014 SPU_STATUS_ISOLATED_STATE)) { 1015 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 1016 eieio(); 1017 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1018 SPU_STATUS_RUNNING); 1019 out_be32(&prob->spu_runcntl_RW, 0x2); 1020 eieio(); 1021 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1022 SPU_STATUS_RUNNING); 1023 } 1024 if (in_be32(&prob->spu_status_R) & 1025 SPU_STATUS_WAITING_FOR_CHANNEL) { 1026 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 1027 eieio(); 1028 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1029 SPU_STATUS_RUNNING); 1030 } 1031 return 1; 1032 } 1033 return 0; 1034} 1035 1036static inline void clear_spu_status(struct spu_state *csa, struct spu *spu) 1037{ 1038 struct spu_problem __iomem *prob = spu->problem; 1039 1040 /* Restore, Step 10: 1041 * If SPU_Status[R]=0 and SPU_Status[E,L,IS]=1, 1042 * release SPU from isolate state. 1043 */ 1044 if (!(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING)) { 1045 if (in_be32(&prob->spu_status_R) & 1046 SPU_STATUS_ISOLATED_EXIT_STATUS) { 1047 spu_mfc_sr1_set(spu, 1048 MFC_STATE1_MASTER_RUN_CONTROL_MASK); 1049 eieio(); 1050 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1051 eieio(); 1052 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1053 SPU_STATUS_RUNNING); 1054 } 1055 if ((in_be32(&prob->spu_status_R) & 1056 SPU_STATUS_ISOLATED_LOAD_STATUS) 1057 || (in_be32(&prob->spu_status_R) & 1058 SPU_STATUS_ISOLATED_STATE)) { 1059 spu_mfc_sr1_set(spu, 1060 MFC_STATE1_MASTER_RUN_CONTROL_MASK); 1061 eieio(); 1062 out_be32(&prob->spu_runcntl_RW, 0x2); 1063 eieio(); 1064 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1065 SPU_STATUS_RUNNING); 1066 } 1067 } 1068} 1069 1070static inline void reset_ch_part1(struct spu_state *csa, struct spu *spu) 1071{ 1072 struct spu_priv2 __iomem *priv2 = spu->priv2; 1073 u64 ch_indices[7] = { 0UL, 3UL, 4UL, 24UL, 25UL, 27UL }; 1074 u64 idx; 1075 int i; 1076 1077 /* Restore, Step 20: 1078 */ 1079 1080 /* Reset CH 1 */ 1081 out_be64(&priv2->spu_chnlcntptr_RW, 1); 1082 out_be64(&priv2->spu_chnldata_RW, 0UL); 1083 1084 /* Reset the following CH: [0,3,4,24,25,27] */ 1085 for (i = 0; i < 7; i++) { 1086 idx = ch_indices[i]; 1087 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1088 eieio(); 1089 out_be64(&priv2->spu_chnldata_RW, 0UL); 1090 out_be64(&priv2->spu_chnlcnt_RW, 0UL); 1091 eieio(); 1092 } 1093} 1094 1095static inline void reset_ch_part2(struct spu_state *csa, struct spu *spu) 1096{ 1097 struct spu_priv2 __iomem *priv2 = spu->priv2; 1098 u64 ch_indices[5] = { 21UL, 23UL, 28UL, 29UL, 30UL }; 1099 u64 ch_counts[5] = { 16UL, 1UL, 1UL, 0UL, 1UL }; 1100 u64 idx; 1101 int i; 1102 1103 /* Restore, Step 21: 1104 * Reset the following CH: [21, 23, 28, 29, 30] 1105 */ 1106 for (i = 0; i < 5; i++) { 1107 idx = ch_indices[i]; 1108 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1109 eieio(); 1110 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]); 1111 eieio(); 1112 } 1113} 1114 1115static inline void setup_spu_status_part1(struct spu_state *csa, 1116 struct spu *spu) 1117{ 1118 u32 status_P = SPU_STATUS_STOPPED_BY_STOP; 1119 u32 status_I = SPU_STATUS_INVALID_INSTR; 1120 u32 status_H = SPU_STATUS_STOPPED_BY_HALT; 1121 u32 status_S = SPU_STATUS_SINGLE_STEP; 1122 u32 status_S_I = SPU_STATUS_SINGLE_STEP | SPU_STATUS_INVALID_INSTR; 1123 u32 status_S_P = SPU_STATUS_SINGLE_STEP | SPU_STATUS_STOPPED_BY_STOP; 1124 u32 status_P_H = SPU_STATUS_STOPPED_BY_HALT |SPU_STATUS_STOPPED_BY_STOP; 1125 u32 status_P_I = SPU_STATUS_STOPPED_BY_STOP |SPU_STATUS_INVALID_INSTR; 1126 u32 status_code; 1127 1128 /* Restore, Step 27: 1129 * If the CSA.SPU_Status[I,S,H,P]=1 then add the correct 1130 * instruction sequence to the end of the SPU based restore 1131 * code (after the "context restored" stop and signal) to 1132 * restore the correct SPU status. 1133 * 1134 * NOTE: Rather than modifying the SPU executable, we 1135 * instead add a new 'stopped_status' field to the 1136 * LSCSA. The SPU-side restore reads this field and 1137 * takes the appropriate action when exiting. 1138 */ 1139 1140 status_code = 1141 (csa->prob.spu_status_R >> SPU_STOP_STATUS_SHIFT) & 0xFFFF; 1142 if ((csa->prob.spu_status_R & status_P_I) == status_P_I) { 1143 1144 /* SPU_Status[P,I]=1 - Illegal Instruction followed 1145 * by Stop and Signal instruction, followed by 'br -4'. 1146 * 1147 */ 1148 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P_I; 1149 csa->lscsa->stopped_status.slot[1] = status_code; 1150 1151 } else if ((csa->prob.spu_status_R & status_P_H) == status_P_H) { 1152 1153 /* SPU_Status[P,H]=1 - Halt Conditional, followed 1154 * by Stop and Signal instruction, followed by 1155 * 'br -4'. 1156 */ 1157 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P_H; 1158 csa->lscsa->stopped_status.slot[1] = status_code; 1159 1160 } else if ((csa->prob.spu_status_R & status_S_P) == status_S_P) { 1161 1162 /* SPU_Status[S,P]=1 - Stop and Signal instruction 1163 * followed by 'br -4'. 1164 */ 1165 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S_P; 1166 csa->lscsa->stopped_status.slot[1] = status_code; 1167 1168 } else if ((csa->prob.spu_status_R & status_S_I) == status_S_I) { 1169 1170 /* SPU_Status[S,I]=1 - Illegal instruction followed 1171 * by 'br -4'. 1172 */ 1173 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S_I; 1174 csa->lscsa->stopped_status.slot[1] = status_code; 1175 1176 } else if ((csa->prob.spu_status_R & status_P) == status_P) { 1177 1178 /* SPU_Status[P]=1 - Stop and Signal instruction 1179 * followed by 'br -4'. 1180 */ 1181 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P; 1182 csa->lscsa->stopped_status.slot[1] = status_code; 1183 1184 } else if ((csa->prob.spu_status_R & status_H) == status_H) { 1185 1186 /* SPU_Status[H]=1 - Halt Conditional, followed 1187 * by 'br -4'. 1188 */ 1189 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_H; 1190 1191 } else if ((csa->prob.spu_status_R & status_S) == status_S) { 1192 1193 /* SPU_Status[S]=1 - Two nop instructions. 1194 */ 1195 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S; 1196 1197 } else if ((csa->prob.spu_status_R & status_I) == status_I) { 1198 1199 /* SPU_Status[I]=1 - Illegal instruction followed 1200 * by 'br -4'. 1201 */ 1202 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_I; 1203 1204 } 1205} 1206 1207static inline void setup_spu_status_part2(struct spu_state *csa, 1208 struct spu *spu) 1209{ 1210 u32 mask; 1211 1212 /* Restore, Step 28: 1213 * If the CSA.SPU_Status[I,S,H,P,R]=0 then 1214 * add a 'br *' instruction to the end of 1215 * the SPU based restore code. 1216 * 1217 * NOTE: Rather than modifying the SPU executable, we 1218 * instead add a new 'stopped_status' field to the 1219 * LSCSA. The SPU-side restore reads this field and 1220 * takes the appropriate action when exiting. 1221 */ 1222 mask = SPU_STATUS_INVALID_INSTR | 1223 SPU_STATUS_SINGLE_STEP | 1224 SPU_STATUS_STOPPED_BY_HALT | 1225 SPU_STATUS_STOPPED_BY_STOP | SPU_STATUS_RUNNING; 1226 if (!(csa->prob.spu_status_R & mask)) { 1227 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_R; 1228 } 1229} 1230 1231static inline void restore_mfc_rag(struct spu_state *csa, struct spu *spu) 1232{ 1233 /* Restore, Step 29: 1234 * Restore RA_GROUP_ID register and the 1235 * RA_ENABLE reigster from the CSA. 1236 */ 1237 spu_resource_allocation_groupID_set(spu, 1238 csa->priv1.resource_allocation_groupID_RW); 1239 spu_resource_allocation_enable_set(spu, 1240 csa->priv1.resource_allocation_enable_RW); 1241} 1242 1243static inline void send_restore_code(struct spu_state *csa, struct spu *spu) 1244{ 1245 unsigned long addr = (unsigned long)&spu_restore_code[0]; 1246 unsigned int ls_offset = 0x0; 1247 unsigned int size = sizeof(spu_restore_code); 1248 unsigned int tag = 0; 1249 unsigned int rclass = 0; 1250 unsigned int cmd = MFC_GETFS_CMD; 1251 1252 /* Restore, Step 37: 1253 * Issue MFC DMA command to copy context 1254 * restore code to local storage. 1255 */ 1256 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 1257} 1258 1259static inline void setup_decr(struct spu_state *csa, struct spu *spu) 1260{ 1261 /* Restore, Step 34: 1262 * If CSA.MFC_CNTL[Ds]=1 (decrementer was 1263 * running) then adjust decrementer, set 1264 * decrementer running status in LSCSA, 1265 * and set decrementer "wrapped" status 1266 * in LSCSA. 1267 */ 1268 if (csa->priv2.mfc_control_RW & MFC_CNTL_DECREMENTER_RUNNING) { 1269 cycles_t resume_time = get_cycles(); 1270 cycles_t delta_time = resume_time - csa->suspend_time; 1271 1272 csa->lscsa->decr.slot[0] -= delta_time; 1273 } 1274} 1275 1276static inline void setup_ppu_mb(struct spu_state *csa, struct spu *spu) 1277{ 1278 /* Restore, Step 35: 1279 * Copy the CSA.PU_MB data into the LSCSA. 1280 */ 1281 csa->lscsa->ppu_mb.slot[0] = csa->prob.pu_mb_R; 1282} 1283 1284static inline void setup_ppuint_mb(struct spu_state *csa, struct spu *spu) 1285{ 1286 /* Restore, Step 36: 1287 * Copy the CSA.PUINT_MB data into the LSCSA. 1288 */ 1289 csa->lscsa->ppuint_mb.slot[0] = csa->priv2.puint_mb_R; 1290} 1291 1292static inline int check_restore_status(struct spu_state *csa, struct spu *spu) 1293{ 1294 struct spu_problem __iomem *prob = spu->problem; 1295 u32 complete; 1296 1297 /* Restore, Step 40: 1298 * If SPU_Status[P]=1 and SPU_Status[SC] = "success", 1299 * context restore succeeded, otherwise context restore 1300 * failed. 1301 */ 1302 complete = ((SPU_RESTORE_COMPLETE << SPU_STOP_STATUS_SHIFT) | 1303 SPU_STATUS_STOPPED_BY_STOP); 1304 return (in_be32(&prob->spu_status_R) != complete) ? 1 : 0; 1305} 1306 1307static inline void restore_spu_privcntl(struct spu_state *csa, struct spu *spu) 1308{ 1309 struct spu_priv2 __iomem *priv2 = spu->priv2; 1310 1311 /* Restore, Step 41: 1312 * Restore SPU_PrivCntl from the CSA. 1313 */ 1314 out_be64(&priv2->spu_privcntl_RW, csa->priv2.spu_privcntl_RW); 1315 eieio(); 1316} 1317 1318static inline void restore_status_part1(struct spu_state *csa, struct spu *spu) 1319{ 1320 struct spu_problem __iomem *prob = spu->problem; 1321 u32 mask; 1322 1323 /* Restore, Step 42: 1324 * If any CSA.SPU_Status[I,S,H,P]=1, then 1325 * restore the error or single step state. 1326 */ 1327 mask = SPU_STATUS_INVALID_INSTR | 1328 SPU_STATUS_SINGLE_STEP | 1329 SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP; 1330 if (csa->prob.spu_status_R & mask) { 1331 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1332 eieio(); 1333 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1334 SPU_STATUS_RUNNING); 1335 } 1336} 1337 1338static inline void restore_status_part2(struct spu_state *csa, struct spu *spu) 1339{ 1340 struct spu_problem __iomem *prob = spu->problem; 1341 u32 mask; 1342 1343 /* Restore, Step 43: 1344 * If all CSA.SPU_Status[I,S,H,P,R]=0 then write 1345 * SPU_RunCntl[R0R1]='01', wait for SPU_Status[R]=1, 1346 * then write '00' to SPU_RunCntl[R0R1] and wait 1347 * for SPU_Status[R]=0. 1348 */ 1349 mask = SPU_STATUS_INVALID_INSTR | 1350 SPU_STATUS_SINGLE_STEP | 1351 SPU_STATUS_STOPPED_BY_HALT | 1352 SPU_STATUS_STOPPED_BY_STOP | SPU_STATUS_RUNNING; 1353 if (!(csa->prob.spu_status_R & mask)) { 1354 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1355 eieio(); 1356 POLL_WHILE_FALSE(in_be32(&prob->spu_status_R) & 1357 SPU_STATUS_RUNNING); 1358 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 1359 eieio(); 1360 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1361 SPU_STATUS_RUNNING); 1362 } 1363} 1364 1365static inline void restore_ls_16kb(struct spu_state *csa, struct spu *spu) 1366{ 1367 unsigned long addr = (unsigned long)&csa->lscsa->ls[0]; 1368 unsigned int ls_offset = 0x0; 1369 unsigned int size = 16384; 1370 unsigned int tag = 0; 1371 unsigned int rclass = 0; 1372 unsigned int cmd = MFC_GET_CMD; 1373 1374 /* Restore, Step 44: 1375 * Issue a DMA command to restore the first 1376 * 16kb of local storage from CSA. 1377 */ 1378 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 1379} 1380 1381static inline void clear_interrupts(struct spu_state *csa, struct spu *spu) 1382{ 1383 /* Restore, Step 49: 1384 * Write INT_MASK_class0 with value of 0. 1385 * Write INT_MASK_class1 with value of 0. 1386 * Write INT_MASK_class2 with value of 0. 1387 * Write INT_STAT_class0 with value of -1. 1388 * Write INT_STAT_class1 with value of -1. 1389 * Write INT_STAT_class2 with value of -1. 1390 */ 1391 spin_lock_irq(&spu->register_lock); 1392 spu_int_mask_set(spu, 0, 0ul); 1393 spu_int_mask_set(spu, 1, 0ul); 1394 spu_int_mask_set(spu, 2, 0ul); 1395 spu_int_stat_clear(spu, 0, ~0ul); 1396 spu_int_stat_clear(spu, 1, ~0ul); 1397 spu_int_stat_clear(spu, 2, ~0ul); 1398 spin_unlock_irq(&spu->register_lock); 1399} 1400 1401static inline void restore_mfc_queues(struct spu_state *csa, struct spu *spu) 1402{ 1403 struct spu_priv2 __iomem *priv2 = spu->priv2; 1404 int i; 1405 1406 /* Restore, Step 50: 1407 * If MFC_Cntl[Se]!=0 then restore 1408 * MFC command queues. 1409 */ 1410 if ((csa->priv2.mfc_control_RW & MFC_CNTL_DMA_QUEUES_EMPTY_MASK) == 0) { 1411 for (i = 0; i < 8; i++) { 1412 out_be64(&priv2->puq[i].mfc_cq_data0_RW, 1413 csa->priv2.puq[i].mfc_cq_data0_RW); 1414 out_be64(&priv2->puq[i].mfc_cq_data1_RW, 1415 csa->priv2.puq[i].mfc_cq_data1_RW); 1416 out_be64(&priv2->puq[i].mfc_cq_data2_RW, 1417 csa->priv2.puq[i].mfc_cq_data2_RW); 1418 out_be64(&priv2->puq[i].mfc_cq_data3_RW, 1419 csa->priv2.puq[i].mfc_cq_data3_RW); 1420 } 1421 for (i = 0; i < 16; i++) { 1422 out_be64(&priv2->spuq[i].mfc_cq_data0_RW, 1423 csa->priv2.spuq[i].mfc_cq_data0_RW); 1424 out_be64(&priv2->spuq[i].mfc_cq_data1_RW, 1425 csa->priv2.spuq[i].mfc_cq_data1_RW); 1426 out_be64(&priv2->spuq[i].mfc_cq_data2_RW, 1427 csa->priv2.spuq[i].mfc_cq_data2_RW); 1428 out_be64(&priv2->spuq[i].mfc_cq_data3_RW, 1429 csa->priv2.spuq[i].mfc_cq_data3_RW); 1430 } 1431 } 1432 eieio(); 1433} 1434 1435static inline void restore_ppu_querymask(struct spu_state *csa, struct spu *spu) 1436{ 1437 struct spu_problem __iomem *prob = spu->problem; 1438 1439 /* Restore, Step 51: 1440 * Restore the PPU_QueryMask register from CSA. 1441 */ 1442 out_be32(&prob->dma_querymask_RW, csa->prob.dma_querymask_RW); 1443 eieio(); 1444} 1445 1446static inline void restore_ppu_querytype(struct spu_state *csa, struct spu *spu) 1447{ 1448 struct spu_problem __iomem *prob = spu->problem; 1449 1450 /* Restore, Step 52: 1451 * Restore the PPU_QueryType register from CSA. 1452 */ 1453 out_be32(&prob->dma_querytype_RW, csa->prob.dma_querytype_RW); 1454 eieio(); 1455} 1456 1457static inline void restore_mfc_csr_tsq(struct spu_state *csa, struct spu *spu) 1458{ 1459 struct spu_priv2 __iomem *priv2 = spu->priv2; 1460 1461 /* Restore, Step 53: 1462 * Restore the MFC_CSR_TSQ register from CSA. 1463 */ 1464 out_be64(&priv2->spu_tag_status_query_RW, 1465 csa->priv2.spu_tag_status_query_RW); 1466 eieio(); 1467} 1468 1469static inline void restore_mfc_csr_cmd(struct spu_state *csa, struct spu *spu) 1470{ 1471 struct spu_priv2 __iomem *priv2 = spu->priv2; 1472 1473 /* Restore, Step 54: 1474 * Restore the MFC_CSR_CMD1 and MFC_CSR_CMD2 1475 * registers from CSA. 1476 */ 1477 out_be64(&priv2->spu_cmd_buf1_RW, csa->priv2.spu_cmd_buf1_RW); 1478 out_be64(&priv2->spu_cmd_buf2_RW, csa->priv2.spu_cmd_buf2_RW); 1479 eieio(); 1480} 1481 1482static inline void restore_mfc_csr_ato(struct spu_state *csa, struct spu *spu) 1483{ 1484 struct spu_priv2 __iomem *priv2 = spu->priv2; 1485 1486 /* Restore, Step 55: 1487 * Restore the MFC_CSR_ATO register from CSA. 1488 */ 1489 out_be64(&priv2->spu_atomic_status_RW, csa->priv2.spu_atomic_status_RW); 1490} 1491 1492static inline void restore_mfc_tclass_id(struct spu_state *csa, struct spu *spu) 1493{ 1494 /* Restore, Step 56: 1495 * Restore the MFC_TCLASS_ID register from CSA. 1496 */ 1497 spu_mfc_tclass_id_set(spu, csa->priv1.mfc_tclass_id_RW); 1498 eieio(); 1499} 1500 1501static inline void set_llr_event(struct spu_state *csa, struct spu *spu) 1502{ 1503 u64 ch0_cnt, ch0_data; 1504 u64 ch1_data; 1505 1506 /* Restore, Step 57: 1507 * Set the Lock Line Reservation Lost Event by: 1508 * 1. OR CSA.SPU_Event_Status with bit 21 (Lr) set to 1. 1509 * 2. If CSA.SPU_Channel_0_Count=0 and 1510 * CSA.SPU_Wr_Event_Mask[Lr]=1 and 1511 * CSA.SPU_Event_Status[Lr]=0 then set 1512 * CSA.SPU_Event_Status_Count=1. 1513 */ 1514 ch0_cnt = csa->spu_chnlcnt_RW[0]; 1515 ch0_data = csa->spu_chnldata_RW[0]; 1516 ch1_data = csa->spu_chnldata_RW[1]; 1517 csa->spu_chnldata_RW[0] |= MFC_LLR_LOST_EVENT; 1518 if ((ch0_cnt == 0) && !(ch0_data & MFC_LLR_LOST_EVENT) && 1519 (ch1_data & MFC_LLR_LOST_EVENT)) { 1520 csa->spu_chnlcnt_RW[0] = 1; 1521 } 1522} 1523 1524static inline void restore_decr_wrapped(struct spu_state *csa, struct spu *spu) 1525{ 1526 /* Restore, Step 58: 1527 * If the status of the CSA software decrementer 1528 * "wrapped" flag is set, OR in a '1' to 1529 * CSA.SPU_Event_Status[Tm]. 1530 */ 1531 if (csa->lscsa->decr_status.slot[0] == 1) { 1532 csa->spu_chnldata_RW[0] |= 0x20; 1533 } 1534 if ((csa->lscsa->decr_status.slot[0] == 1) && 1535 (csa->spu_chnlcnt_RW[0] == 0 && 1536 ((csa->spu_chnldata_RW[2] & 0x20) == 0x0) && 1537 ((csa->spu_chnldata_RW[0] & 0x20) != 0x1))) { 1538 csa->spu_chnlcnt_RW[0] = 1; 1539 } 1540} 1541 1542static inline void restore_ch_part1(struct spu_state *csa, struct spu *spu) 1543{ 1544 struct spu_priv2 __iomem *priv2 = spu->priv2; 1545 u64 idx, ch_indices[7] = { 0UL, 3UL, 4UL, 24UL, 25UL, 27UL }; 1546 int i; 1547 1548 /* Restore, Step 59: 1549 */ 1550 1551 /* Restore CH 1 without count */ 1552 out_be64(&priv2->spu_chnlcntptr_RW, 1); 1553 out_be64(&priv2->spu_chnldata_RW, csa->spu_chnldata_RW[1]); 1554 1555 /* Restore the following CH: [0,3,4,24,25,27] */ 1556 for (i = 0; i < 7; i++) { 1557 idx = ch_indices[i]; 1558 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1559 eieio(); 1560 out_be64(&priv2->spu_chnldata_RW, csa->spu_chnldata_RW[idx]); 1561 out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[idx]); 1562 eieio(); 1563 } 1564} 1565 1566static inline void restore_ch_part2(struct spu_state *csa, struct spu *spu) 1567{ 1568 struct spu_priv2 __iomem *priv2 = spu->priv2; 1569 u64 ch_indices[3] = { 9UL, 21UL, 23UL }; 1570 u64 ch_counts[3] = { 1UL, 16UL, 1UL }; 1571 u64 idx; 1572 int i; 1573 1574 /* Restore, Step 60: 1575 * Restore the following CH: [9,21,23]. 1576 */ 1577 ch_counts[0] = 1UL; 1578 ch_counts[1] = csa->spu_chnlcnt_RW[21]; 1579 ch_counts[2] = 1UL; 1580 for (i = 0; i < 3; i++) { 1581 idx = ch_indices[i]; 1582 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1583 eieio(); 1584 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]); 1585 eieio(); 1586 } 1587} 1588 1589static inline void restore_spu_lslr(struct spu_state *csa, struct spu *spu) 1590{ 1591 struct spu_priv2 __iomem *priv2 = spu->priv2; 1592 1593 /* Restore, Step 61: 1594 * Restore the SPU_LSLR register from CSA. 1595 */ 1596 out_be64(&priv2->spu_lslr_RW, csa->priv2.spu_lslr_RW); 1597 eieio(); 1598} 1599 1600static inline void restore_spu_cfg(struct spu_state *csa, struct spu *spu) 1601{ 1602 struct spu_priv2 __iomem *priv2 = spu->priv2; 1603 1604 /* Restore, Step 62: 1605 * Restore the SPU_Cfg register from CSA. 1606 */ 1607 out_be64(&priv2->spu_cfg_RW, csa->priv2.spu_cfg_RW); 1608 eieio(); 1609} 1610 1611static inline void restore_pm_trace(struct spu_state *csa, struct spu *spu) 1612{ 1613 /* Restore, Step 63: 1614 * Restore PM_Trace_Tag_Wait_Mask from CSA. 1615 * Not performed by this implementation. 1616 */ 1617} 1618 1619static inline void restore_spu_npc(struct spu_state *csa, struct spu *spu) 1620{ 1621 struct spu_problem __iomem *prob = spu->problem; 1622 1623 /* Restore, Step 64: 1624 * Restore SPU_NPC from CSA. 1625 */ 1626 out_be32(&prob->spu_npc_RW, csa->prob.spu_npc_RW); 1627 eieio(); 1628} 1629 1630static inline void restore_spu_mb(struct spu_state *csa, struct spu *spu) 1631{ 1632 struct spu_priv2 __iomem *priv2 = spu->priv2; 1633 int i; 1634 1635 /* Restore, Step 65: 1636 * Restore MFC_RdSPU_MB from CSA. 1637 */ 1638 out_be64(&priv2->spu_chnlcntptr_RW, 29UL); 1639 eieio(); 1640 out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[29]); 1641 for (i = 0; i < 4; i++) { 1642 out_be64(&priv2->spu_chnldata_RW, csa->spu_mailbox_data[i]); 1643 } 1644 eieio(); 1645} 1646 1647static inline void check_ppu_mb_stat(struct spu_state *csa, struct spu *spu) 1648{ 1649 struct spu_problem __iomem *prob = spu->problem; 1650 u32 dummy = 0; 1651 1652 /* Restore, Step 66: 1653 * If CSA.MB_Stat[P]=0 (mailbox empty) then 1654 * read from the PPU_MB register. 1655 */ 1656 if ((csa->prob.mb_stat_R & 0xFF) == 0) { 1657 dummy = in_be32(&prob->pu_mb_R); 1658 eieio(); 1659 } 1660} 1661 1662static inline void check_ppuint_mb_stat(struct spu_state *csa, struct spu *spu) 1663{ 1664 struct spu_priv2 __iomem *priv2 = spu->priv2; 1665 u64 dummy = 0UL; 1666 1667 /* Restore, Step 66: 1668 * If CSA.MB_Stat[I]=0 (mailbox empty) then 1669 * read from the PPUINT_MB register. 1670 */ 1671 if ((csa->prob.mb_stat_R & 0xFF0000) == 0) { 1672 dummy = in_be64(&priv2->puint_mb_R); 1673 eieio(); 1674 spu_int_stat_clear(spu, 2, CLASS2_ENABLE_MAILBOX_INTR); 1675 eieio(); 1676 } 1677} 1678 1679static inline void restore_mfc_sr1(struct spu_state *csa, struct spu *spu) 1680{ 1681 /* Restore, Step 69: 1682 * Restore the MFC_SR1 register from CSA. 1683 */ 1684 spu_mfc_sr1_set(spu, csa->priv1.mfc_sr1_RW); 1685 eieio(); 1686} 1687 1688static inline void restore_other_spu_access(struct spu_state *csa, 1689 struct spu *spu) 1690{ 1691 /* Restore, Step 70: 1692 * Restore other SPU mappings to this SPU. TBD. 1693 */ 1694} 1695 1696static inline void restore_spu_runcntl(struct spu_state *csa, struct spu *spu) 1697{ 1698 struct spu_problem __iomem *prob = spu->problem; 1699 1700 /* Restore, Step 71: 1701 * If CSA.SPU_Status[R]=1 then write 1702 * SPU_RunCntl[R0R1]='01'. 1703 */ 1704 if (csa->prob.spu_status_R & SPU_STATUS_RUNNING) { 1705 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1706 eieio(); 1707 } 1708} 1709 1710static inline void restore_mfc_cntl(struct spu_state *csa, struct spu *spu) 1711{ 1712 struct spu_priv2 __iomem *priv2 = spu->priv2; 1713 1714 /* Restore, Step 72: 1715 * Restore the MFC_CNTL register for the CSA. 1716 */ 1717 out_be64(&priv2->mfc_control_RW, csa->priv2.mfc_control_RW); 1718 eieio(); 1719 if ((csa->priv2.mfc_control_RW & MFC_CNTL_SUSPEND_DMA_QUEUE_MASK)) { 1720 out_be64(&priv2->mfc_control_RW, MFC_CNTL_RESTART_DMA_COMMAND); 1721 eieio(); 1722 } 1723} 1724 1725static inline void enable_user_access(struct spu_state *csa, struct spu *spu) 1726{ 1727 /* Restore, Step 73: 1728 * Enable user-space access (if provided) to this 1729 * SPU by mapping the virtual pages assigned to 1730 * the SPU memory-mapped I/O (MMIO) for problem 1731 * state. TBD. 1732 */ 1733} 1734 1735static inline void reset_switch_active(struct spu_state *csa, struct spu *spu) 1736{ 1737 /* Restore, Step 74: 1738 * Reset the "context switch active" flag. 1739 */ 1740 clear_bit(SPU_CONTEXT_SWITCH_ACTIVE, &spu->flags); 1741 mb(); 1742} 1743 1744static inline void reenable_interrupts(struct spu_state *csa, struct spu *spu) 1745{ 1746 /* Restore, Step 75: 1747 * Re-enable SPU interrupts. 1748 */ 1749 spin_lock_irq(&spu->register_lock); 1750 spu_int_mask_set(spu, 0, csa->priv1.int_mask_class0_RW); 1751 spu_int_mask_set(spu, 1, csa->priv1.int_mask_class1_RW); 1752 spu_int_mask_set(spu, 2, csa->priv1.int_mask_class2_RW); 1753 spin_unlock_irq(&spu->register_lock); 1754} 1755 1756static int quiece_spu(struct spu_state *prev, struct spu *spu) 1757{ 1758 /* 1759 * Combined steps 2-18 of SPU context save sequence, which 1760 * quiesce the SPU state (disable SPU execution, MFC command 1761 * queues, decrementer, SPU interrupts, etc.). 1762 * 1763 * Returns 0 on success. 1764 * 2 if failed step 2. 1765 * 6 if failed step 6. 1766 */ 1767 1768 if (check_spu_isolate(prev, spu)) { /* Step 2. */ 1769 return 2; 1770 } 1771 disable_interrupts(prev, spu); /* Step 3. */ 1772 set_watchdog_timer(prev, spu); /* Step 4. */ 1773 inhibit_user_access(prev, spu); /* Step 5. */ 1774 if (check_spu_isolate(prev, spu)) { /* Step 6. */ 1775 return 6; 1776 } 1777 set_switch_pending(prev, spu); /* Step 7. */ 1778 save_mfc_cntl(prev, spu); /* Step 8. */ 1779 save_spu_runcntl(prev, spu); /* Step 9. */ 1780 save_mfc_sr1(prev, spu); /* Step 10. */ 1781 save_spu_status(prev, spu); /* Step 11. */ 1782 save_mfc_decr(prev, spu); /* Step 12. */ 1783 halt_mfc_decr(prev, spu); /* Step 13. */ 1784 save_timebase(prev, spu); /* Step 14. */ 1785 remove_other_spu_access(prev, spu); /* Step 15. */ 1786 do_mfc_mssync(prev, spu); /* Step 16. */ 1787 issue_mfc_tlbie(prev, spu); /* Step 17. */ 1788 handle_pending_interrupts(prev, spu); /* Step 18. */ 1789 1790 return 0; 1791} 1792 1793static void save_csa(struct spu_state *prev, struct spu *spu) 1794{ 1795 /* 1796 * Combine steps 19-44 of SPU context save sequence, which 1797 * save regions of the privileged & problem state areas. 1798 */ 1799 1800 save_mfc_queues(prev, spu); /* Step 19. */ 1801 save_ppu_querymask(prev, spu); /* Step 20. */ 1802 save_ppu_querytype(prev, spu); /* Step 21. */ 1803 save_mfc_csr_tsq(prev, spu); /* Step 22. */ 1804 save_mfc_csr_cmd(prev, spu); /* Step 23. */ 1805 save_mfc_csr_ato(prev, spu); /* Step 24. */ 1806 save_mfc_tclass_id(prev, spu); /* Step 25. */ 1807 set_mfc_tclass_id(prev, spu); /* Step 26. */ 1808 purge_mfc_queue(prev, spu); /* Step 27. */ 1809 wait_purge_complete(prev, spu); /* Step 28. */ 1810 setup_mfc_sr1(prev, spu); /* Step 30. */ 1811 save_spu_npc(prev, spu); /* Step 31. */ 1812 save_spu_privcntl(prev, spu); /* Step 32. */ 1813 reset_spu_privcntl(prev, spu); /* Step 33. */ 1814 save_spu_lslr(prev, spu); /* Step 34. */ 1815 reset_spu_lslr(prev, spu); /* Step 35. */ 1816 save_spu_cfg(prev, spu); /* Step 36. */ 1817 save_pm_trace(prev, spu); /* Step 37. */ 1818 save_mfc_rag(prev, spu); /* Step 38. */ 1819 save_ppu_mb_stat(prev, spu); /* Step 39. */ 1820 save_ppu_mb(prev, spu); /* Step 40. */ 1821 save_ppuint_mb(prev, spu); /* Step 41. */ 1822 save_ch_part1(prev, spu); /* Step 42. */ 1823 save_spu_mb(prev, spu); /* Step 43. */ 1824 save_mfc_cmd(prev, spu); /* Step 44. */ 1825 reset_ch(prev, spu); /* Step 45. */ 1826} 1827 1828static void save_lscsa(struct spu_state *prev, struct spu *spu) 1829{ 1830 /* 1831 * Perform steps 46-57 of SPU context save sequence, 1832 * which save regions of the local store and register 1833 * file. 1834 */ 1835 1836 resume_mfc_queue(prev, spu); /* Step 46. */ 1837 setup_mfc_slbs(prev, spu); /* Step 47. */ 1838 set_switch_active(prev, spu); /* Step 48. */ 1839 enable_interrupts(prev, spu); /* Step 49. */ 1840 save_ls_16kb(prev, spu); /* Step 50. */ 1841 set_spu_npc(prev, spu); /* Step 51. */ 1842 set_signot1(prev, spu); /* Step 52. */ 1843 set_signot2(prev, spu); /* Step 53. */ 1844 send_save_code(prev, spu); /* Step 54. */ 1845 set_ppu_querymask(prev, spu); /* Step 55. */ 1846 wait_tag_complete(prev, spu); /* Step 56. */ 1847 wait_spu_stopped(prev, spu); /* Step 57. */ 1848} 1849 1850static void force_spu_isolate_exit(struct spu *spu) 1851{ 1852 struct spu_problem __iomem *prob = spu->problem; 1853 struct spu_priv2 __iomem *priv2 = spu->priv2; 1854 1855 /* Stop SPE execution and wait for completion. */ 1856 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 1857 iobarrier_rw(); 1858 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING); 1859 1860 /* Restart SPE master runcntl. */ 1861 spu_mfc_sr1_set(spu, MFC_STATE1_MASTER_RUN_CONTROL_MASK); 1862 iobarrier_w(); 1863 1864 /* Initiate isolate exit request and wait for completion. */ 1865 out_be64(&priv2->spu_privcntl_RW, 4LL); 1866 iobarrier_w(); 1867 out_be32(&prob->spu_runcntl_RW, 2); 1868 iobarrier_rw(); 1869 POLL_WHILE_FALSE((in_be32(&prob->spu_status_R) 1870 & SPU_STATUS_STOPPED_BY_STOP)); 1871 1872 /* Reset load request to normal. */ 1873 out_be64(&priv2->spu_privcntl_RW, SPU_PRIVCNT_LOAD_REQUEST_NORMAL); 1874 iobarrier_w(); 1875} 1876 1877/** 1878 * stop_spu_isolate 1879 * Check SPU run-control state and force isolated 1880 * exit function as necessary. 1881 */ 1882static void stop_spu_isolate(struct spu *spu) 1883{ 1884 struct spu_problem __iomem *prob = spu->problem; 1885 1886 if (in_be32(&prob->spu_status_R) & SPU_STATUS_ISOLATED_STATE) { 1887 /* The SPU is in isolated state; the only way 1888 * to get it out is to perform an isolated 1889 * exit (clean) operation. 1890 */ 1891 force_spu_isolate_exit(spu); 1892 } 1893} 1894 1895static void harvest(struct spu_state *prev, struct spu *spu) 1896{ 1897 /* 1898 * Perform steps 2-25 of SPU context restore sequence, 1899 * which resets an SPU either after a failed save, or 1900 * when using SPU for first time. 1901 */ 1902 1903 disable_interrupts(prev, spu); /* Step 2. */ 1904 inhibit_user_access(prev, spu); /* Step 3. */ 1905 terminate_spu_app(prev, spu); /* Step 4. */ 1906 set_switch_pending(prev, spu); /* Step 5. */ 1907 stop_spu_isolate(spu); /* NEW. */ 1908 remove_other_spu_access(prev, spu); /* Step 6. */ 1909 suspend_mfc(prev, spu); /* Step 7. */ 1910 wait_suspend_mfc_complete(prev, spu); /* Step 8. */ 1911 if (!suspend_spe(prev, spu)) /* Step 9. */ 1912 clear_spu_status(prev, spu); /* Step 10. */ 1913 do_mfc_mssync(prev, spu); /* Step 11. */ 1914 issue_mfc_tlbie(prev, spu); /* Step 12. */ 1915 handle_pending_interrupts(prev, spu); /* Step 13. */ 1916 purge_mfc_queue(prev, spu); /* Step 14. */ 1917 wait_purge_complete(prev, spu); /* Step 15. */ 1918 reset_spu_privcntl(prev, spu); /* Step 16. */ 1919 reset_spu_lslr(prev, spu); /* Step 17. */ 1920 setup_mfc_sr1(prev, spu); /* Step 18. */ 1921 spu_invalidate_slbs(spu); /* Step 19. */ 1922 reset_ch_part1(prev, spu); /* Step 20. */ 1923 reset_ch_part2(prev, spu); /* Step 21. */ 1924 enable_interrupts(prev, spu); /* Step 22. */ 1925 set_switch_active(prev, spu); /* Step 23. */ 1926 set_mfc_tclass_id(prev, spu); /* Step 24. */ 1927 resume_mfc_queue(prev, spu); /* Step 25. */ 1928} 1929 1930static void restore_lscsa(struct spu_state *next, struct spu *spu) 1931{ 1932 /* 1933 * Perform steps 26-40 of SPU context restore sequence, 1934 * which restores regions of the local store and register 1935 * file. 1936 */ 1937 1938 set_watchdog_timer(next, spu); /* Step 26. */ 1939 setup_spu_status_part1(next, spu); /* Step 27. */ 1940 setup_spu_status_part2(next, spu); /* Step 28. */ 1941 restore_mfc_rag(next, spu); /* Step 29. */ 1942 setup_mfc_slbs(next, spu); /* Step 30. */ 1943 set_spu_npc(next, spu); /* Step 31. */ 1944 set_signot1(next, spu); /* Step 32. */ 1945 set_signot2(next, spu); /* Step 33. */ 1946 setup_decr(next, spu); /* Step 34. */ 1947 setup_ppu_mb(next, spu); /* Step 35. */ 1948 setup_ppuint_mb(next, spu); /* Step 36. */ 1949 send_restore_code(next, spu); /* Step 37. */ 1950 set_ppu_querymask(next, spu); /* Step 38. */ 1951 wait_tag_complete(next, spu); /* Step 39. */ 1952 wait_spu_stopped(next, spu); /* Step 40. */ 1953} 1954 1955static void restore_csa(struct spu_state *next, struct spu *spu) 1956{ 1957 /* 1958 * Combine steps 41-76 of SPU context restore sequence, which 1959 * restore regions of the privileged & problem state areas. 1960 */ 1961 1962 restore_spu_privcntl(next, spu); /* Step 41. */ 1963 restore_status_part1(next, spu); /* Step 42. */ 1964 restore_status_part2(next, spu); /* Step 43. */ 1965 restore_ls_16kb(next, spu); /* Step 44. */ 1966 wait_tag_complete(next, spu); /* Step 45. */ 1967 suspend_mfc(next, spu); /* Step 46. */ 1968 wait_suspend_mfc_complete(next, spu); /* Step 47. */ 1969 issue_mfc_tlbie(next, spu); /* Step 48. */ 1970 clear_interrupts(next, spu); /* Step 49. */ 1971 restore_mfc_queues(next, spu); /* Step 50. */ 1972 restore_ppu_querymask(next, spu); /* Step 51. */ 1973 restore_ppu_querytype(next, spu); /* Step 52. */ 1974 restore_mfc_csr_tsq(next, spu); /* Step 53. */ 1975 restore_mfc_csr_cmd(next, spu); /* Step 54. */ 1976 restore_mfc_csr_ato(next, spu); /* Step 55. */ 1977 restore_mfc_tclass_id(next, spu); /* Step 56. */ 1978 set_llr_event(next, spu); /* Step 57. */ 1979 restore_decr_wrapped(next, spu); /* Step 58. */ 1980 restore_ch_part1(next, spu); /* Step 59. */ 1981 restore_ch_part2(next, spu); /* Step 60. */ 1982 restore_spu_lslr(next, spu); /* Step 61. */ 1983 restore_spu_cfg(next, spu); /* Step 62. */ 1984 restore_pm_trace(next, spu); /* Step 63. */ 1985 restore_spu_npc(next, spu); /* Step 64. */ 1986 restore_spu_mb(next, spu); /* Step 65. */ 1987 check_ppu_mb_stat(next, spu); /* Step 66. */ 1988 check_ppuint_mb_stat(next, spu); /* Step 67. */ 1989 spu_invalidate_slbs(spu); /* Modified Step 68. */ 1990 restore_mfc_sr1(next, spu); /* Step 69. */ 1991 restore_other_spu_access(next, spu); /* Step 70. */ 1992 restore_spu_runcntl(next, spu); /* Step 71. */ 1993 restore_mfc_cntl(next, spu); /* Step 72. */ 1994 enable_user_access(next, spu); /* Step 73. */ 1995 reset_switch_active(next, spu); /* Step 74. */ 1996 reenable_interrupts(next, spu); /* Step 75. */ 1997} 1998 1999static int __do_spu_save(struct spu_state *prev, struct spu *spu) 2000{ 2001 int rc; 2002 2003 /* 2004 * SPU context save can be broken into three phases: 2005 * 2006 * (a) quiesce [steps 2-16]. 2007 * (b) save of CSA, performed by PPE [steps 17-42] 2008 * (c) save of LSCSA, mostly performed by SPU [steps 43-52]. 2009 * 2010 * Returns 0 on success. 2011 * 2,6 if failed to quiece SPU 2012 * 53 if SPU-side of save failed. 2013 */ 2014 2015 rc = quiece_spu(prev, spu); /* Steps 2-16. */ 2016 switch (rc) { 2017 default: 2018 case 2: 2019 case 6: 2020 harvest(prev, spu); 2021 return rc; 2022 break; 2023 case 0: 2024 break; 2025 } 2026 save_csa(prev, spu); /* Steps 17-43. */ 2027 save_lscsa(prev, spu); /* Steps 44-53. */ 2028 return check_save_status(prev, spu); /* Step 54. */ 2029} 2030 2031static int __do_spu_restore(struct spu_state *next, struct spu *spu) 2032{ 2033 int rc; 2034 2035 /* 2036 * SPU context restore can be broken into three phases: 2037 * 2038 * (a) harvest (or reset) SPU [steps 2-24]. 2039 * (b) restore LSCSA [steps 25-40], mostly performed by SPU. 2040 * (c) restore CSA [steps 41-76], performed by PPE. 2041 * 2042 * The 'harvest' step is not performed here, but rather 2043 * as needed below. 2044 */ 2045 2046 restore_lscsa(next, spu); /* Steps 24-39. */ 2047 rc = check_restore_status(next, spu); /* Step 40. */ 2048 switch (rc) { 2049 default: 2050 /* Failed. Return now. */ 2051 return rc; 2052 break; 2053 case 0: 2054 /* Fall through to next step. */ 2055 break; 2056 } 2057 restore_csa(next, spu); 2058 2059 return 0; 2060} 2061 2062/** 2063 * spu_save - SPU context save, with locking. 2064 * @prev: pointer to SPU context save area, to be saved. 2065 * @spu: pointer to SPU iomem structure. 2066 * 2067 * Acquire locks, perform the save operation then return. 2068 */ 2069int spu_save(struct spu_state *prev, struct spu *spu) 2070{ 2071 int rc; 2072 2073 acquire_spu_lock(spu); /* Step 1. */ 2074 prev->dar = spu->dar; 2075 prev->dsisr = spu->dsisr; 2076 spu->dar = 0; 2077 spu->dsisr = 0; 2078 rc = __do_spu_save(prev, spu); /* Steps 2-53. */ 2079 release_spu_lock(spu); 2080 if (rc != 0 && rc != 2 && rc != 6) { 2081 panic("%s failed on SPU[%d], rc=%d.\n", 2082 __func__, spu->number, rc); 2083 } 2084 return 0; 2085} 2086EXPORT_SYMBOL_GPL(spu_save); 2087 2088/** 2089 * spu_restore - SPU context restore, with harvest and locking. 2090 * @new: pointer to SPU context save area, to be restored. 2091 * @spu: pointer to SPU iomem structure. 2092 * 2093 * Perform harvest + restore, as we may not be coming 2094 * from a previous successful save operation, and the 2095 * hardware state is unknown. 2096 */ 2097int spu_restore(struct spu_state *new, struct spu *spu) 2098{ 2099 int rc; 2100 2101 acquire_spu_lock(spu); 2102 harvest(NULL, spu); 2103 spu->slb_replace = 0; 2104 new->dar = 0; 2105 new->dsisr = 0; 2106 spu->class_0_pending = 0; 2107 rc = __do_spu_restore(new, spu); 2108 release_spu_lock(spu); 2109 if (rc) { 2110 panic("%s failed on SPU[%d] rc=%d.\n", 2111 __func__, spu->number, rc); 2112 } 2113 return rc; 2114} 2115EXPORT_SYMBOL_GPL(spu_restore); 2116 2117/** 2118 * spu_harvest - SPU harvest (reset) operation 2119 * @spu: pointer to SPU iomem structure. 2120 * 2121 * Perform SPU harvest (reset) operation. 2122 */ 2123void spu_harvest(struct spu *spu) 2124{ 2125 acquire_spu_lock(spu); 2126 harvest(NULL, spu); 2127 release_spu_lock(spu); 2128} 2129 2130static void init_prob(struct spu_state *csa) 2131{ 2132 csa->spu_chnlcnt_RW[9] = 1; 2133 csa->spu_chnlcnt_RW[21] = 16; 2134 csa->spu_chnlcnt_RW[23] = 1; 2135 csa->spu_chnlcnt_RW[28] = 1; 2136 csa->spu_chnlcnt_RW[30] = 1; 2137 csa->prob.spu_runcntl_RW = SPU_RUNCNTL_STOP; 2138 csa->prob.mb_stat_R = 0x000400; 2139} 2140 2141static void init_priv1(struct spu_state *csa) 2142{ 2143 /* Enable decode, relocate, tlbie response, master runcntl. */ 2144 csa->priv1.mfc_sr1_RW = MFC_STATE1_LOCAL_STORAGE_DECODE_MASK | 2145 MFC_STATE1_MASTER_RUN_CONTROL_MASK | 2146 MFC_STATE1_PROBLEM_STATE_MASK | 2147 MFC_STATE1_RELOCATE_MASK | MFC_STATE1_BUS_TLBIE_MASK; 2148 2149 /* Enable OS-specific set of interrupts. */ 2150 csa->priv1.int_mask_class0_RW = CLASS0_ENABLE_DMA_ALIGNMENT_INTR | 2151 CLASS0_ENABLE_INVALID_DMA_COMMAND_INTR | 2152 CLASS0_ENABLE_SPU_ERROR_INTR; 2153 csa->priv1.int_mask_class1_RW = CLASS1_ENABLE_SEGMENT_FAULT_INTR | 2154 CLASS1_ENABLE_STORAGE_FAULT_INTR; 2155 csa->priv1.int_mask_class2_RW = CLASS2_ENABLE_SPU_STOP_INTR | 2156 CLASS2_ENABLE_SPU_HALT_INTR | 2157 CLASS2_ENABLE_SPU_DMA_TAG_GROUP_COMPLETE_INTR; 2158} 2159 2160static void init_priv2(struct spu_state *csa) 2161{ 2162 csa->priv2.spu_lslr_RW = LS_ADDR_MASK; 2163 csa->priv2.mfc_control_RW = MFC_CNTL_RESUME_DMA_QUEUE | 2164 MFC_CNTL_NORMAL_DMA_QUEUE_OPERATION | 2165 MFC_CNTL_DMA_QUEUES_EMPTY_MASK; 2166} 2167 2168/** 2169 * spu_alloc_csa - allocate and initialize an SPU context save area. 2170 * 2171 * Allocate and initialize the contents of an SPU context save area. 2172 * This includes enabling address translation, interrupt masks, etc., 2173 * as appropriate for the given OS environment. 2174 * 2175 * Note that storage for the 'lscsa' is allocated separately, 2176 * as it is by far the largest of the context save regions, 2177 * and may need to be pinned or otherwise specially aligned. 2178 */ 2179int spu_init_csa(struct spu_state *csa) 2180{ 2181 int rc; 2182 2183 if (!csa) 2184 return -EINVAL; 2185 memset(csa, 0, sizeof(struct spu_state)); 2186 2187 rc = spu_alloc_lscsa(csa); 2188 if (rc) 2189 return rc; 2190 2191 spin_lock_init(&csa->register_lock); 2192 2193 init_prob(csa); 2194 init_priv1(csa); 2195 init_priv2(csa); 2196 2197 return 0; 2198} 2199EXPORT_SYMBOL_GPL(spu_init_csa); 2200 2201void spu_fini_csa(struct spu_state *csa) 2202{ 2203 spu_free_lscsa(csa); 2204} 2205EXPORT_SYMBOL_GPL(spu_fini_csa); 2206