arch/powerpc/platforms/cell/spufs/switch.c

   1 /*
   2  * spu_switch.c
   3  *
   4  * (C) Copyright IBM Corp. 2005
   5  *
   6  * Author: Mark Nutter <mnutter@us.ibm.com>
   7  *
   8  * Host-side part of SPU context switch sequence outlined in
   9  * Synergistic Processor Element, Book IV.
  10  *
  11  * A fully premptive switch of an SPE is very expensive in terms
  12  * of time and system resources.  SPE Book IV indicates that SPE
  13  * allocation should follow a "serially reusable device" model,
  14  * in which the SPE is assigned a task until it completes.  When
  15  * this is not possible, this sequence may be used to premptively
  16  * save, and then later (optionally) restore the context of a
  17  * program executing on an SPE.
  18  *
  19  *
  20  * This program is free software; you can redistribute it and/or modify
  21  * it under the terms of the GNU General Public License as published by
  22  * the Free Software Foundation; either version 2, or (at your option)
  23  * any later version.
  24  *
  25  * This program is distributed in the hope that it will be useful,
  26  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  27  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  28  * GNU General Public License for more details.
  29  *
  30  * You should have received a copy of the GNU General Public License
  31  * along with this program; if not, write to the Free Software
  32  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  33  */
  34
  35 #include <linux/config.h>
  36 #include <linux/module.h>
  37 #include <linux/errno.h>
  38 #include <linux/sched.h>
  39 #include <linux/kernel.h>
  40 #include <linux/mm.h>
  41 #include <linux/vmalloc.h>
  42 #include <linux/smp.h>
  43 #include <linux/smp_lock.h>
  44 #include <linux/stddef.h>
  45 #include <linux/unistd.h>
  46
  47 #include <asm/io.h>
  48 #include <asm/spu.h>
  49 #include <asm/spu_csa.h>
  50 #include <asm/mmu_context.h>
  51
  52 #include "spu_save_dump.h"
  53 #include "spu_restore_dump.h"
  54
  55 #if 0
  56 #define POLL_WHILE_TRUE(_c) {                           \
  57     do {                                                \
  58     } while (_c);                                       \
  59   }
  60 #else
  61 #define RELAX_SPIN_COUNT                                1000
  62 #define POLL_WHILE_TRUE(_c) {                           \
  63     do {                                                \
  64         int _i;                                         \
  65         for (_i=0; _i<RELAX_SPIN_COUNT && (_c); _i++) { \
  66             cpu_relax();                                \
  67         }                                               \
  68         if (unlikely(_c)) yield();                      \
  69         else break;                                     \
  70     } while (_c);                                       \
  71   }
  72 #endif                          /* debug */
  73
  74 #define POLL_WHILE_FALSE(_c)    POLL_WHILE_TRUE(!(_c))
  75
  76 static inline void acquire_spu_lock(struct spu *spu)
  77 {
  78         /* Save, Step 1:
  79          * Restore, Step 1:
  80          *    Acquire SPU-specific mutual exclusion lock.
  81          *    TBD.
  82          */
  83 }
  84
  85 static inline void release_spu_lock(struct spu *spu)
  86 {
  87         /* Restore, Step 76:
  88          *    Release SPU-specific mutual exclusion lock.
  89          *    TBD.
  90          */
  91 }
  92
  93 static inline int check_spu_isolate(struct spu_state *csa, struct spu *spu)
  94 {
  95         struct spu_problem __iomem *prob = spu->problem;
  96         u32 isolate_state;
  97
  98         /* Save, Step 2:
  99          * Save, Step 6:
 100          *     If SPU_Status[E,L,IS] any field is '1', this
 101          *     SPU is in isolate state and cannot be context
 102          *     saved at this time.
 103          */
 104         isolate_state = SPU_STATUS_ISOLATED_STATE |
 105             SPU_STATUS_ISOLATED_LOAD_STAUTUS | SPU_STATUS_ISOLATED_EXIT_STAUTUS;
 106         return (in_be32(&prob->spu_status_R) & isolate_state) ? 1 : 0;
 107 }
 108
 109 static inline void disable_interrupts(struct spu_state *csa, struct spu *spu)
 110 {
 111         /* Save, Step 3:
 112          * Restore, Step 2:
 113          *     Save INT_Mask_class0 in CSA.
 114          *     Write INT_MASK_class0 with value of 0.
 115          *     Save INT_Mask_class1 in CSA.
 116          *     Write INT_MASK_class1 with value of 0.
 117          *     Save INT_Mask_class2 in CSA.
 118          *     Write INT_MASK_class2 with value of 0.
 119          */
 120         spin_lock_irq(&spu->register_lock);
 121         if (csa) {
 122                 csa->priv1.int_mask_class0_RW = spu_int_mask_get(spu, 0);
 123                 csa->priv1.int_mask_class1_RW = spu_int_mask_get(spu, 1);
 124                 csa->priv1.int_mask_class2_RW = spu_int_mask_get(spu, 2);
 125         }
 126         spu_int_mask_set(spu, 0, 0ul);
 127         spu_int_mask_set(spu, 1, 0ul);
 128         spu_int_mask_set(spu, 2, 0ul);
 129         eieio();
 130         spin_unlock_irq(&spu->register_lock);
 131 }
 132
 133 static inline void set_watchdog_timer(struct spu_state *csa, struct spu *spu)
 134 {
 135         /* Save, Step 4:
 136          * Restore, Step 25.
 137          *    Set a software watchdog timer, which specifies the
 138          *    maximum allowable time for a context save sequence.
 139          *
 140          *    For present, this implementation will not set a global
 141          *    watchdog timer, as virtualization & variable system load
 142          *    may cause unpredictable execution times.
 143          */
 144 }
 145
 146 static inline void inhibit_user_access(struct spu_state *csa, struct spu *spu)
 147 {
 148         /* Save, Step 5:
 149          * Restore, Step 3:
 150          *     Inhibit user-space access (if provided) to this
 151          *     SPU by unmapping the virtual pages assigned to
 152          *     the SPU memory-mapped I/O (MMIO) for problem
 153          *     state. TBD.
 154          */
 155 }
 156
 157 static inline void set_switch_pending(struct spu_state *csa, struct spu *spu)
 158 {
 159         /* Save, Step 7:
 160          * Restore, Step 5:
 161          *     Set a software context switch pending flag.
 162          */
 163         set_bit(SPU_CONTEXT_SWITCH_PENDING, &spu->flags);
 164         mb();
 165 }
 166
 167 static inline void save_mfc_cntl(struct spu_state *csa, struct spu *spu)
 168 {
 169         struct spu_priv2 __iomem *priv2 = spu->priv2;
 170
 171         /* Save, Step 8:
 172          *     Read and save MFC_CNTL[Ss].
 173          */
 174         if (csa) {
 175                 csa->priv2.mfc_control_RW = in_be64(&priv2->mfc_control_RW) &
 176                     MFC_CNTL_SUSPEND_DMA_STATUS_MASK;
 177         }
 178 }
 179
 180 static inline void save_spu_runcntl(struct spu_state *csa, struct spu *spu)
 181 {
 182         struct spu_problem __iomem *prob = spu->problem;
 183
 184         /* Save, Step 9:
 185          *     Save SPU_Runcntl in the CSA.  This value contains
 186          *     the "Application Desired State".
 187          */
 188         csa->prob.spu_runcntl_RW = in_be32(&prob->spu_runcntl_RW);
 189 }
 190
 191 static inline void save_mfc_sr1(struct spu_state *csa, struct spu *spu)
 192 {
 193         /* Save, Step 10:
 194          *     Save MFC_SR1 in the CSA.
 195          */
 196         csa->priv1.mfc_sr1_RW = spu_mfc_sr1_get(spu);
 197 }
 198
 199 static inline void save_spu_status(struct spu_state *csa, struct spu *spu)
 200 {
 201         struct spu_problem __iomem *prob = spu->problem;
 202
 203         /* Save, Step 11:
 204          *     Read SPU_Status[R], and save to CSA.
 205          */
 206         if ((in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) == 0) {
 207                 csa->prob.spu_status_R = in_be32(&prob->spu_status_R);
 208         } else {
 209                 u32 stopped;
 210
 211                 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
 212                 eieio();
 213                 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
 214                                 SPU_STATUS_RUNNING);
 215                 stopped =
 216                     SPU_STATUS_INVALID_INSTR | SPU_STATUS_SINGLE_STEP |
 217                     SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP;
 218                 if ((in_be32(&prob->spu_status_R) & stopped) == 0)
 219                         csa->prob.spu_status_R = SPU_STATUS_RUNNING;
 220                 else
 221                         csa->prob.spu_status_R = in_be32(&prob->spu_status_R);
 222         }
 223 }
 224
 225 static inline void save_mfc_decr(struct spu_state *csa, struct spu *spu)
 226 {
 227         struct spu_priv2 __iomem *priv2 = spu->priv2;
 228
 229         /* Save, Step 12:
 230          *     Read MFC_CNTL[Ds].  Update saved copy of
 231          *     CSA.MFC_CNTL[Ds].
 232          */
 233         if (in_be64(&priv2->mfc_control_RW) & MFC_CNTL_DECREMENTER_RUNNING) {
 234                 csa->priv2.mfc_control_RW |= MFC_CNTL_DECREMENTER_RUNNING;
 235                 csa->suspend_time = get_cycles();
 236                 out_be64(&priv2->spu_chnlcntptr_RW, 7ULL);
 237                 eieio();
 238                 csa->spu_chnldata_RW[7] = in_be64(&priv2->spu_chnldata_RW);
 239                 eieio();
 240         }
 241 }
 242
 243 static inline void halt_mfc_decr(struct spu_state *csa, struct spu *spu)
 244 {
 245         struct spu_priv2 __iomem *priv2 = spu->priv2;
 246
 247         /* Save, Step 13:
 248          *     Write MFC_CNTL[Dh] set to a '1' to halt
 249          *     the decrementer.
 250          */
 251         out_be64(&priv2->mfc_control_RW, MFC_CNTL_DECREMENTER_HALTED);
 252         eieio();
 253 }
 254
 255 static inline void save_timebase(struct spu_state *csa, struct spu *spu)
 256 {
 257         /* Save, Step 14:
 258          *    Read PPE Timebase High and Timebase low registers
 259          *    and save in CSA.  TBD.
 260          */
 261         csa->suspend_time = get_cycles();
 262 }
 263
 264 static inline void remove_other_spu_access(struct spu_state *csa,
 265                                            struct spu *spu)
 266 {
 267         /* Save, Step 15:
 268          *     Remove other SPU access to this SPU by unmapping
 269          *     this SPU's pages from their address space.  TBD.
 270          */
 271 }
 272
 273 static inline void do_mfc_mssync(struct spu_state *csa, struct spu *spu)
 274 {
 275         struct spu_problem __iomem *prob = spu->problem;
 276
 277         /* Save, Step 16:
 278          * Restore, Step 11.
 279          *     Write SPU_MSSync register. Poll SPU_MSSync[P]
 280          *     for a value of 0.
 281          */
 282         out_be64(&prob->spc_mssync_RW, 1UL);
 283         POLL_WHILE_TRUE(in_be64(&prob->spc_mssync_RW) & MS_SYNC_PENDING);
 284 }
 285
 286 static inline void issue_mfc_tlbie(struct spu_state *csa, struct spu *spu)
 287 {
 288         /* Save, Step 17:
 289          * Restore, Step 12.
 290          * Restore, Step 48.
 291          *     Write TLB_Invalidate_Entry[IS,VPN,L,Lp]=0 register.
 292          *     Then issue a PPE sync instruction.
 293          */
 294         spu_tlb_invalidate(spu);
 295         mb();
 296 }
 297
 298 static inline void handle_pending_interrupts(struct spu_state *csa,
 299                                              struct spu *spu)
 300 {
 301         /* Save, Step 18:
 302          *     Handle any pending interrupts from this SPU
 303          *     here.  This is OS or hypervisor specific.  One
 304          *     option is to re-enable interrupts to handle any
 305          *     pending interrupts, with the interrupt handlers
 306          *     recognizing the software Context Switch Pending
 307          *     flag, to ensure the SPU execution or MFC command
 308          *     queue is not restarted.  TBD.
 309          */
 310 }
 311
 312 static inline void save_mfc_queues(struct spu_state *csa, struct spu *spu)
 313 {
 314         struct spu_priv2 __iomem *priv2 = spu->priv2;
 315         int i;
 316
 317         /* Save, Step 19:
 318          *     If MFC_Cntl[Se]=0 then save
 319          *     MFC command queues.
 320          */
 321         if ((in_be64(&priv2->mfc_control_RW) & MFC_CNTL_DMA_QUEUES_EMPTY) == 0) {
 322                 for (i = 0; i < 8; i++) {
 323                         csa->priv2.puq[i].mfc_cq_data0_RW =
 324                             in_be64(&priv2->puq[i].mfc_cq_data0_RW);
 325                         csa->priv2.puq[i].mfc_cq_data1_RW =
 326                             in_be64(&priv2->puq[i].mfc_cq_data1_RW);
 327                         csa->priv2.puq[i].mfc_cq_data2_RW =
 328                             in_be64(&priv2->puq[i].mfc_cq_data2_RW);
 329                         csa->priv2.puq[i].mfc_cq_data3_RW =
 330                             in_be64(&priv2->puq[i].mfc_cq_data3_RW);
 331                 }
 332                 for (i = 0; i < 16; i++) {
 333                         csa->priv2.spuq[i].mfc_cq_data0_RW =
 334                             in_be64(&priv2->spuq[i].mfc_cq_data0_RW);
 335                         csa->priv2.spuq[i].mfc_cq_data1_RW =
 336                             in_be64(&priv2->spuq[i].mfc_cq_data1_RW);
 337                         csa->priv2.spuq[i].mfc_cq_data2_RW =
 338                             in_be64(&priv2->spuq[i].mfc_cq_data2_RW);
 339                         csa->priv2.spuq[i].mfc_cq_data3_RW =
 340                             in_be64(&priv2->spuq[i].mfc_cq_data3_RW);
 341                 }
 342         }
 343 }
 344
 345 static inline void save_ppu_querymask(struct spu_state *csa, struct spu *spu)
 346 {
 347         struct spu_problem __iomem *prob = spu->problem;
 348
 349         /* Save, Step 20:
 350          *     Save the PPU_QueryMask register
 351          *     in the CSA.
 352          */
 353         csa->prob.dma_querymask_RW = in_be32(&prob->dma_querymask_RW);
 354 }
 355
 356 static inline void save_ppu_querytype(struct spu_state *csa, struct spu *spu)
 357 {
 358         struct spu_problem __iomem *prob = spu->problem;
 359
 360         /* Save, Step 21:
 361          *     Save the PPU_QueryType register
 362          *     in the CSA.
 363          */
 364         csa->prob.dma_querytype_RW = in_be32(&prob->dma_querytype_RW);
 365 }
 366
 367 static inline void save_mfc_csr_tsq(struct spu_state *csa, struct spu *spu)
 368 {
 369         struct spu_priv2 __iomem *priv2 = spu->priv2;
 370
 371         /* Save, Step 22:
 372          *     Save the MFC_CSR_TSQ register
 373          *     in the LSCSA.
 374          */
 375         csa->priv2.spu_tag_status_query_RW =
 376             in_be64(&priv2->spu_tag_status_query_RW);
 377 }
 378
 379 static inline void save_mfc_csr_cmd(struct spu_state *csa, struct spu *spu)
 380 {
 381         struct spu_priv2 __iomem *priv2 = spu->priv2;
 382
 383         /* Save, Step 23:
 384          *     Save the MFC_CSR_CMD1 and MFC_CSR_CMD2
 385          *     registers in the CSA.
 386          */
 387         csa->priv2.spu_cmd_buf1_RW = in_be64(&priv2->spu_cmd_buf1_RW);
 388         csa->priv2.spu_cmd_buf2_RW = in_be64(&priv2->spu_cmd_buf2_RW);
 389 }
 390
 391 static inline void save_mfc_csr_ato(struct spu_state *csa, struct spu *spu)
 392 {
 393         struct spu_priv2 __iomem *priv2 = spu->priv2;
 394
 395         /* Save, Step 24:
 396          *     Save the MFC_CSR_ATO register in
 397          *     the CSA.
 398          */
 399         csa->priv2.spu_atomic_status_RW = in_be64(&priv2->spu_atomic_status_RW);
 400 }
 401
 402 static inline void save_mfc_tclass_id(struct spu_state *csa, struct spu *spu)
 403 {
 404         /* Save, Step 25:
 405          *     Save the MFC_TCLASS_ID register in
 406          *     the CSA.
 407          */
 408         csa->priv1.mfc_tclass_id_RW = spu_mfc_tclass_id_get(spu);
 409 }
 410
 411 static inline void set_mfc_tclass_id(struct spu_state *csa, struct spu *spu)
 412 {
 413         /* Save, Step 26:
 414          * Restore, Step 23.
 415          *     Write the MFC_TCLASS_ID register with
 416          *     the value 0x10000000.
 417          */
 418         spu_mfc_tclass_id_set(spu, 0x10000000);
 419         eieio();
 420 }
 421
 422 static inline void purge_mfc_queue(struct spu_state *csa, struct spu *spu)
 423 {
 424         struct spu_priv2 __iomem *priv2 = spu->priv2;
 425
 426         /* Save, Step 27:
 427          * Restore, Step 14.
 428          *     Write MFC_CNTL[Pc]=1 (purge queue).
 429          */
 430         out_be64(&priv2->mfc_control_RW, MFC_CNTL_PURGE_DMA_REQUEST);
 431         eieio();
 432 }
 433
 434 static inline void wait_purge_complete(struct spu_state *csa, struct spu *spu)
 435 {
 436         struct spu_priv2 __iomem *priv2 = spu->priv2;
 437
 438         /* Save, Step 28:
 439          *     Poll MFC_CNTL[Ps] until value '11' is read
 440          *     (purge complete).
 441          */
 442         POLL_WHILE_FALSE(in_be64(&priv2->mfc_control_RW) &
 443                          MFC_CNTL_PURGE_DMA_COMPLETE);
 444 }
 445
 446 static inline void save_mfc_slbs(struct spu_state *csa, struct spu *spu)
 447 {
 448         struct spu_priv2 __iomem *priv2 = spu->priv2;
 449         int i;
 450
 451         /* Save, Step 29:
 452          *     If MFC_SR1[R]='1', save SLBs in CSA.
 453          */
 454         if (spu_mfc_sr1_get(spu) & MFC_STATE1_RELOCATE_MASK) {
 455                 csa->priv2.slb_index_W = in_be64(&priv2->slb_index_W);
 456                 for (i = 0; i < 8; i++) {
 457                         out_be64(&priv2->slb_index_W, i);
 458                         eieio();
 459                         csa->slb_esid_RW[i] = in_be64(&priv2->slb_esid_RW);
 460                         csa->slb_vsid_RW[i] = in_be64(&priv2->slb_vsid_RW);
 461                         eieio();
 462                 }
 463         }
 464 }
 465
 466 static inline void setup_mfc_sr1(struct spu_state *csa, struct spu *spu)
 467 {
 468         /* Save, Step 30:
 469          * Restore, Step 18:
 470          *     Write MFC_SR1 with MFC_SR1[D=0,S=1] and
 471          *     MFC_SR1[TL,R,Pr,T] set correctly for the
 472          *     OS specific environment.
 473          *
 474          *     Implementation note: The SPU-side code
 475          *     for save/restore is privileged, so the
 476          *     MFC_SR1[Pr] bit is not set.
 477          *
 478          */
 479         spu_mfc_sr1_set(spu, (MFC_STATE1_MASTER_RUN_CONTROL_MASK |
 480                               MFC_STATE1_RELOCATE_MASK |
 481                               MFC_STATE1_BUS_TLBIE_MASK));
 482 }
 483
 484 static inline void save_spu_npc(struct spu_state *csa, struct spu *spu)
 485 {
 486         struct spu_problem __iomem *prob = spu->problem;
 487
 488         /* Save, Step 31:
 489          *     Save SPU_NPC in the CSA.
 490          */
 491         csa->prob.spu_npc_RW = in_be32(&prob->spu_npc_RW);
 492 }
 493
 494 static inline void save_spu_privcntl(struct spu_state *csa, struct spu *spu)
 495 {
 496         struct spu_priv2 __iomem *priv2 = spu->priv2;
 497
 498         /* Save, Step 32:
 499          *     Save SPU_PrivCntl in the CSA.
 500          */
 501         csa->priv2.spu_privcntl_RW = in_be64(&priv2->spu_privcntl_RW);
 502 }
 503
 504 static inline void reset_spu_privcntl(struct spu_state *csa, struct spu *spu)
 505 {
 506         struct spu_priv2 __iomem *priv2 = spu->priv2;
 507
 508         /* Save, Step 33:
 509          * Restore, Step 16:
 510          *     Write SPU_PrivCntl[S,Le,A] fields reset to 0.
 511          */
 512         out_be64(&priv2->spu_privcntl_RW, 0UL);
 513         eieio();
 514 }
 515
 516 static inline void save_spu_lslr(struct spu_state *csa, struct spu *spu)
 517 {
 518         struct spu_priv2 __iomem *priv2 = spu->priv2;
 519
 520         /* Save, Step 34:
 521          *     Save SPU_LSLR in the CSA.
 522          */
 523         csa->priv2.spu_lslr_RW = in_be64(&priv2->spu_lslr_RW);
 524 }
 525
 526 static inline void reset_spu_lslr(struct spu_state *csa, struct spu *spu)
 527 {
 528         struct spu_priv2 __iomem *priv2 = spu->priv2;
 529
 530         /* Save, Step 35:
 531          * Restore, Step 17.
 532          *     Reset SPU_LSLR.
 533          */
 534         out_be64(&priv2->spu_lslr_RW, LS_ADDR_MASK);
 535         eieio();
 536 }
 537
 538 static inline void save_spu_cfg(struct spu_state *csa, struct spu *spu)
 539 {
 540         struct spu_priv2 __iomem *priv2 = spu->priv2;
 541
 542         /* Save, Step 36:
 543          *     Save SPU_Cfg in the CSA.
 544          */
 545         csa->priv2.spu_cfg_RW = in_be64(&priv2->spu_cfg_RW);
 546 }
 547
 548 static inline void save_pm_trace(struct spu_state *csa, struct spu *spu)
 549 {
 550         /* Save, Step 37:
 551          *     Save PM_Trace_Tag_Wait_Mask in the CSA.
 552          *     Not performed by this implementation.
 553          */
 554 }
 555
 556 static inline void save_mfc_rag(struct spu_state *csa, struct spu *spu)
 557 {
 558         /* Save, Step 38:
 559          *     Save RA_GROUP_ID register and the
 560          *     RA_ENABLE reigster in the CSA.
 561          */
 562         csa->priv1.resource_allocation_groupID_RW =
 563                 spu_resource_allocation_groupID_get(spu);
 564         csa->priv1.resource_allocation_enable_RW =
 565                 spu_resource_allocation_enable_get(spu);
 566 }
 567
 568 static inline void save_ppu_mb_stat(struct spu_state *csa, struct spu *spu)
 569 {
 570         struct spu_problem __iomem *prob = spu->problem;
 571
 572         /* Save, Step 39:
 573          *     Save MB_Stat register in the CSA.
 574          */
 575         csa->prob.mb_stat_R = in_be32(&prob->mb_stat_R);
 576 }
 577
 578 static inline void save_ppu_mb(struct spu_state *csa, struct spu *spu)
 579 {
 580         struct spu_problem __iomem *prob = spu->problem;
 581
 582         /* Save, Step 40:
 583          *     Save the PPU_MB register in the CSA.
 584          */
 585         csa->prob.pu_mb_R = in_be32(&prob->pu_mb_R);
 586 }
 587
 588 static inline void save_ppuint_mb(struct spu_state *csa, struct spu *spu)
 589 {
 590         struct spu_priv2 __iomem *priv2 = spu->priv2;
 591
 592         /* Save, Step 41:
 593          *     Save the PPUINT_MB register in the CSA.
 594          */
 595         csa->priv2.puint_mb_R = in_be64(&priv2->puint_mb_R);
 596 }
 597
 598 static inline void save_ch_part1(struct spu_state *csa, struct spu *spu)
 599 {
 600         struct spu_priv2 __iomem *priv2 = spu->priv2;
 601         u64 idx, ch_indices[7] = { 0UL, 1UL, 3UL, 4UL, 24UL, 25UL, 27UL };
 602         int i;
 603
 604         /* Save, Step 42:
 605          *     Save the following CH: [0,1,3,4,24,25,27]
 606          */
 607         for (i = 0; i < 7; i++) {
 608                 idx = ch_indices[i];
 609                 out_be64(&priv2->spu_chnlcntptr_RW, idx);
 610                 eieio();
 611                 csa->spu_chnldata_RW[idx] = in_be64(&priv2->spu_chnldata_RW);
 612                 csa->spu_chnlcnt_RW[idx] = in_be64(&priv2->spu_chnlcnt_RW);
 613                 out_be64(&priv2->spu_chnldata_RW, 0UL);
 614                 out_be64(&priv2->spu_chnlcnt_RW, 0UL);
 615                 eieio();
 616         }
 617 }
 618
 619 static inline void save_spu_mb(struct spu_state *csa, struct spu *spu)
 620 {
 621         struct spu_priv2 __iomem *priv2 = spu->priv2;
 622         int i;
 623
 624         /* Save, Step 43:
 625          *     Save SPU Read Mailbox Channel.
 626          */
 627         out_be64(&priv2->spu_chnlcntptr_RW, 29UL);
 628         eieio();
 629         csa->spu_chnlcnt_RW[29] = in_be64(&priv2->spu_chnlcnt_RW);
 630         for (i = 0; i < 4; i++) {
 631                 csa->spu_mailbox_data[i] = in_be64(&priv2->spu_chnldata_RW);
 632         }
 633         out_be64(&priv2->spu_chnlcnt_RW, 0UL);
 634         eieio();
 635 }
 636
 637 static inline void save_mfc_cmd(struct spu_state *csa, struct spu *spu)
 638 {
 639         struct spu_priv2 __iomem *priv2 = spu->priv2;
 640
 641         /* Save, Step 44:
 642          *     Save MFC_CMD Channel.
 643          */
 644         out_be64(&priv2->spu_chnlcntptr_RW, 21UL);
 645         eieio();
 646         csa->spu_chnlcnt_RW[21] = in_be64(&priv2->spu_chnlcnt_RW);
 647         eieio();
 648 }
 649
 650 static inline void reset_ch(struct spu_state *csa, struct spu *spu)
 651 {
 652         struct spu_priv2 __iomem *priv2 = spu->priv2;
 653         u64 ch_indices[4] = { 21UL, 23UL, 28UL, 30UL };
 654         u64 ch_counts[4] = { 16UL, 1UL, 1UL, 1UL };
 655         u64 idx;
 656         int i;
 657
 658         /* Save, Step 45:
 659          *     Reset the following CH: [21, 23, 28, 30]
 660          */
 661         for (i = 0; i < 4; i++) {
 662                 idx = ch_indices[i];
 663                 out_be64(&priv2->spu_chnlcntptr_RW, idx);
 664                 eieio();
 665                 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]);
 666                 eieio();
 667         }
 668 }
 669
 670 static inline void resume_mfc_queue(struct spu_state *csa, struct spu *spu)
 671 {
 672         struct spu_priv2 __iomem *priv2 = spu->priv2;
 673
 674         /* Save, Step 46:
 675          * Restore, Step 25.
 676          *     Write MFC_CNTL[Sc]=0 (resume queue processing).
 677          */
 678         out_be64(&priv2->mfc_control_RW, MFC_CNTL_RESUME_DMA_QUEUE);
 679 }
 680
 681 static inline void invalidate_slbs(struct spu_state *csa, struct spu *spu)
 682 {
 683         struct spu_priv2 __iomem *priv2 = spu->priv2;
 684
 685         /* Save, Step 45:
 686          * Restore, Step 19:
 687          *     If MFC_SR1[R]=1, write 0 to SLB_Invalidate_All.
 688          */
 689         if (spu_mfc_sr1_get(spu) & MFC_STATE1_RELOCATE_MASK) {
 690                 out_be64(&priv2->slb_invalidate_all_W, 0UL);
 691                 eieio();
 692         }
 693 }
 694
 695 static inline void get_kernel_slb(u64 ea, u64 slb[2])
 696 {
 697         slb[0] = (get_kernel_vsid(ea) << SLB_VSID_SHIFT) | SLB_VSID_KERNEL;
 698         slb[1] = (ea & ESID_MASK) | SLB_ESID_V;
 699
 700         /* Large pages are used for kernel text/data, but not vmalloc.  */
 701         if (cpu_has_feature(CPU_FTR_16M_PAGE)
 702             && REGION_ID(ea) == KERNEL_REGION_ID)
 703                 slb[0] |= SLB_VSID_L;
 704 }
 705
 706 static inline void load_mfc_slb(struct spu *spu, u64 slb[2], int slbe)
 707 {
 708         struct spu_priv2 __iomem *priv2 = spu->priv2;
 709
 710         out_be64(&priv2->slb_index_W, slbe);
 711         eieio();
 712         out_be64(&priv2->slb_vsid_RW, slb[0]);
 713         out_be64(&priv2->slb_esid_RW, slb[1]);
 714         eieio();
 715 }
 716
 717 static inline void setup_mfc_slbs(struct spu_state *csa, struct spu *spu)
 718 {
 719         u64 code_slb[2];
 720         u64 lscsa_slb[2];
 721
 722         /* Save, Step 47:
 723          * Restore, Step 30.
 724          *     If MFC_SR1[R]=1, write 0 to SLB_Invalidate_All
 725          *     register, then initialize SLB_VSID and SLB_ESID
 726          *     to provide access to SPU context save code and
 727          *     LSCSA.
 728          *
 729          *     This implementation places both the context
 730          *     switch code and LSCSA in kernel address space.
 731          *
 732          *     Further this implementation assumes that the
 733          *     MFC_SR1[R]=1 (in other words, assume that
 734          *     translation is desired by OS environment).
 735          */
 736         invalidate_slbs(csa, spu);
 737         get_kernel_slb((unsigned long)&spu_save_code[0], code_slb);
 738         get_kernel_slb((unsigned long)csa->lscsa, lscsa_slb);
 739         load_mfc_slb(spu, code_slb, 0);
 740         if ((lscsa_slb[0] != code_slb[0]) || (lscsa_slb[1] != code_slb[1]))
 741                 load_mfc_slb(spu, lscsa_slb, 1);
 742 }
 743
 744 static inline void set_switch_active(struct spu_state *csa, struct spu *spu)
 745 {
 746         /* Save, Step 48:
 747          * Restore, Step 23.
 748          *     Change the software context switch pending flag
 749          *     to context switch active.
 750          */
 751         set_bit(SPU_CONTEXT_SWITCH_ACTIVE, &spu->flags);
 752         clear_bit(SPU_CONTEXT_SWITCH_PENDING, &spu->flags);
 753         mb();
 754 }
 755
 756 static inline void enable_interrupts(struct spu_state *csa, struct spu *spu)
 757 {
 758         unsigned long class1_mask = CLASS1_ENABLE_SEGMENT_FAULT_INTR |
 759             CLASS1_ENABLE_STORAGE_FAULT_INTR;
 760
 761         /* Save, Step 49:
 762          * Restore, Step 22:
 763          *     Reset and then enable interrupts, as
 764          *     needed by OS.
 765          *
 766          *     This implementation enables only class1
 767          *     (translation) interrupts.
 768          */
 769         spin_lock_irq(&spu->register_lock);
 770         spu_int_stat_clear(spu, 0, ~0ul);
 771         spu_int_stat_clear(spu, 1, ~0ul);
 772         spu_int_stat_clear(spu, 2, ~0ul);
 773         spu_int_mask_set(spu, 0, 0ul);
 774         spu_int_mask_set(spu, 1, class1_mask);
 775         spu_int_mask_set(spu, 2, 0ul);
 776         spin_unlock_irq(&spu->register_lock);
 777 }
 778
 779 static inline int send_mfc_dma(struct spu *spu, unsigned long ea,
 780                                unsigned int ls_offset, unsigned int size,
 781                                unsigned int tag, unsigned int rclass,
 782                                unsigned int cmd)
 783 {
 784         struct spu_problem __iomem *prob = spu->problem;
 785         union mfc_tag_size_class_cmd command;
 786         unsigned int transfer_size;
 787         volatile unsigned int status = 0x0;
 788
 789         while (size > 0) {
 790                 transfer_size =
 791                     (size > MFC_MAX_DMA_SIZE) ? MFC_MAX_DMA_SIZE : size;
 792                 command.u.mfc_size = transfer_size;
 793                 command.u.mfc_tag = tag;
 794                 command.u.mfc_rclassid = rclass;
 795                 command.u.mfc_cmd = cmd;
 796                 do {
 797                         out_be32(&prob->mfc_lsa_W, ls_offset);
 798                         out_be64(&prob->mfc_ea_W, ea);
 799                         out_be64(&prob->mfc_union_W.all64, command.all64);
 800                         status =
 801                             in_be32(&prob->mfc_union_W.by32.mfc_class_cmd32);
 802                         if (unlikely(status & 0x2)) {
 803                                 cpu_relax();
 804                         }
 805                 } while (status & 0x3);
 806                 size -= transfer_size;
 807                 ea += transfer_size;
 808                 ls_offset += transfer_size;
 809         }
 810         return 0;
 811 }
 812
 813 static inline void save_ls_16kb(struct spu_state *csa, struct spu *spu)
 814 {
 815         unsigned long addr = (unsigned long)&csa->lscsa->ls[0];
 816         unsigned int ls_offset = 0x0;
 817         unsigned int size = 16384;
 818         unsigned int tag = 0;
 819         unsigned int rclass = 0;
 820         unsigned int cmd = MFC_PUT_CMD;
 821
 822         /* Save, Step 50:
 823          *     Issue a DMA command to copy the first 16K bytes
 824          *     of local storage to the CSA.
 825          */
 826         send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd);
 827 }
 828
 829 static inline void set_spu_npc(struct spu_state *csa, struct spu *spu)
 830 {
 831         struct spu_problem __iomem *prob = spu->problem;
 832
 833         /* Save, Step 51:
 834          * Restore, Step 31.
 835          *     Write SPU_NPC[IE]=0 and SPU_NPC[LSA] to entry
 836          *     point address of context save code in local
 837          *     storage.
 838          *
 839          *     This implementation uses SPU-side save/restore
 840          *     programs with entry points at LSA of 0.
 841          */
 842         out_be32(&prob->spu_npc_RW, 0);
 843         eieio();
 844 }
 845
 846 static inline void set_signot1(struct spu_state *csa, struct spu *spu)
 847 {
 848         struct spu_problem __iomem *prob = spu->problem;
 849         union {
 850                 u64 ull;
 851                 u32 ui[2];
 852         } addr64;
 853
 854         /* Save, Step 52:
 855          * Restore, Step 32:
 856          *    Write SPU_Sig_Notify_1 register with upper 32-bits
 857          *    of the CSA.LSCSA effective address.
 858          */
 859         addr64.ull = (u64) csa->lscsa;
 860         out_be32(&prob->signal_notify1, addr64.ui[0]);
 861         eieio();
 862 }
 863
 864 static inline void set_signot2(struct spu_state *csa, struct spu *spu)
 865 {
 866         struct spu_problem __iomem *prob = spu->problem;
 867         union {
 868                 u64 ull;
 869                 u32 ui[2];
 870         } addr64;
 871
 872         /* Save, Step 53:
 873          * Restore, Step 33:
 874          *    Write SPU_Sig_Notify_2 register with lower 32-bits
 875          *    of the CSA.LSCSA effective address.
 876          */
 877         addr64.ull = (u64) csa->lscsa;
 878         out_be32(&prob->signal_notify2, addr64.ui[1]);
 879         eieio();
 880 }
 881
 882 static inline void send_save_code(struct spu_state *csa, struct spu *spu)
 883 {
 884         unsigned long addr = (unsigned long)&spu_save_code[0];
 885         unsigned int ls_offset = 0x0;
 886         unsigned int size = sizeof(spu_save_code);
 887         unsigned int tag = 0;
 888         unsigned int rclass = 0;
 889         unsigned int cmd = MFC_GETFS_CMD;
 890
 891         /* Save, Step 54:
 892          *     Issue a DMA command to copy context save code
 893          *     to local storage and start SPU.
 894          */
 895         send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd);
 896 }
 897
 898 static inline void set_ppu_querymask(struct spu_state *csa, struct spu *spu)
 899 {
 900         struct spu_problem __iomem *prob = spu->problem;
 901
 902         /* Save, Step 55:
 903          * Restore, Step 38.
 904          *     Write PPU_QueryMask=1 (enable Tag Group 0)
 905          *     and issue eieio instruction.
 906          */
 907         out_be32(&prob->dma_querymask_RW, MFC_TAGID_TO_TAGMASK(0));
 908         eieio();
 909 }
 910
 911 static inline void wait_tag_complete(struct spu_state *csa, struct spu *spu)
 912 {
 913         struct spu_problem __iomem *prob = spu->problem;
 914         u32 mask = MFC_TAGID_TO_TAGMASK(0);
 915         unsigned long flags;
 916
 917         /* Save, Step 56:
 918          * Restore, Step 39.
 919          * Restore, Step 39.
 920          * Restore, Step 46.
 921          *     Poll PPU_TagStatus[gn] until 01 (Tag group 0 complete)
 922          *     or write PPU_QueryType[TS]=01 and wait for Tag Group
 923          *     Complete Interrupt.  Write INT_Stat_Class0 or
 924          *     INT_Stat_Class2 with value of 'handled'.
 925          */
 926         POLL_WHILE_FALSE(in_be32(&prob->dma_tagstatus_R) & mask);
 927
 928         local_irq_save(flags);
 929         spu_int_stat_clear(spu, 0, ~(0ul));
 930         spu_int_stat_clear(spu, 2, ~(0ul));
 931         local_irq_restore(flags);
 932 }
 933
 934 static inline void wait_spu_stopped(struct spu_state *csa, struct spu *spu)
 935 {
 936         struct spu_problem __iomem *prob = spu->problem;
 937         unsigned long flags;
 938
 939         /* Save, Step 57:
 940          * Restore, Step 40.
 941          *     Poll until SPU_Status[R]=0 or wait for SPU Class 0
 942          *     or SPU Class 2 interrupt.  Write INT_Stat_class0
 943          *     or INT_Stat_class2 with value of handled.
 944          */
 945         POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING);
 946
 947         local_irq_save(flags);
 948         spu_int_stat_clear(spu, 0, ~(0ul));
 949         spu_int_stat_clear(spu, 2, ~(0ul));
 950         local_irq_restore(flags);
 951 }
 952
 953 static inline int check_save_status(struct spu_state *csa, struct spu *spu)
 954 {
 955         struct spu_problem __iomem *prob = spu->problem;
 956         u32 complete;
 957
 958         /* Save, Step 54:
 959          *     If SPU_Status[P]=1 and SPU_Status[SC] = "success",
 960          *     context save succeeded, otherwise context save
 961          *     failed.
 962          */
 963         complete = ((SPU_SAVE_COMPLETE << SPU_STOP_STATUS_SHIFT) |
 964                     SPU_STATUS_STOPPED_BY_STOP);
 965         return (in_be32(&prob->spu_status_R) != complete) ? 1 : 0;
 966 }
 967
 968 static inline void terminate_spu_app(struct spu_state *csa, struct spu *spu)
 969 {
 970         /* Restore, Step 4:
 971          *    If required, notify the "using application" that
 972          *    the SPU task has been terminated.  TBD.
 973          */
 974 }
 975
 976 static inline void suspend_mfc(struct spu_state *csa, struct spu *spu)
 977 {
 978         struct spu_priv2 __iomem *priv2 = spu->priv2;
 979
 980         /* Restore, Step 7:
 981          * Restore, Step 47.
 982          *     Write MFC_Cntl[Dh,Sc]='1','1' to suspend
 983          *     the queue and halt the decrementer.
 984          */
 985         out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE |
 986                  MFC_CNTL_DECREMENTER_HALTED);
 987         eieio();
 988 }
 989
 990 static inline void wait_suspend_mfc_complete(struct spu_state *csa,
 991                                              struct spu *spu)
 992 {
 993         struct spu_priv2 __iomem *priv2 = spu->priv2;
 994
 995         /* Restore, Step 8:
 996          * Restore, Step 47.
 997          *     Poll MFC_CNTL[Ss] until 11 is returned.
 998          */
 999         POLL_WHILE_FALSE(in_be64(&priv2->mfc_control_RW) &
1000                          MFC_CNTL_SUSPEND_COMPLETE);
1001 }
1002
1003 static inline int suspend_spe(struct spu_state *csa, struct spu *spu)
1004 {
1005         struct spu_problem __iomem *prob = spu->problem;
1006
1007         /* Restore, Step 9:
1008          *    If SPU_Status[R]=1, stop SPU execution
1009          *    and wait for stop to complete.
1010          *
1011          *    Returns       1 if SPU_Status[R]=1 on entry.
1012          *                  0 otherwise
1013          */
1014         if (in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) {
1015                 if (in_be32(&prob->spu_status_R) &
1016                     SPU_STATUS_ISOLATED_EXIT_STAUTUS) {
1017                         POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1018                                         SPU_STATUS_RUNNING);
1019                 }
1020                 if ((in_be32(&prob->spu_status_R) &
1021                      SPU_STATUS_ISOLATED_LOAD_STAUTUS)
1022                     || (in_be32(&prob->spu_status_R) &
1023                         SPU_STATUS_ISOLATED_STATE)) {
1024                         out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
1025                         eieio();
1026                         POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1027                                         SPU_STATUS_RUNNING);
1028                         out_be32(&prob->spu_runcntl_RW, 0x2);
1029                         eieio();
1030                         POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1031                                         SPU_STATUS_RUNNING);
1032                 }
1033                 if (in_be32(&prob->spu_status_R) &
1034                     SPU_STATUS_WAITING_FOR_CHANNEL) {
1035                         out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
1036                         eieio();
1037                         POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1038                                         SPU_STATUS_RUNNING);
1039                 }
1040                 return 1;
1041         }
1042         return 0;
1043 }
1044
1045 static inline void clear_spu_status(struct spu_state *csa, struct spu *spu)
1046 {
1047         struct spu_problem __iomem *prob = spu->problem;
1048
1049         /* Restore, Step 10:
1050          *    If SPU_Status[R]=0 and SPU_Status[E,L,IS]=1,
1051          *    release SPU from isolate state.
1052          */
1053         if (!(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING)) {
1054                 if (in_be32(&prob->spu_status_R) &
1055                     SPU_STATUS_ISOLATED_EXIT_STAUTUS) {
1056                         spu_mfc_sr1_set(spu,
1057                                         MFC_STATE1_MASTER_RUN_CONTROL_MASK);
1058                         eieio();
1059                         out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE);
1060                         eieio();
1061                         POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1062                                         SPU_STATUS_RUNNING);
1063                 }
1064                 if ((in_be32(&prob->spu_status_R) &
1065                      SPU_STATUS_ISOLATED_LOAD_STAUTUS)
1066                     || (in_be32(&prob->spu_status_R) &
1067                         SPU_STATUS_ISOLATED_STATE)) {
1068                         spu_mfc_sr1_set(spu,
1069                                         MFC_STATE1_MASTER_RUN_CONTROL_MASK);
1070                         eieio();
1071                         out_be32(&prob->spu_runcntl_RW, 0x2);
1072                         eieio();
1073                         POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1074                                         SPU_STATUS_RUNNING);
1075                 }
1076         }
1077 }
1078
1079 static inline void reset_ch_part1(struct spu_state *csa, struct spu *spu)
1080 {
1081         struct spu_priv2 __iomem *priv2 = spu->priv2;
1082         u64 ch_indices[7] = { 0UL, 1UL, 3UL, 4UL, 24UL, 25UL, 27UL };
1083         u64 idx;
1084         int i;
1085
1086         /* Restore, Step 20:
1087          *     Reset the following CH: [0,1,3,4,24,25,27]
1088          */
1089         for (i = 0; i < 7; i++) {
1090                 idx = ch_indices[i];
1091                 out_be64(&priv2->spu_chnlcntptr_RW, idx);
1092                 eieio();
1093                 out_be64(&priv2->spu_chnldata_RW, 0UL);
1094                 out_be64(&priv2->spu_chnlcnt_RW, 0UL);
1095                 eieio();
1096         }
1097 }
1098
1099 static inline void reset_ch_part2(struct spu_state *csa, struct spu *spu)
1100 {
1101         struct spu_priv2 __iomem *priv2 = spu->priv2;
1102         u64 ch_indices[5] = { 21UL, 23UL, 28UL, 29UL, 30UL };
1103         u64 ch_counts[5] = { 16UL, 1UL, 1UL, 0UL, 1UL };
1104         u64 idx;
1105         int i;
1106
1107         /* Restore, Step 21:
1108          *     Reset the following CH: [21, 23, 28, 29, 30]
1109          */
1110         for (i = 0; i < 5; i++) {
1111                 idx = ch_indices[i];
1112                 out_be64(&priv2->spu_chnlcntptr_RW, idx);
1113                 eieio();
1114                 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]);
1115                 eieio();
1116         }
1117 }
1118
1119 static inline void setup_spu_status_part1(struct spu_state *csa,
1120                                           struct spu *spu)
1121 {
1122         u32 status_P = SPU_STATUS_STOPPED_BY_STOP;
1123         u32 status_I = SPU_STATUS_INVALID_INSTR;
1124         u32 status_H = SPU_STATUS_STOPPED_BY_HALT;
1125         u32 status_S = SPU_STATUS_SINGLE_STEP;
1126         u32 status_S_I = SPU_STATUS_SINGLE_STEP | SPU_STATUS_INVALID_INSTR;
1127         u32 status_S_P = SPU_STATUS_SINGLE_STEP | SPU_STATUS_STOPPED_BY_STOP;
1128         u32 status_P_H = SPU_STATUS_STOPPED_BY_HALT |SPU_STATUS_STOPPED_BY_STOP;
1129         u32 status_P_I = SPU_STATUS_STOPPED_BY_STOP |SPU_STATUS_INVALID_INSTR;
1130         u32 status_code;
1131
1132         /* Restore, Step 27:
1133          *     If the CSA.SPU_Status[I,S,H,P]=1 then add the correct
1134          *     instruction sequence to the end of the SPU based restore
1135          *     code (after the "context restored" stop and signal) to
1136          *     restore the correct SPU status.
1137          *
1138          *     NOTE: Rather than modifying the SPU executable, we
1139          *     instead add a new 'stopped_status' field to the
1140          *     LSCSA.  The SPU-side restore reads this field and
1141          *     takes the appropriate action when exiting.
1142          */
1143
1144         status_code =
1145             (csa->prob.spu_status_R >> SPU_STOP_STATUS_SHIFT) & 0xFFFF;
1146         if ((csa->prob.spu_status_R & status_P_I) == status_P_I) {
1147
1148                 /* SPU_Status[P,I]=1 - Illegal Instruction followed
1149                  * by Stop and Signal instruction, followed by 'br -4'.
1150                  *
1151                  */
1152                 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P_I;
1153                 csa->lscsa->stopped_status.slot[1] = status_code;
1154
1155         } else if ((csa->prob.spu_status_R & status_P_H) == status_P_H) {
1156
1157                 /* SPU_Status[P,H]=1 - Halt Conditional, followed
1158                  * by Stop and Signal instruction, followed by
1159                  * 'br -4'.
1160                  */
1161                 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P_H;
1162                 csa->lscsa->stopped_status.slot[1] = status_code;
1163
1164         } else if ((csa->prob.spu_status_R & status_S_P) == status_S_P) {
1165
1166                 /* SPU_Status[S,P]=1 - Stop and Signal instruction
1167                  * followed by 'br -4'.
1168                  */
1169                 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S_P;
1170                 csa->lscsa->stopped_status.slot[1] = status_code;
1171
1172         } else if ((csa->prob.spu_status_R & status_S_I) == status_S_I) {
1173
1174                 /* SPU_Status[S,I]=1 - Illegal instruction followed
1175                  * by 'br -4'.
1176                  */
1177                 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S_I;
1178                 csa->lscsa->stopped_status.slot[1] = status_code;
1179
1180         } else if ((csa->prob.spu_status_R & status_P) == status_P) {
1181
1182                 /* SPU_Status[P]=1 - Stop and Signal instruction
1183                  * followed by 'br -4'.
1184                  */
1185                 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P;
1186                 csa->lscsa->stopped_status.slot[1] = status_code;
1187
1188         } else if ((csa->prob.spu_status_R & status_H) == status_H) {
1189
1190                 /* SPU_Status[H]=1 - Halt Conditional, followed
1191                  * by 'br -4'.
1192                  */
1193                 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_H;
1194
1195         } else if ((csa->prob.spu_status_R & status_S) == status_S) {
1196
1197                 /* SPU_Status[S]=1 - Two nop instructions.
1198                  */
1199                 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S;
1200
1201         } else if ((csa->prob.spu_status_R & status_I) == status_I) {
1202
1203                 /* SPU_Status[I]=1 - Illegal instruction followed
1204                  * by 'br -4'.
1205                  */
1206                 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_I;
1207
1208         }
1209 }
1210
1211 static inline void setup_spu_status_part2(struct spu_state *csa,
1212                                           struct spu *spu)
1213 {
1214         u32 mask;
1215
1216         /* Restore, Step 28:
1217          *     If the CSA.SPU_Status[I,S,H,P,R]=0 then
1218          *     add a 'br *' instruction to the end of
1219          *     the SPU based restore code.
1220          *
1221          *     NOTE: Rather than modifying the SPU executable, we
1222          *     instead add a new 'stopped_status' field to the
1223          *     LSCSA.  The SPU-side restore reads this field and
1224          *     takes the appropriate action when exiting.
1225          */
1226         mask = SPU_STATUS_INVALID_INSTR |
1227             SPU_STATUS_SINGLE_STEP |
1228             SPU_STATUS_STOPPED_BY_HALT |
1229             SPU_STATUS_STOPPED_BY_STOP | SPU_STATUS_RUNNING;
1230         if (!(csa->prob.spu_status_R & mask)) {
1231                 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_R;
1232         }
1233 }
1234
1235 static inline void restore_mfc_rag(struct spu_state *csa, struct spu *spu)
1236 {
1237         /* Restore, Step 29:
1238          *     Restore RA_GROUP_ID register and the
1239          *     RA_ENABLE reigster from the CSA.
1240          */
1241         spu_resource_allocation_groupID_set(spu,
1242                         csa->priv1.resource_allocation_groupID_RW);
1243         spu_resource_allocation_enable_set(spu,
1244                         csa->priv1.resource_allocation_enable_RW);
1245 }
1246
1247 static inline void send_restore_code(struct spu_state *csa, struct spu *spu)
1248 {
1249         unsigned long addr = (unsigned long)&spu_restore_code[0];
1250         unsigned int ls_offset = 0x0;
1251         unsigned int size = sizeof(spu_restore_code);
1252         unsigned int tag = 0;
1253         unsigned int rclass = 0;
1254         unsigned int cmd = MFC_GETFS_CMD;
1255
1256         /* Restore, Step 37:
1257          *     Issue MFC DMA command to copy context
1258          *     restore code to local storage.
1259          */
1260         send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd);
1261 }
1262
1263 static inline void setup_decr(struct spu_state *csa, struct spu *spu)
1264 {
1265         /* Restore, Step 34:
1266          *     If CSA.MFC_CNTL[Ds]=1 (decrementer was
1267          *     running) then adjust decrementer, set
1268          *     decrementer running status in LSCSA,
1269          *     and set decrementer "wrapped" status
1270          *     in LSCSA.
1271          */
1272         if (csa->priv2.mfc_control_RW & MFC_CNTL_DECREMENTER_RUNNING) {
1273                 cycles_t resume_time = get_cycles();
1274                 cycles_t delta_time = resume_time - csa->suspend_time;
1275
1276                 csa->lscsa->decr.slot[0] = delta_time;
1277         }
1278 }
1279
1280 static inline void setup_ppu_mb(struct spu_state *csa, struct spu *spu)
1281 {
1282         /* Restore, Step 35:
1283          *     Copy the CSA.PU_MB data into the LSCSA.
1284          */
1285         csa->lscsa->ppu_mb.slot[0] = csa->prob.pu_mb_R;
1286 }
1287
1288 static inline void setup_ppuint_mb(struct spu_state *csa, struct spu *spu)
1289 {
1290         /* Restore, Step 36:
1291          *     Copy the CSA.PUINT_MB data into the LSCSA.
1292          */
1293         csa->lscsa->ppuint_mb.slot[0] = csa->priv2.puint_mb_R;
1294 }
1295
1296 static inline int check_restore_status(struct spu_state *csa, struct spu *spu)
1297 {
1298         struct spu_problem __iomem *prob = spu->problem;
1299         u32 complete;
1300
1301         /* Restore, Step 40:
1302          *     If SPU_Status[P]=1 and SPU_Status[SC] = "success",
1303          *     context restore succeeded, otherwise context restore
1304          *     failed.
1305          */
1306         complete = ((SPU_RESTORE_COMPLETE << SPU_STOP_STATUS_SHIFT) |
1307                     SPU_STATUS_STOPPED_BY_STOP);
1308         return (in_be32(&prob->spu_status_R) != complete) ? 1 : 0;
1309 }
1310
1311 static inline void restore_spu_privcntl(struct spu_state *csa, struct spu *spu)
1312 {
1313         struct spu_priv2 __iomem *priv2 = spu->priv2;
1314
1315         /* Restore, Step 41:
1316          *     Restore SPU_PrivCntl from the CSA.
1317          */
1318         out_be64(&priv2->spu_privcntl_RW, csa->priv2.spu_privcntl_RW);
1319         eieio();
1320 }
1321
1322 static inline void restore_status_part1(struct spu_state *csa, struct spu *spu)
1323 {
1324         struct spu_problem __iomem *prob = spu->problem;
1325         u32 mask;
1326
1327         /* Restore, Step 42:
1328          *     If any CSA.SPU_Status[I,S,H,P]=1, then
1329          *     restore the error or single step state.
1330          */
1331         mask = SPU_STATUS_INVALID_INSTR |
1332             SPU_STATUS_SINGLE_STEP |
1333             SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP;
1334         if (csa->prob.spu_status_R & mask) {
1335                 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE);
1336                 eieio();
1337                 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1338                                 SPU_STATUS_RUNNING);
1339         }
1340 }
1341
1342 static inline void restore_status_part2(struct spu_state *csa, struct spu *spu)
1343 {
1344         struct spu_problem __iomem *prob = spu->problem;
1345         u32 mask;
1346
1347         /* Restore, Step 43:
1348          *     If all CSA.SPU_Status[I,S,H,P,R]=0 then write
1349          *     SPU_RunCntl[R0R1]='01', wait for SPU_Status[R]=1,
1350          *     then write '00' to SPU_RunCntl[R0R1] and wait
1351          *     for SPU_Status[R]=0.
1352          */
1353         mask = SPU_STATUS_INVALID_INSTR |
1354             SPU_STATUS_SINGLE_STEP |
1355             SPU_STATUS_STOPPED_BY_HALT |
1356             SPU_STATUS_STOPPED_BY_STOP | SPU_STATUS_RUNNING;
1357         if (!(csa->prob.spu_status_R & mask)) {
1358                 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE);
1359                 eieio();
1360                 POLL_WHILE_FALSE(in_be32(&prob->spu_status_R) &
1361                                  SPU_STATUS_RUNNING);
1362                 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
1363                 eieio();
1364                 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1365                                 SPU_STATUS_RUNNING);
1366         }
1367 }
1368
1369 static inline void restore_ls_16kb(struct spu_state *csa, struct spu *spu)
1370 {
1371         unsigned long addr = (unsigned long)&csa->lscsa->ls[0];
1372         unsigned int ls_offset = 0x0;
1373         unsigned int size = 16384;
1374         unsigned int tag = 0;
1375         unsigned int rclass = 0;
1376         unsigned int cmd = MFC_GET_CMD;
1377
1378         /* Restore, Step 44:
1379          *     Issue a DMA command to restore the first
1380          *     16kb of local storage from CSA.
1381          */
1382         send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd);
1383 }
1384
1385 static inline void clear_interrupts(struct spu_state *csa, struct spu *spu)
1386 {
1387         /* Restore, Step 49:
1388          *     Write INT_MASK_class0 with value of 0.
1389          *     Write INT_MASK_class1 with value of 0.
1390          *     Write INT_MASK_class2 with value of 0.
1391          *     Write INT_STAT_class0 with value of -1.
1392          *     Write INT_STAT_class1 with value of -1.
1393          *     Write INT_STAT_class2 with value of -1.
1394          */
1395         spin_lock_irq(&spu->register_lock);
1396         spu_int_mask_set(spu, 0, 0ul);
1397         spu_int_mask_set(spu, 1, 0ul);
1398         spu_int_mask_set(spu, 2, 0ul);
1399         spu_int_stat_clear(spu, 0, ~0ul);
1400         spu_int_stat_clear(spu, 1, ~0ul);
1401         spu_int_stat_clear(spu, 2, ~0ul);
1402         spin_unlock_irq(&spu->register_lock);
1403 }
1404
1405 static inline void restore_mfc_queues(struct spu_state *csa, struct spu *spu)
1406 {
1407         struct spu_priv2 __iomem *priv2 = spu->priv2;
1408         int i;
1409
1410         /* Restore, Step 50:
1411          *     If MFC_Cntl[Se]!=0 then restore
1412          *     MFC command queues.
1413          */
1414         if ((csa->priv2.mfc_control_RW & MFC_CNTL_DMA_QUEUES_EMPTY_MASK) == 0) {
1415                 for (i = 0; i < 8; i++) {
1416                         out_be64(&priv2->puq[i].mfc_cq_data0_RW,
1417                                  csa->priv2.puq[i].mfc_cq_data0_RW);
1418                         out_be64(&priv2->puq[i].mfc_cq_data1_RW,
1419                                  csa->priv2.puq[i].mfc_cq_data1_RW);
1420                         out_be64(&priv2->puq[i].mfc_cq_data2_RW,
1421                                  csa->priv2.puq[i].mfc_cq_data2_RW);
1422                         out_be64(&priv2->puq[i].mfc_cq_data3_RW,
1423                                  csa->priv2.puq[i].mfc_cq_data3_RW);
1424                 }
1425                 for (i = 0; i < 16; i++) {
1426                         out_be64(&priv2->spuq[i].mfc_cq_data0_RW,
1427                                  csa->priv2.spuq[i].mfc_cq_data0_RW);
1428                         out_be64(&priv2->spuq[i].mfc_cq_data1_RW,
1429                                  csa->priv2.spuq[i].mfc_cq_data1_RW);
1430                         out_be64(&priv2->spuq[i].mfc_cq_data2_RW,
1431                                  csa->priv2.spuq[i].mfc_cq_data2_RW);
1432                         out_be64(&priv2->spuq[i].mfc_cq_data3_RW,
1433                                  csa->priv2.spuq[i].mfc_cq_data3_RW);
1434                 }
1435         }
1436         eieio();
1437 }
1438
1439 static inline void restore_ppu_querymask(struct spu_state *csa, struct spu *spu)
1440 {
1441         struct spu_problem __iomem *prob = spu->problem;
1442
1443         /* Restore, Step 51:
1444          *     Restore the PPU_QueryMask register from CSA.
1445          */
1446         out_be32(&prob->dma_querymask_RW, csa->prob.dma_querymask_RW);
1447         eieio();
1448 }
1449
1450 static inline void restore_ppu_querytype(struct spu_state *csa, struct spu *spu)
1451 {
1452         struct spu_problem __iomem *prob = spu->problem;
1453
1454         /* Restore, Step 52:
1455          *     Restore the PPU_QueryType register from CSA.
1456          */
1457         out_be32(&prob->dma_querytype_RW, csa->prob.dma_querytype_RW);
1458         eieio();
1459 }
1460
1461 static inline void restore_mfc_csr_tsq(struct spu_state *csa, struct spu *spu)
1462 {
1463         struct spu_priv2 __iomem *priv2 = spu->priv2;
1464
1465         /* Restore, Step 53:
1466          *     Restore the MFC_CSR_TSQ register from CSA.
1467          */
1468         out_be64(&priv2->spu_tag_status_query_RW,
1469                  csa->priv2.spu_tag_status_query_RW);
1470         eieio();
1471 }
1472
1473 static inline void restore_mfc_csr_cmd(struct spu_state *csa, struct spu *spu)
1474 {
1475         struct spu_priv2 __iomem *priv2 = spu->priv2;
1476
1477         /* Restore, Step 54:
1478          *     Restore the MFC_CSR_CMD1 and MFC_CSR_CMD2
1479          *     registers from CSA.
1480          */
1481         out_be64(&priv2->spu_cmd_buf1_RW, csa->priv2.spu_cmd_buf1_RW);
1482         out_be64(&priv2->spu_cmd_buf2_RW, csa->priv2.spu_cmd_buf2_RW);
1483         eieio();
1484 }
1485
1486 static inline void restore_mfc_csr_ato(struct spu_state *csa, struct spu *spu)
1487 {
1488         struct spu_priv2 __iomem *priv2 = spu->priv2;
1489
1490         /* Restore, Step 55:
1491          *     Restore the MFC_CSR_ATO register from CSA.
1492          */
1493         out_be64(&priv2->spu_atomic_status_RW, csa->priv2.spu_atomic_status_RW);
1494 }
1495
1496 static inline void restore_mfc_tclass_id(struct spu_state *csa, struct spu *spu)
1497 {
1498         /* Restore, Step 56:
1499          *     Restore the MFC_TCLASS_ID register from CSA.
1500          */
1501         spu_mfc_tclass_id_set(spu, csa->priv1.mfc_tclass_id_RW);
1502         eieio();
1503 }
1504
1505 static inline void set_llr_event(struct spu_state *csa, struct spu *spu)
1506 {
1507         u64 ch0_cnt, ch0_data;
1508         u64 ch1_data;
1509
1510         /* Restore, Step 57:
1511          *    Set the Lock Line Reservation Lost Event by:
1512          *      1. OR CSA.SPU_Event_Status with bit 21 (Lr) set to 1.
1513          *      2. If CSA.SPU_Channel_0_Count=0 and
1514          *         CSA.SPU_Wr_Event_Mask[Lr]=1 and
1515          *         CSA.SPU_Event_Status[Lr]=0 then set
1516          *         CSA.SPU_Event_Status_Count=1.
1517          */
1518         ch0_cnt = csa->spu_chnlcnt_RW[0];
1519         ch0_data = csa->spu_chnldata_RW[0];
1520         ch1_data = csa->spu_chnldata_RW[1];
1521         csa->spu_chnldata_RW[0] |= MFC_LLR_LOST_EVENT;
1522         if ((ch0_cnt == 0) && !(ch0_data & MFC_LLR_LOST_EVENT) &&
1523             (ch1_data & MFC_LLR_LOST_EVENT)) {
1524                 csa->spu_chnlcnt_RW[0] = 1;
1525         }
1526 }
1527
1528 static inline void restore_decr_wrapped(struct spu_state *csa, struct spu *spu)
1529 {
1530         /* Restore, Step 58:
1531          *     If the status of the CSA software decrementer
1532          *     "wrapped" flag is set, OR in a '1' to
1533          *     CSA.SPU_Event_Status[Tm].
1534          */
1535         if (csa->lscsa->decr_status.slot[0] == 1) {
1536                 csa->spu_chnldata_RW[0] |= 0x20;
1537         }
1538         if ((csa->lscsa->decr_status.slot[0] == 1) &&
1539             (csa->spu_chnlcnt_RW[0] == 0 &&
1540              ((csa->spu_chnldata_RW[2] & 0x20) == 0x0) &&
1541              ((csa->spu_chnldata_RW[0] & 0x20) != 0x1))) {
1542                 csa->spu_chnlcnt_RW[0] = 1;
1543         }
1544 }
1545
1546 static inline void restore_ch_part1(struct spu_state *csa, struct spu *spu)
1547 {
1548         struct spu_priv2 __iomem *priv2 = spu->priv2;
1549         u64 idx, ch_indices[7] = { 0UL, 1UL, 3UL, 4UL, 24UL, 25UL, 27UL };
1550         int i;
1551
1552         /* Restore, Step 59:
1553          *     Restore the following CH: [0,1,3,4,24,25,27]
1554          */
1555         for (i = 0; i < 7; i++) {
1556                 idx = ch_indices[i];
1557                 out_be64(&priv2->spu_chnlcntptr_RW, idx);
1558                 eieio();
1559                 out_be64(&priv2->spu_chnldata_RW, csa->spu_chnldata_RW[idx]);
1560                 out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[idx]);
1561                 eieio();
1562         }
1563 }
1564
1565 static inline void restore_ch_part2(struct spu_state *csa, struct spu *spu)
1566 {
1567         struct spu_priv2 __iomem *priv2 = spu->priv2;
1568         u64 ch_indices[3] = { 9UL, 21UL, 23UL };
1569         u64 ch_counts[3] = { 1UL, 16UL, 1UL };
1570         u64 idx;
1571         int i;
1572
1573         /* Restore, Step 60:
1574          *     Restore the following CH: [9,21,23].
1575          */
1576         ch_counts[0] = 1UL;
1577         ch_counts[1] = csa->spu_chnlcnt_RW[21];
1578         ch_counts[2] = 1UL;
1579         for (i = 0; i < 3; i++) {
1580                 idx = ch_indices[i];
1581                 out_be64(&priv2->spu_chnlcntptr_RW, idx);
1582                 eieio();
1583                 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]);
1584                 eieio();
1585         }
1586 }
1587
1588 static inline void restore_spu_lslr(struct spu_state *csa, struct spu *spu)
1589 {
1590         struct spu_priv2 __iomem *priv2 = spu->priv2;
1591
1592         /* Restore, Step 61:
1593          *     Restore the SPU_LSLR register from CSA.
1594          */
1595         out_be64(&priv2->spu_lslr_RW, csa->priv2.spu_lslr_RW);
1596         eieio();
1597 }
1598
1599 static inline void restore_spu_cfg(struct spu_state *csa, struct spu *spu)
1600 {
1601         struct spu_priv2 __iomem *priv2 = spu->priv2;
1602
1603         /* Restore, Step 62:
1604          *     Restore the SPU_Cfg register from CSA.
1605          */
1606         out_be64(&priv2->spu_cfg_RW, csa->priv2.spu_cfg_RW);
1607         eieio();
1608 }
1609
1610 static inline void restore_pm_trace(struct spu_state *csa, struct spu *spu)
1611 {
1612         /* Restore, Step 63:
1613          *     Restore PM_Trace_Tag_Wait_Mask from CSA.
1614          *     Not performed by this implementation.
1615          */
1616 }
1617
1618 static inline void restore_spu_npc(struct spu_state *csa, struct spu *spu)
1619 {
1620         struct spu_problem __iomem *prob = spu->problem;
1621
1622         /* Restore, Step 64:
1623          *     Restore SPU_NPC from CSA.
1624          */
1625         out_be32(&prob->spu_npc_RW, csa->prob.spu_npc_RW);
1626         eieio();
1627 }
1628
1629 static inline void restore_spu_mb(struct spu_state *csa, struct spu *spu)
1630 {
1631         struct spu_priv2 __iomem *priv2 = spu->priv2;
1632         int i;
1633
1634         /* Restore, Step 65:
1635          *     Restore MFC_RdSPU_MB from CSA.
1636          */
1637         out_be64(&priv2->spu_chnlcntptr_RW, 29UL);
1638         eieio();
1639         out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[29]);
1640         for (i = 0; i < 4; i++) {
1641                 out_be64(&priv2->spu_chnldata_RW, csa->spu_mailbox_data[i]);
1642         }
1643         eieio();
1644 }
1645
1646 static inline void check_ppu_mb_stat(struct spu_state *csa, struct spu *spu)
1647 {
1648         struct spu_problem __iomem *prob = spu->problem;
1649         u32 dummy = 0;
1650
1651         /* Restore, Step 66:
1652          *     If CSA.MB_Stat[P]=0 (mailbox empty) then
1653          *     read from the PPU_MB register.
1654          */
1655         if ((csa->prob.mb_stat_R & 0xFF) == 0) {
1656                 dummy = in_be32(&prob->pu_mb_R);
1657                 eieio();
1658         }
1659 }
1660
1661 static inline void check_ppuint_mb_stat(struct spu_state *csa, struct spu *spu)
1662 {
1663         struct spu_priv2 __iomem *priv2 = spu->priv2;
1664         u64 dummy = 0UL;
1665
1666         /* Restore, Step 66:
1667          *     If CSA.MB_Stat[I]=0 (mailbox empty) then
1668          *     read from the PPUINT_MB register.
1669          */
1670         if ((csa->prob.mb_stat_R & 0xFF0000) == 0) {
1671                 dummy = in_be64(&priv2->puint_mb_R);
1672                 eieio();
1673                 spu_int_stat_clear(spu, 2, CLASS2_ENABLE_MAILBOX_INTR);
1674                 eieio();
1675         }
1676 }
1677
1678 static inline void restore_mfc_slbs(struct spu_state *csa, struct spu *spu)
1679 {
1680         struct spu_priv2 __iomem *priv2 = spu->priv2;
1681         int i;
1682
1683         /* Restore, Step 68:
1684          *     If MFC_SR1[R]='1', restore SLBs from CSA.
1685          */
1686         if (csa->priv1.mfc_sr1_RW & MFC_STATE1_RELOCATE_MASK) {
1687                 for (i = 0; i < 8; i++) {
1688                         out_be64(&priv2->slb_index_W, i);
1689                         eieio();
1690                         out_be64(&priv2->slb_esid_RW, csa->slb_esid_RW[i]);
1691                         out_be64(&priv2->slb_vsid_RW, csa->slb_vsid_RW[i]);
1692                         eieio();
1693                 }
1694                 out_be64(&priv2->slb_index_W, csa->priv2.slb_index_W);
1695                 eieio();
1696         }
1697 }
1698
1699 static inline void restore_mfc_sr1(struct spu_state *csa, struct spu *spu)
1700 {
1701         /* Restore, Step 69:
1702          *     Restore the MFC_SR1 register from CSA.
1703          */
1704         spu_mfc_sr1_set(spu, csa->priv1.mfc_sr1_RW);
1705         eieio();
1706 }
1707
1708 static inline void restore_other_spu_access(struct spu_state *csa,
1709                                             struct spu *spu)
1710 {
1711         /* Restore, Step 70:
1712          *     Restore other SPU mappings to this SPU. TBD.
1713          */
1714 }
1715
1716 static inline void restore_spu_runcntl(struct spu_state *csa, struct spu *spu)
1717 {
1718         struct spu_problem __iomem *prob = spu->problem;
1719
1720         /* Restore, Step 71:
1721          *     If CSA.SPU_Status[R]=1 then write
1722          *     SPU_RunCntl[R0R1]='01'.
1723          */
1724         if (csa->prob.spu_status_R & SPU_STATUS_RUNNING) {
1725                 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE);
1726                 eieio();
1727         }
1728 }
1729
1730 static inline void restore_mfc_cntl(struct spu_state *csa, struct spu *spu)
1731 {
1732         struct spu_priv2 __iomem *priv2 = spu->priv2;
1733
1734         /* Restore, Step 72:
1735          *    Restore the MFC_CNTL register for the CSA.
1736          */
1737         out_be64(&priv2->mfc_control_RW, csa->priv2.mfc_control_RW);
1738         eieio();
1739 }
1740
1741 static inline void enable_user_access(struct spu_state *csa, struct spu *spu)
1742 {
1743         /* Restore, Step 73:
1744          *     Enable user-space access (if provided) to this
1745          *     SPU by mapping the virtual pages assigned to
1746          *     the SPU memory-mapped I/O (MMIO) for problem
1747          *     state. TBD.
1748          */
1749 }
1750
1751 static inline void reset_switch_active(struct spu_state *csa, struct spu *spu)
1752 {
1753         /* Restore, Step 74:
1754          *     Reset the "context switch active" flag.
1755          */
1756         clear_bit(SPU_CONTEXT_SWITCH_ACTIVE, &spu->flags);
1757         mb();
1758 }
1759
1760 static inline void reenable_interrupts(struct spu_state *csa, struct spu *spu)
1761 {
1762         /* Restore, Step 75:
1763          *     Re-enable SPU interrupts.
1764          */
1765         spin_lock_irq(&spu->register_lock);
1766         spu_int_mask_set(spu, 0, csa->priv1.int_mask_class0_RW);
1767         spu_int_mask_set(spu, 1, csa->priv1.int_mask_class1_RW);
1768         spu_int_mask_set(spu, 2, csa->priv1.int_mask_class2_RW);
1769         spin_unlock_irq(&spu->register_lock);
1770 }
1771
1772 static int quiece_spu(struct spu_state *prev, struct spu *spu)
1773 {
1774         /*
1775          * Combined steps 2-18 of SPU context save sequence, which
1776          * quiesce the SPU state (disable SPU execution, MFC command
1777          * queues, decrementer, SPU interrupts, etc.).
1778          *
1779          * Returns      0 on success.
1780          *              2 if failed step 2.
1781          *              6 if failed step 6.
1782          */
1783
1784         if (check_spu_isolate(prev, spu)) {     /* Step 2. */
1785                 return 2;
1786         }
1787         disable_interrupts(prev, spu);          /* Step 3. */
1788         set_watchdog_timer(prev, spu);          /* Step 4. */
1789         inhibit_user_access(prev, spu);         /* Step 5. */
1790         if (check_spu_isolate(prev, spu)) {     /* Step 6. */
1791                 return 6;
1792         }
1793         set_switch_pending(prev, spu);          /* Step 7. */
1794         save_mfc_cntl(prev, spu);               /* Step 8. */
1795         save_spu_runcntl(prev, spu);            /* Step 9. */
1796         save_mfc_sr1(prev, spu);                /* Step 10. */
1797         save_spu_status(prev, spu);             /* Step 11. */
1798         save_mfc_decr(prev, spu);               /* Step 12. */
1799         halt_mfc_decr(prev, spu);               /* Step 13. */
1800         save_timebase(prev, spu);               /* Step 14. */
1801         remove_other_spu_access(prev, spu);     /* Step 15. */
1802         do_mfc_mssync(prev, spu);               /* Step 16. */
1803         issue_mfc_tlbie(prev, spu);             /* Step 17. */
1804         handle_pending_interrupts(prev, spu);   /* Step 18. */
1805
1806         return 0;
1807 }
1808
1809 static void save_csa(struct spu_state *prev, struct spu *spu)
1810 {
1811         /*
1812          * Combine steps 19-44 of SPU context save sequence, which
1813          * save regions of the privileged & problem state areas.
1814          */
1815
1816         save_mfc_queues(prev, spu);     /* Step 19. */
1817         save_ppu_querymask(prev, spu);  /* Step 20. */
1818         save_ppu_querytype(prev, spu);  /* Step 21. */
1819         save_mfc_csr_tsq(prev, spu);    /* Step 22. */
1820         save_mfc_csr_cmd(prev, spu);    /* Step 23. */
1821         save_mfc_csr_ato(prev, spu);    /* Step 24. */
1822         save_mfc_tclass_id(prev, spu);  /* Step 25. */
1823         set_mfc_tclass_id(prev, spu);   /* Step 26. */
1824         purge_mfc_queue(prev, spu);     /* Step 27. */
1825         wait_purge_complete(prev, spu); /* Step 28. */
1826         save_mfc_slbs(prev, spu);       /* Step 29. */
1827         setup_mfc_sr1(prev, spu);       /* Step 30. */
1828         save_spu_npc(prev, spu);        /* Step 31. */
1829         save_spu_privcntl(prev, spu);   /* Step 32. */
1830         reset_spu_privcntl(prev, spu);  /* Step 33. */
1831         save_spu_lslr(prev, spu);       /* Step 34. */
1832         reset_spu_lslr(prev, spu);      /* Step 35. */
1833         save_spu_cfg(prev, spu);        /* Step 36. */
1834         save_pm_trace(prev, spu);       /* Step 37. */
1835         save_mfc_rag(prev, spu);        /* Step 38. */
1836         save_ppu_mb_stat(prev, spu);    /* Step 39. */
1837         save_ppu_mb(prev, spu);         /* Step 40. */
1838         save_ppuint_mb(prev, spu);      /* Step 41. */
1839         save_ch_part1(prev, spu);       /* Step 42. */
1840         save_spu_mb(prev, spu);         /* Step 43. */
1841         save_mfc_cmd(prev, spu);        /* Step 44. */
1842         reset_ch(prev, spu);            /* Step 45. */
1843 }
1844
1845 static void save_lscsa(struct spu_state *prev, struct spu *spu)
1846 {
1847         /*
1848          * Perform steps 46-57 of SPU context save sequence,
1849          * which save regions of the local store and register
1850          * file.
1851          */
1852
1853         resume_mfc_queue(prev, spu);    /* Step 46. */
1854         setup_mfc_slbs(prev, spu);      /* Step 47. */
1855         set_switch_active(prev, spu);   /* Step 48. */
1856         enable_interrupts(prev, spu);   /* Step 49. */
1857         save_ls_16kb(prev, spu);        /* Step 50. */
1858         set_spu_npc(prev, spu);         /* Step 51. */
1859         set_signot1(prev, spu);         /* Step 52. */
1860         set_signot2(prev, spu);         /* Step 53. */
1861         send_save_code(prev, spu);      /* Step 54. */
1862         set_ppu_querymask(prev, spu);   /* Step 55. */
1863         wait_tag_complete(prev, spu);   /* Step 56. */
1864         wait_spu_stopped(prev, spu);    /* Step 57. */
1865 }
1866
1867 static void harvest(struct spu_state *prev, struct spu *spu)
1868 {
1869         /*
1870          * Perform steps 2-25 of SPU context restore sequence,
1871          * which resets an SPU either after a failed save, or
1872          * when using SPU for first time.
1873          */
1874
1875         disable_interrupts(prev, spu);          /* Step 2.  */
1876         inhibit_user_access(prev, spu);         /* Step 3.  */
1877         terminate_spu_app(prev, spu);           /* Step 4.  */
1878         set_switch_pending(prev, spu);          /* Step 5.  */
1879         remove_other_spu_access(prev, spu);     /* Step 6.  */
1880         suspend_mfc(prev, spu);                 /* Step 7.  */
1881         wait_suspend_mfc_complete(prev, spu);   /* Step 8.  */
1882         if (!suspend_spe(prev, spu))            /* Step 9.  */
1883                 clear_spu_status(prev, spu);    /* Step 10. */
1884         do_mfc_mssync(prev, spu);               /* Step 11. */
1885         issue_mfc_tlbie(prev, spu);             /* Step 12. */
1886         handle_pending_interrupts(prev, spu);   /* Step 13. */
1887         purge_mfc_queue(prev, spu);             /* Step 14. */
1888         wait_purge_complete(prev, spu);         /* Step 15. */
1889         reset_spu_privcntl(prev, spu);          /* Step 16. */
1890         reset_spu_lslr(prev, spu);              /* Step 17. */
1891         setup_mfc_sr1(prev, spu);               /* Step 18. */
1892         invalidate_slbs(prev, spu);             /* Step 19. */
1893         reset_ch_part1(prev, spu);              /* Step 20. */
1894         reset_ch_part2(prev, spu);              /* Step 21. */
1895         enable_interrupts(prev, spu);           /* Step 22. */
1896         set_switch_active(prev, spu);           /* Step 23. */
1897         set_mfc_tclass_id(prev, spu);           /* Step 24. */
1898         resume_mfc_queue(prev, spu);            /* Step 25. */
1899 }
1900
1901 static void restore_lscsa(struct spu_state *next, struct spu *spu)
1902 {
1903         /*
1904          * Perform steps 26-40 of SPU context restore sequence,
1905          * which restores regions of the local store and register
1906          * file.
1907          */
1908
1909         set_watchdog_timer(next, spu);          /* Step 26. */
1910         setup_spu_status_part1(next, spu);      /* Step 27. */
1911         setup_spu_status_part2(next, spu);      /* Step 28. */
1912         restore_mfc_rag(next, spu);             /* Step 29. */
1913         setup_mfc_slbs(next, spu);              /* Step 30. */
1914         set_spu_npc(next, spu);                 /* Step 31. */
1915         set_signot1(next, spu);                 /* Step 32. */
1916         set_signot2(next, spu);                 /* Step 33. */
1917         setup_decr(next, spu);                  /* Step 34. */
1918         setup_ppu_mb(next, spu);                /* Step 35. */
1919         setup_ppuint_mb(next, spu);             /* Step 36. */
1920         send_restore_code(next, spu);           /* Step 37. */
1921         set_ppu_querymask(next, spu);           /* Step 38. */
1922         wait_tag_complete(next, spu);           /* Step 39. */
1923         wait_spu_stopped(next, spu);            /* Step 40. */
1924 }
1925
1926 static void restore_csa(struct spu_state *next, struct spu *spu)
1927 {
1928         /*
1929          * Combine steps 41-76 of SPU context restore sequence, which
1930          * restore regions of the privileged & problem state areas.
1931          */
1932
1933         restore_spu_privcntl(next, spu);        /* Step 41. */
1934         restore_status_part1(next, spu);        /* Step 42. */
1935         restore_status_part2(next, spu);        /* Step 43. */
1936         restore_ls_16kb(next, spu);             /* Step 44. */
1937         wait_tag_complete(next, spu);           /* Step 45. */
1938         suspend_mfc(next, spu);                 /* Step 46. */
1939         wait_suspend_mfc_complete(next, spu);   /* Step 47. */
1940         issue_mfc_tlbie(next, spu);             /* Step 48. */
1941         clear_interrupts(next, spu);            /* Step 49. */
1942         restore_mfc_queues(next, spu);          /* Step 50. */
1943         restore_ppu_querymask(next, spu);       /* Step 51. */
1944         restore_ppu_querytype(next, spu);       /* Step 52. */
1945         restore_mfc_csr_tsq(next, spu);         /* Step 53. */
1946         restore_mfc_csr_cmd(next, spu);         /* Step 54. */
1947         restore_mfc_csr_ato(next, spu);         /* Step 55. */
1948         restore_mfc_tclass_id(next, spu);       /* Step 56. */
1949         set_llr_event(next, spu);               /* Step 57. */
1950         restore_decr_wrapped(next, spu);        /* Step 58. */
1951         restore_ch_part1(next, spu);            /* Step 59. */
1952         restore_ch_part2(next, spu);            /* Step 60. */
1953         restore_spu_lslr(next, spu);            /* Step 61. */
1954         restore_spu_cfg(next, spu);             /* Step 62. */
1955         restore_pm_trace(next, spu);            /* Step 63. */
1956         restore_spu_npc(next, spu);             /* Step 64. */
1957         restore_spu_mb(next, spu);              /* Step 65. */
1958         check_ppu_mb_stat(next, spu);           /* Step 66. */
1959         check_ppuint_mb_stat(next, spu);        /* Step 67. */
1960         restore_mfc_slbs(next, spu);            /* Step 68. */
1961         restore_mfc_sr1(next, spu);             /* Step 69. */
1962         restore_other_spu_access(next, spu);    /* Step 70. */
1963         restore_spu_runcntl(next, spu);         /* Step 71. */
1964         restore_mfc_cntl(next, spu);            /* Step 72. */
1965         enable_user_access(next, spu);          /* Step 73. */
1966         reset_switch_active(next, spu);         /* Step 74. */
1967         reenable_interrupts(next, spu);         /* Step 75. */
1968 }
1969
1970 static int __do_spu_save(struct spu_state *prev, struct spu *spu)
1971 {
1972         int rc;
1973
1974         /*
1975          * SPU context save can be broken into three phases:
1976          *
1977          *     (a) quiesce [steps 2-16].
1978          *     (b) save of CSA, performed by PPE [steps 17-42]
1979          *     (c) save of LSCSA, mostly performed by SPU [steps 43-52].
1980          *
1981          * Returns      0 on success.
1982          *              2,6 if failed to quiece SPU
1983          *              53 if SPU-side of save failed.
1984          */
1985
1986         rc = quiece_spu(prev, spu);             /* Steps 2-16. */
1987         switch (rc) {
1988         default:
1989         case 2:
1990         case 6:
1991                 harvest(prev, spu);
1992                 return rc;
1993                 break;
1994         case 0:
1995                 break;
1996         }
1997         save_csa(prev, spu);                    /* Steps 17-43. */
1998         save_lscsa(prev, spu);                  /* Steps 44-53. */
1999         return check_save_status(prev, spu);    /* Step 54.     */
2000 }
2001
2002 static int __do_spu_restore(struct spu_state *next, struct spu *spu)
2003 {
2004         int rc;
2005
2006         /*
2007          * SPU context restore can be broken into three phases:
2008          *
2009          *    (a) harvest (or reset) SPU [steps 2-24].
2010          *    (b) restore LSCSA [steps 25-40], mostly performed by SPU.
2011          *    (c) restore CSA [steps 41-76], performed by PPE.
2012          *
2013          * The 'harvest' step is not performed here, but rather
2014          * as needed below.
2015          */
2016
2017         restore_lscsa(next, spu);               /* Steps 24-39. */
2018         rc = check_restore_status(next, spu);   /* Step 40.     */
2019         switch (rc) {
2020         default:
2021                 /* Failed. Return now. */
2022                 return rc;
2023                 break;
2024         case 0:
2025                 /* Fall through to next step. */
2026                 break;
2027         }
2028         restore_csa(next, spu);
2029
2030         return 0;
2031 }
2032
2033 /**
2034  * spu_save - SPU context save, with locking.
2035  * @prev: pointer to SPU context save area, to be saved.
2036  * @spu: pointer to SPU iomem structure.
2037  *
2038  * Acquire locks, perform the save operation then return.
2039  */
2040 int spu_save(struct spu_state *prev, struct spu *spu)
2041 {
2042         int rc;
2043
2044         acquire_spu_lock(spu);          /* Step 1.     */
2045         rc = __do_spu_save(prev, spu);  /* Steps 2-53. */
2046         release_spu_lock(spu);
2047         if (rc) {
2048                 panic("%s failed on SPU[%d], rc=%d.\n",
2049                       __func__, spu->number, rc);
2050         }
2051         return rc;
2052 }
2053
2054 /**
2055  * spu_restore - SPU context restore, with harvest and locking.
2056  * @new: pointer to SPU context save area, to be restored.
2057  * @spu: pointer to SPU iomem structure.
2058  *
2059  * Perform harvest + restore, as we may not be coming
2060  * from a previous succesful save operation, and the
2061  * hardware state is unknown.
2062  */
2063 int spu_restore(struct spu_state *new, struct spu *spu)
2064 {
2065         int rc;
2066
2067         acquire_spu_lock(spu);
2068         harvest(NULL, spu);
2069         spu->stop_code = 0;
2070         spu->dar = 0;
2071         spu->dsisr = 0;
2072         spu->slb_replace = 0;
2073         spu->class_0_pending = 0;
2074         rc = __do_spu_restore(new, spu);
2075         release_spu_lock(spu);
2076         if (rc) {
2077                 panic("%s failed on SPU[%d] rc=%d.\n",
2078                        __func__, spu->number, rc);
2079         }
2080         return rc;
2081 }
2082
2083 /**
2084  * spu_harvest - SPU harvest (reset) operation
2085  * @spu: pointer to SPU iomem structure.
2086  *
2087  * Perform SPU harvest (reset) operation.
2088  */
2089 void spu_harvest(struct spu *spu)
2090 {
2091         acquire_spu_lock(spu);
2092         harvest(NULL, spu);
2093         release_spu_lock(spu);
2094 }
2095
2096 static void init_prob(struct spu_state *csa)
2097 {
2098         csa->spu_chnlcnt_RW[9] = 1;
2099         csa->spu_chnlcnt_RW[21] = 16;
2100         csa->spu_chnlcnt_RW[23] = 1;
2101         csa->spu_chnlcnt_RW[28] = 1;
2102         csa->spu_chnlcnt_RW[30] = 1;
2103         csa->prob.spu_runcntl_RW = SPU_RUNCNTL_STOP;
2104 }
2105
2106 static void init_priv1(struct spu_state *csa)
2107 {
2108         /* Enable decode, relocate, tlbie response, master runcntl. */
2109         csa->priv1.mfc_sr1_RW = MFC_STATE1_LOCAL_STORAGE_DECODE_MASK |
2110             MFC_STATE1_MASTER_RUN_CONTROL_MASK |
2111             MFC_STATE1_PROBLEM_STATE_MASK |
2112             MFC_STATE1_RELOCATE_MASK | MFC_STATE1_BUS_TLBIE_MASK;
2113
2114         /* Set storage description.  */
2115         csa->priv1.mfc_sdr_RW = mfspr(SPRN_SDR1);
2116
2117         /* Enable OS-specific set of interrupts. */
2118         csa->priv1.int_mask_class0_RW = CLASS0_ENABLE_DMA_ALIGNMENT_INTR |
2119             CLASS0_ENABLE_INVALID_DMA_COMMAND_INTR |
2120             CLASS0_ENABLE_SPU_ERROR_INTR;
2121         csa->priv1.int_mask_class1_RW = CLASS1_ENABLE_SEGMENT_FAULT_INTR |
2122             CLASS1_ENABLE_STORAGE_FAULT_INTR;
2123         csa->priv1.int_mask_class2_RW = CLASS2_ENABLE_SPU_STOP_INTR |
2124             CLASS2_ENABLE_SPU_HALT_INTR;
2125 }
2126
2127 static void init_priv2(struct spu_state *csa)
2128 {
2129         csa->priv2.spu_lslr_RW = LS_ADDR_MASK;
2130         csa->priv2.mfc_control_RW = MFC_CNTL_RESUME_DMA_QUEUE |
2131             MFC_CNTL_NORMAL_DMA_QUEUE_OPERATION |
2132             MFC_CNTL_DMA_QUEUES_EMPTY_MASK;
2133 }
2134
2135 /**
2136  * spu_alloc_csa - allocate and initialize an SPU context save area.
2137  *
2138  * Allocate and initialize the contents of an SPU context save area.
2139  * This includes enabling address translation, interrupt masks, etc.,
2140  * as appropriate for the given OS environment.
2141  *
2142  * Note that storage for the 'lscsa' is allocated separately,
2143  * as it is by far the largest of the context save regions,
2144  * and may need to be pinned or otherwise specially aligned.
2145  */
2146 void spu_init_csa(struct spu_state *csa)
2147 {
2148         struct spu_lscsa *lscsa;
2149         unsigned char *p;
2150
2151         if (!csa)
2152                 return;
2153         memset(csa, 0, sizeof(struct spu_state));
2154
2155         lscsa = vmalloc(sizeof(struct spu_lscsa));
2156         if (!lscsa)
2157                 return;
2158
2159         memset(lscsa, 0, sizeof(struct spu_lscsa));
2160         csa->lscsa = lscsa;
2161         csa->register_lock = SPIN_LOCK_UNLOCKED;
2162
2163         /* Set LS pages reserved to allow for user-space mapping. */
2164         for (p = lscsa->ls; p < lscsa->ls + LS_SIZE; p += PAGE_SIZE)
2165                 SetPageReserved(vmalloc_to_page(p));
2166
2167         init_prob(csa);
2168         init_priv1(csa);
2169         init_priv2(csa);
2170 }
2171
2172 void spu_fini_csa(struct spu_state *csa)
2173 {
2174         /* Clear reserved bit before vfree. */
2175         unsigned char *p;
2176         for (p = csa->lscsa->ls; p < csa->lscsa->ls + LS_SIZE; p += PAGE_SIZE)
2177                 ClearPageReserved(vmalloc_to_page(p));
2178
2179         vfree(csa->lscsa);
2180 }