1 /* sched.c - SPU scheduler.
3 * Copyright (C) IBM 2005
4 * Author: Mark Nutter <mnutter@us.ibm.com>
6 * 2006-03-31 NUMA domains added.
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 2, or (at your option)
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, write to the Free Software
20 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
25 #include <linux/module.h>
26 #include <linux/errno.h>
27 #include <linux/sched.h>
28 #include <linux/kernel.h>
30 #include <linux/completion.h>
31 #include <linux/vmalloc.h>
32 #include <linux/smp.h>
33 #include <linux/smp_lock.h>
34 #include <linux/stddef.h>
35 #include <linux/unistd.h>
36 #include <linux/numa.h>
37 #include <linux/mutex.h>
38 #include <linux/notifier.h>
41 #include <asm/mmu_context.h>
43 #include <asm/spu_csa.h>
44 #include <asm/spu_priv1.h>
47 #define SPU_MIN_TIMESLICE (100 * HZ / 1000)
49 #define SPU_BITMAP_SIZE (((MAX_PRIO+BITS_PER_LONG)/BITS_PER_LONG)+1)
50 struct spu_prio_array {
51 unsigned long bitmap[SPU_BITMAP_SIZE];
52 wait_queue_head_t waitq[MAX_PRIO];
53 struct list_head active_list[MAX_NUMNODES];
54 struct mutex active_mutex[MAX_NUMNODES];
57 static struct spu_prio_array *spu_prio;
59 static inline int node_allowed(int node)
63 if (!nr_cpus_node(node))
65 mask = node_to_cpumask(node);
66 if (!cpus_intersects(mask, current->cpus_allowed))
72 * spu_add_to_active_list - add spu to active list
73 * @spu: spu to add to the active list
75 static void spu_add_to_active_list(struct spu *spu)
77 mutex_lock(&spu_prio->active_mutex[spu->node]);
78 list_add_tail(&spu->list, &spu_prio->active_list[spu->node]);
79 mutex_unlock(&spu_prio->active_mutex[spu->node]);
83 * spu_remove_from_active_list - remove spu from active list
84 * @spu: spu to remove from the active list
86 * This function removes an spu from the active list. If the spu was
87 * found on the active list the function returns 1, else it doesn't do
88 * anything and returns 0.
90 static int spu_remove_from_active_list(struct spu *spu)
96 mutex_lock(&spu_prio->active_mutex[node]);
97 list_for_each_entry(tmp, &spu_prio->active_list[node], list) {
99 list_del_init(&spu->list);
104 mutex_unlock(&spu_prio->active_mutex[node]);
108 static inline void mm_needs_global_tlbie(struct mm_struct *mm)
110 int nr = (NR_CPUS > 1) ? NR_CPUS : NR_CPUS + 1;
112 /* Global TLBIE broadcast required with SPEs. */
113 __cpus_setall(&mm->cpu_vm_mask, nr);
116 static BLOCKING_NOTIFIER_HEAD(spu_switch_notifier);
118 static void spu_switch_notify(struct spu *spu, struct spu_context *ctx)
120 blocking_notifier_call_chain(&spu_switch_notifier,
121 ctx ? ctx->object_id : 0, spu);
124 int spu_switch_event_register(struct notifier_block * n)
126 return blocking_notifier_chain_register(&spu_switch_notifier, n);
129 int spu_switch_event_unregister(struct notifier_block * n)
131 return blocking_notifier_chain_unregister(&spu_switch_notifier, n);
135 * spu_bind_context - bind spu context to physical spu
136 * @spu: physical spu to bind to
137 * @ctx: context to bind
139 static void spu_bind_context(struct spu *spu, struct spu_context *ctx)
141 pr_debug("%s: pid=%d SPU=%d NODE=%d\n", __FUNCTION__, current->pid,
142 spu->number, spu->node);
146 ctx->ops = &spu_hw_ops;
147 spu->pid = current->pid;
148 spu->mm = ctx->owner;
149 mm_needs_global_tlbie(spu->mm);
150 spu->ibox_callback = spufs_ibox_callback;
151 spu->wbox_callback = spufs_wbox_callback;
152 spu->stop_callback = spufs_stop_callback;
153 spu->mfc_callback = spufs_mfc_callback;
154 spu->dma_callback = spufs_dma_callback;
156 spu_unmap_mappings(ctx);
157 spu_restore(&ctx->csa, spu);
158 spu->timestamp = jiffies;
159 spu_cpu_affinity_set(spu, raw_smp_processor_id());
160 spu_switch_notify(spu, ctx);
161 spu_add_to_active_list(spu);
162 ctx->state = SPU_STATE_RUNNABLE;
166 * spu_unbind_context - unbind spu context from physical spu
167 * @spu: physical spu to unbind from
168 * @ctx: context to unbind
170 * If the spu was on the active list the function returns 1, else 0.
172 static int spu_unbind_context(struct spu *spu, struct spu_context *ctx)
174 int was_active = spu_remove_from_active_list(spu);
176 pr_debug("%s: unbind pid=%d SPU=%d NODE=%d\n", __FUNCTION__,
177 spu->pid, spu->number, spu->node);
179 spu_switch_notify(spu, NULL);
180 spu_unmap_mappings(ctx);
181 spu_save(&ctx->csa, spu);
182 spu->timestamp = jiffies;
183 ctx->state = SPU_STATE_SAVED;
184 spu->ibox_callback = NULL;
185 spu->wbox_callback = NULL;
186 spu->stop_callback = NULL;
187 spu->mfc_callback = NULL;
188 spu->dma_callback = NULL;
191 ctx->ops = &spu_backing_ops;
199 static inline void spu_add_wq(wait_queue_head_t * wq, wait_queue_t * wait,
202 prepare_to_wait_exclusive(wq, wait, TASK_INTERRUPTIBLE);
203 set_bit(prio, spu_prio->bitmap);
206 static inline void spu_del_wq(wait_queue_head_t * wq, wait_queue_t * wait,
211 __set_current_state(TASK_RUNNING);
213 spin_lock_irqsave(&wq->lock, flags);
215 remove_wait_queue_locked(wq, wait);
216 if (list_empty(&wq->task_list))
217 clear_bit(prio, spu_prio->bitmap);
219 spin_unlock_irqrestore(&wq->lock, flags);
222 static void spu_prio_wait(struct spu_context *ctx, u64 flags)
224 int prio = ctx->prio;
225 wait_queue_head_t *wq = &spu_prio->waitq[prio];
231 spu_add_wq(wq, &wait, prio);
233 if (!signal_pending(current)) {
234 mutex_unlock(&ctx->state_mutex);
235 pr_debug("%s: pid=%d prio=%d\n", __FUNCTION__,
236 current->pid, current->prio);
238 mutex_lock(&ctx->state_mutex);
241 spu_del_wq(wq, &wait, prio);
244 static void spu_prio_wakeup(void)
246 int best = sched_find_first_bit(spu_prio->bitmap);
247 if (best < MAX_PRIO) {
248 wait_queue_head_t *wq = &spu_prio->waitq[best];
249 wake_up_interruptible_nr(wq, 1);
253 static struct spu *spu_get_idle(struct spu_context *ctx, u64 flags)
255 struct spu *spu = NULL;
256 int node = cpu_to_node(raw_smp_processor_id());
259 for (n = 0; n < MAX_NUMNODES; n++, node++) {
260 node = (node < MAX_NUMNODES) ? node : 0;
261 if (!node_allowed(node))
263 spu = spu_alloc_node(node);
270 static inline struct spu *spu_get(struct spu_context *ctx, u64 flags)
272 /* Future: spu_get_idle() if possible,
273 * otherwise try to preempt an active
276 return spu_get_idle(ctx, flags);
279 /* The three externally callable interfaces
280 * for the scheduler begin here.
282 * spu_activate - bind a context to SPU, waiting as needed.
283 * spu_deactivate - unbind a context from its SPU.
284 * spu_yield - yield an SPU if others are waiting.
287 int spu_activate(struct spu_context *ctx, u64 flags)
295 spu = spu_get(ctx, flags);
297 if (ctx->spu != NULL) {
302 spu_bind_context(spu, ctx);
305 spu_prio_wait(ctx, flags);
306 if (signal_pending(current)) {
315 void spu_deactivate(struct spu_context *ctx)
323 was_active = spu_unbind_context(spu, ctx);
330 void spu_yield(struct spu_context *ctx)
335 if (mutex_trylock(&ctx->state_mutex)) {
336 if ((spu = ctx->spu) != NULL) {
337 int best = sched_find_first_bit(spu_prio->bitmap);
338 if (best < MAX_PRIO) {
339 pr_debug("%s: yielding SPU %d NODE %d\n",
340 __FUNCTION__, spu->number, spu->node);
345 mutex_unlock(&ctx->state_mutex);
347 if (unlikely(need_yield))
351 int __init spu_sched_init(void)
355 spu_prio = kzalloc(sizeof(struct spu_prio_array), GFP_KERNEL);
357 printk(KERN_WARNING "%s: Unable to allocate priority queue.\n",
361 for (i = 0; i < MAX_PRIO; i++) {
362 init_waitqueue_head(&spu_prio->waitq[i]);
363 __clear_bit(i, spu_prio->bitmap);
365 __set_bit(MAX_PRIO, spu_prio->bitmap);
366 for (i = 0; i < MAX_NUMNODES; i++) {
367 mutex_init(&spu_prio->active_mutex[i]);
368 INIT_LIST_HEAD(&spu_prio->active_list[i]);
373 void __exit spu_sched_exit(void)
375 struct spu *spu, *tmp;
378 for (node = 0; node < MAX_NUMNODES; node++) {
379 mutex_lock(&spu_prio->active_mutex[node]);
380 list_for_each_entry_safe(spu, tmp, &spu_prio->active_list[node],
382 list_del_init(&spu->list);
385 mutex_unlock(&spu_prio->active_mutex[node]);