From 258ab3e9b264103d4eddc10ed7da69e6df8f2a3e Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 25 Nov 2009 16:31:54 +0900 Subject: sched: rename preempt_notifier to sched_notifier and always enable it Rename preempt_notifier to sched_notifier, move it from preempt.h to sched.h, drop sched_ prefixes from ops names and make sched_notifier always enabled. This is to prepare for adding more notification hooks. This patch doesn't make any functional changes. Signed-off-by: Tejun Heo Cc: Avi Kivity Cc: Peter Zijlstra --- kernel/sched.c | 64 +++++++++++++++++++--------------------------------------- 1 file changed, 21 insertions(+), 43 deletions(-) (limited to 'kernel/sched.c') diff --git a/kernel/sched.c b/kernel/sched.c index 3c91f110fc62..6561c3d35614 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -2528,10 +2528,7 @@ static void __sched_fork(struct task_struct *p) INIT_LIST_HEAD(&p->rt.run_list); p->se.on_rq = 0; INIT_LIST_HEAD(&p->se.group_node); - -#ifdef CONFIG_PREEMPT_NOTIFIERS - INIT_HLIST_HEAD(&p->preempt_notifiers); -#endif + INIT_HLIST_HEAD(&p->sched_notifiers); /* * We mark the process as running here, but have not actually @@ -2641,64 +2638,47 @@ void wake_up_new_task(struct task_struct *p, unsigned long clone_flags) task_rq_unlock(rq, &flags); } -#ifdef CONFIG_PREEMPT_NOTIFIERS - /** - * preempt_notifier_register - tell me when current is being preempted & rescheduled + * sched_notifier_register - register scheduler notifier * @notifier: notifier struct to register */ -void preempt_notifier_register(struct preempt_notifier *notifier) +void sched_notifier_register(struct sched_notifier *notifier) { - hlist_add_head(¬ifier->link, ¤t->preempt_notifiers); + hlist_add_head(¬ifier->link, ¤t->sched_notifiers); } -EXPORT_SYMBOL_GPL(preempt_notifier_register); +EXPORT_SYMBOL_GPL(sched_notifier_register); /** - * preempt_notifier_unregister - no longer interested in preemption notifications + * sched_notifier_unregister - unregister scheduler notifier * @notifier: notifier struct to unregister * - * This is safe to call from within a preemption notifier. + * This is safe to call from within a scheduler notifier. */ -void preempt_notifier_unregister(struct preempt_notifier *notifier) +void sched_notifier_unregister(struct sched_notifier *notifier) { hlist_del(¬ifier->link); } -EXPORT_SYMBOL_GPL(preempt_notifier_unregister); +EXPORT_SYMBOL_GPL(sched_notifier_unregister); -static void fire_sched_in_preempt_notifiers(struct task_struct *curr) +static void fire_sched_in_notifiers(struct task_struct *curr) { - struct preempt_notifier *notifier; + struct sched_notifier *notifier; struct hlist_node *node; - hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link) - notifier->ops->sched_in(notifier, raw_smp_processor_id()); + hlist_for_each_entry(notifier, node, &curr->sched_notifiers, link) + notifier->ops->in(notifier, raw_smp_processor_id()); } -static void -fire_sched_out_preempt_notifiers(struct task_struct *curr, - struct task_struct *next) +static void fire_sched_out_notifiers(struct task_struct *curr, + struct task_struct *next) { - struct preempt_notifier *notifier; + struct sched_notifier *notifier; struct hlist_node *node; - hlist_for_each_entry(notifier, node, &curr->preempt_notifiers, link) - notifier->ops->sched_out(notifier, next); -} - -#else /* !CONFIG_PREEMPT_NOTIFIERS */ - -static void fire_sched_in_preempt_notifiers(struct task_struct *curr) -{ + hlist_for_each_entry(notifier, node, &curr->sched_notifiers, link) + notifier->ops->out(notifier, next); } -static void -fire_sched_out_preempt_notifiers(struct task_struct *curr, - struct task_struct *next) -{ -} - -#endif /* CONFIG_PREEMPT_NOTIFIERS */ - /** * prepare_task_switch - prepare to switch tasks * @rq: the runqueue preparing to switch @@ -2716,7 +2696,7 @@ static inline void prepare_task_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next) { - fire_sched_out_preempt_notifiers(prev, next); + fire_sched_out_notifiers(prev, next); prepare_lock_switch(rq, next); prepare_arch_switch(next); } @@ -2758,7 +2738,7 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev) prev_state = prev->state; finish_arch_switch(prev); perf_event_task_sched_in(current, cpu_of(rq)); - fire_sched_in_preempt_notifiers(current); + fire_sched_in_notifiers(current); finish_lock_switch(rq, prev); if (mm) @@ -9540,9 +9520,7 @@ void __init sched_init(void) set_load_weight(&init_task); -#ifdef CONFIG_PREEMPT_NOTIFIERS - INIT_HLIST_HEAD(&init_task.preempt_notifiers); -#endif + INIT_HLIST_HEAD(&init_task.sched_notifiers); #ifdef CONFIG_SMP open_softirq(SCHED_SOFTIRQ, run_rebalance_domains); -- cgit v1.2.3 From 9b8e32f852153ac553440a758933539847b05c32 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 25 Nov 2009 16:31:54 +0900 Subject: sched: update sched_notifier and add wakeup/sleep notifications Update sched_notifier such that * in and out ops are symmetric in the parameter they take. * Use single fire_sched_notifier() macro instead of separate function for each op. * Allow NULL ops. * Add wakeup and sleep notifications. Signed-off-by: Tejun Heo Cc: Avi Kivity Cc: Peter Zijlstra --- include/linux/sched.h | 20 +++++++++++++------- kernel/sched.c | 41 ++++++++++++++++++----------------------- virt/kvm/kvm_main.c | 4 ++-- 3 files changed, 33 insertions(+), 32 deletions(-) (limited to 'kernel/sched.c') diff --git a/include/linux/sched.h b/include/linux/sched.h index 37c97a1f4bec..657372fad030 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1214,15 +1214,21 @@ struct sched_notifier; /** * sched_notifier_ops - notifiers called for scheduling events - * @in: we're about to be rescheduled: - * notifier: struct sched_notifier for the task being scheduled - * cpu: cpu we're scheduled on - * @out: we've just been preempted - * notifier: struct sched_notifier for the task being preempted - * next: the task that's kicking us out + * @wakeup: we're waking up + * notifier: struct sched_notifier for the task being woken up + * @sleep: we're going to bed + * notifier: struct sched_notifier for the task sleeping + * @in: we're now running on the cpu + * notifier: struct sched_notifier for the task being scheduled in + * prev: the task which ran before us + * @out: we're leaving the cpu + * notifier: struct sched_notifier for the task being scheduled out + * next: the task which will run after us */ struct sched_notifier_ops { - void (*in)(struct sched_notifier *notifier, int cpu); + void (*wakeup)(struct sched_notifier *notifier); + void (*sleep)(struct sched_notifier *notifier); + void (*in)(struct sched_notifier *notifier, struct task_struct *prev); void (*out)(struct sched_notifier *notifier, struct task_struct *next); }; diff --git a/kernel/sched.c b/kernel/sched.c index 6561c3d35614..cff8a9c4b0f4 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -1389,6 +1389,16 @@ static const u32 prio_to_wmult[40] = { /* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153, }; +#define fire_sched_notifier(p, callback, args...) do { \ + struct task_struct *__p = (p); \ + struct sched_notifier *__sn; \ + struct hlist_node *__pos; \ + \ + hlist_for_each_entry(__sn, __pos, &__p->sched_notifiers, link) \ + if (__sn->ops->callback) \ + __sn->ops->callback(__sn , ##args); \ +} while (0) + static void activate_task(struct rq *rq, struct task_struct *p, int wakeup); /* @@ -2444,6 +2454,8 @@ out_running: if (p->sched_class->task_wake_up) p->sched_class->task_wake_up(rq, p); #endif + if (success) + fire_sched_notifier(p, wakeup); out: task_rq_unlock(rq, &flags); put_cpu(); @@ -2660,25 +2672,6 @@ void sched_notifier_unregister(struct sched_notifier *notifier) } EXPORT_SYMBOL_GPL(sched_notifier_unregister); -static void fire_sched_in_notifiers(struct task_struct *curr) -{ - struct sched_notifier *notifier; - struct hlist_node *node; - - hlist_for_each_entry(notifier, node, &curr->sched_notifiers, link) - notifier->ops->in(notifier, raw_smp_processor_id()); -} - -static void fire_sched_out_notifiers(struct task_struct *curr, - struct task_struct *next) -{ - struct sched_notifier *notifier; - struct hlist_node *node; - - hlist_for_each_entry(notifier, node, &curr->sched_notifiers, link) - notifier->ops->out(notifier, next); -} - /** * prepare_task_switch - prepare to switch tasks * @rq: the runqueue preparing to switch @@ -2696,7 +2689,7 @@ static inline void prepare_task_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next) { - fire_sched_out_notifiers(prev, next); + fire_sched_notifier(current, out, next); prepare_lock_switch(rq, next); prepare_arch_switch(next); } @@ -2738,7 +2731,7 @@ static void finish_task_switch(struct rq *rq, struct task_struct *prev) prev_state = prev->state; finish_arch_switch(prev); perf_event_task_sched_in(current, cpu_of(rq)); - fire_sched_in_notifiers(current); + fire_sched_notifier(current, in, prev); finish_lock_switch(rq, prev); if (mm) @@ -5424,10 +5417,12 @@ need_resched_nonpreemptible: clear_tsk_need_resched(prev); if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) { - if (unlikely(signal_pending_state(prev->state, prev))) + if (unlikely(signal_pending_state(prev->state, prev))) { prev->state = TASK_RUNNING; - else + } else { + fire_sched_notifier(prev, sleep); deactivate_task(rq, prev, 1); + } switch_count = &prev->nvcsw; } diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 4e8e33fd76cd..006358ddc2e9 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -2695,11 +2695,11 @@ static inline struct kvm_vcpu *sched_notifier_to_vcpu(struct sched_notifier *sn) return container_of(sn, struct kvm_vcpu, sched_notifier); } -static void kvm_sched_in(struct sched_notifier *sn, int cpu) +static void kvm_sched_in(struct sched_notifier *sn, struct task_struct *prev) { struct kvm_vcpu *vcpu = sched_notifier_to_vcpu(sn); - kvm_arch_vcpu_load(vcpu, cpu); + kvm_arch_vcpu_load(vcpu, smp_processor_id()); } static void kvm_sched_out(struct sched_notifier *sn, struct task_struct *next) -- cgit v1.2.3 From 710c15b748f5ce9c573cc047f419cf007a677a9a Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 25 Nov 2009 16:31:54 +0900 Subject: sched: refactor try_to_wake_up() and implement try_to_wake_up_local() Factor ttwu_activate() and ttwu_woken_up() out of try_to_wake_up() and use them to implement try_to_wake_up_local(). try_to_wake_up_local() is similar to try_to_wake_up() but it assumes the caller has this_rq() locked and the target task is bound to this_rq(). try_to_wake_up_local() can be called from wakeup and sleep scheduler notifiers. The factoring out doesn't affect try_to_wake_up() much code-generation-wise. Depending on configuration options, it ends up generating the same object code as before or slightly different one due to different register assignment. The refactoring and local wake up function implementation using refactored functions are based on Peter Zijlstra's suggestion. Signed-off-by: Tejun Heo Cc: Peter Zijlstra --- include/linux/sched.h | 2 + kernel/sched.c | 148 +++++++++++++++++++++++++++++++++++++------------- 2 files changed, 111 insertions(+), 39 deletions(-) (limited to 'kernel/sched.c') diff --git a/include/linux/sched.h b/include/linux/sched.h index 657372fad030..5a9c14fa3386 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2057,6 +2057,8 @@ extern void release_uids(struct user_namespace *ns); extern void do_timer(unsigned long ticks); +extern bool try_to_wake_up_local(struct task_struct *p, unsigned int state, + int wake_flags); extern int wake_up_state(struct task_struct *tsk, unsigned int state); extern int wake_up_process(struct task_struct *tsk); extern void wake_up_new_task(struct task_struct *tsk, diff --git a/kernel/sched.c b/kernel/sched.c index cff8a9c4b0f4..5da2429be090 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -2334,11 +2334,62 @@ void task_oncpu_function_call(struct task_struct *p, preempt_enable(); } -/*** +static inline void ttwu_activate(struct task_struct *p, struct rq *rq, + bool is_sync, bool is_migrate, bool is_local) +{ + schedstat_inc(p, se.nr_wakeups); + if (is_sync) + schedstat_inc(p, se.nr_wakeups_sync); + if (is_migrate) + schedstat_inc(p, se.nr_wakeups_migrate); + if (is_local) + schedstat_inc(p, se.nr_wakeups_local); + else + schedstat_inc(p, se.nr_wakeups_remote); + + activate_task(rq, p, 1); + + /* + * Only attribute actual wakeups done by this task. + */ + if (!in_interrupt()) { + struct sched_entity *se = ¤t->se; + u64 sample = se->sum_exec_runtime; + + if (se->last_wakeup) + sample -= se->last_wakeup; + else + sample -= se->start_runtime; + update_avg(&se->avg_wakeup, sample); + + se->last_wakeup = se->sum_exec_runtime; + } +} + +static inline void ttwu_woken_up(struct task_struct *p, struct rq *rq, + int wake_flags, bool success) +{ + trace_sched_wakeup(rq, p, success); + check_preempt_curr(rq, p, wake_flags); + + p->state = TASK_RUNNING; +#ifdef CONFIG_SMP + if (p->sched_class->task_wake_up) + p->sched_class->task_wake_up(rq, p); +#endif + /* + * Wake up is complete, fire wake up notifier. This allows + * try_to_wake_up_local() to be called from wake up notifiers. + */ + if (success) + fire_sched_notifier(p, wakeup); +} + +/** * try_to_wake_up - wake up a thread * @p: the to-be-woken-up thread * @state: the mask of task states that can be woken - * @sync: do a synchronous wakeup? + * @wake_flags: wake modifier flags (WF_*) * * Put it on the run-queue if it's not already there. The "current" * thread is always on the run-queue (except when the actual @@ -2346,7 +2397,8 @@ void task_oncpu_function_call(struct task_struct *p, * the simpler "current->state = TASK_RUNNING" to mark yourself * runnable without the overhead of this. * - * returns failure only if the task is already active. + * Returns %true if @p was woken up, %false if it was already running + * or @state didn't match @p's state. */ static int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) @@ -2417,48 +2469,61 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, out_activate: #endif /* CONFIG_SMP */ - schedstat_inc(p, se.nr_wakeups); - if (wake_flags & WF_SYNC) - schedstat_inc(p, se.nr_wakeups_sync); - if (orig_cpu != cpu) - schedstat_inc(p, se.nr_wakeups_migrate); - if (cpu == this_cpu) - schedstat_inc(p, se.nr_wakeups_local); - else - schedstat_inc(p, se.nr_wakeups_remote); - activate_task(rq, p, 1); + ttwu_activate(p, rq, wake_flags & WF_SYNC, orig_cpu != cpu, + cpu == this_cpu); success = 1; +out_running: + ttwu_woken_up(p, rq, wake_flags, success); +out: + task_rq_unlock(rq, &flags); + put_cpu(); - /* - * Only attribute actual wakeups done by this task. - */ - if (!in_interrupt()) { - struct sched_entity *se = ¤t->se; - u64 sample = se->sum_exec_runtime; + return success; +} - if (se->last_wakeup) - sample -= se->last_wakeup; - else - sample -= se->start_runtime; - update_avg(&se->avg_wakeup, sample); +/** + * try_to_wake_up_local - try to wake up a local task with rq lock held + * @p: the to-be-woken-up thread + * @state: the mask of task states that can be woken + * @wake_flags: wake modifier flags (WF_*) + * + * Put @p on the run-queue if it's not alredy there. The caller must + * ensure that this_rq() is locked, @p is bound to this_rq() and @p is + * not the current task. this_rq() stays locked over invocation. + * + * This function can be called from wakeup and sleep scheduler + * notifiers. Be careful not to create deep recursion by chaining + * wakeup notifiers. + * + * Returns %true if @p was woken up, %false if it was already running + * or @state didn't match @p's state. + */ +bool try_to_wake_up_local(struct task_struct *p, unsigned int state, + int wake_flags) +{ + struct rq *rq = task_rq(p); + bool success = false; - se->last_wakeup = se->sum_exec_runtime; - } + BUG_ON(rq != this_rq()); + BUG_ON(p == current); + lockdep_assert_held(&rq->lock); -out_running: - trace_sched_wakeup(rq, p, success); - check_preempt_curr(rq, p, wake_flags); + if (!sched_feat(SYNC_WAKEUPS)) + wake_flags &= ~WF_SYNC; - p->state = TASK_RUNNING; -#ifdef CONFIG_SMP - if (p->sched_class->task_wake_up) - p->sched_class->task_wake_up(rq, p); -#endif - if (success) - fire_sched_notifier(p, wakeup); -out: - task_rq_unlock(rq, &flags); - put_cpu(); + if (!(p->state & state)) + return false; + + if (!p->se.on_rq) { + if (likely(!task_running(rq, p))) { + schedstat_inc(rq, ttwu_count); + schedstat_inc(rq, ttwu_local); + } + ttwu_activate(p, rq, wake_flags & WF_SYNC, false, true); + success = true; + } + + ttwu_woken_up(p, rq, wake_flags, success); return success; } @@ -5420,6 +5485,11 @@ need_resched_nonpreemptible: if (unlikely(signal_pending_state(prev->state, prev))) { prev->state = TASK_RUNNING; } else { + /* + * Fire sleep notifier before changing any scheduler + * state. This allows try_to_wake_up_local() to be + * called from sleep notifiers. + */ fire_sched_notifier(prev, sleep); deactivate_task(rq, prev, 1); } -- cgit v1.2.3 From 1af295d613212bbef19b58b155414f268a576d6d Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Wed, 25 Nov 2009 16:31:54 +0900 Subject: sched: implement force_cpus_allowed() set_cpus_allowed_ptr() modifies the allowed cpu mask of a task. The function performs the following checks before applying new mask. * Check whether PF_THREAD_BOUND is set. This is set for bound kthreads so that they can't be moved around. * Check whether the target cpu is still marked active - cpu_active(). Active state is cleared early while downing a cpu. This patch adds force_cpus_allowed() which bypasses the above two checks. The caller is responsible for guaranteeing that the destination cpu doesn't go down until force_cpus_allowed() finishes. The first check is bypassed by factoring out actual migration part into __set_cpus_allowed() from set_cpus_allowed_ptr() and calling the inner function from force_cpus_allowed(). The second check is buried deep down in __migrate_task() which is executed by migration threads. @force parameter is added to __migrate_task(). As the only way to pass parameters from __set_cpus_allowed() is through migration_req, migration_req->force is added and the @force parameter is passed down to __migrate_task(). Please note the naming discrepancy between set_cpus_allowed_ptr() and the new functions. The _ptr suffix is from the days when cpumask api wasn't mature and future changes should drop it from set_cpus_allowed_ptr() too. force_cpus_allowed() will be used for concurrency-managed workqueue. Signed-off-by: Tejun Heo Cc: Rusty Russell Cc: Peter Zijlstra --- include/linux/sched.h | 7 ++++ kernel/sched.c | 89 ++++++++++++++++++++++++++++++++++----------------- 2 files changed, 67 insertions(+), 29 deletions(-) (limited to 'kernel/sched.c') diff --git a/include/linux/sched.h b/include/linux/sched.h index 5a9c14fa3386..2a3c0dcfbb11 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -1851,6 +1851,8 @@ static inline void rcu_copy_process(struct task_struct *p) #ifdef CONFIG_SMP extern int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask); +extern int force_cpus_allowed(struct task_struct *p, + const struct cpumask *new_mask); #else static inline int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) @@ -1859,6 +1861,11 @@ static inline int set_cpus_allowed_ptr(struct task_struct *p, return -EINVAL; return 0; } +static inline int force_cpus_allowed(struct task_struct *p, + const struct cpumask *new_mask) +{ + return set_cpus_allowed_ptr(p, new_mask); +} #endif #ifndef CONFIG_CPUMASK_OFFSTACK diff --git a/kernel/sched.c b/kernel/sched.c index 5da2429be090..e488e07a2866 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -2108,6 +2108,7 @@ struct migration_req { struct task_struct *task; int dest_cpu; + bool force; struct completion done; }; @@ -2116,8 +2117,8 @@ struct migration_req { * The task's runqueue lock must be held. * Returns true if you have to wait for migration thread. */ -static int -migrate_task(struct task_struct *p, int dest_cpu, struct migration_req *req) +static int migrate_task(struct task_struct *p, int dest_cpu, + struct migration_req *req, bool force) { struct rq *rq = task_rq(p); @@ -2134,6 +2135,7 @@ migrate_task(struct task_struct *p, int dest_cpu, struct migration_req *req) init_completion(&req->done); req->task = p; req->dest_cpu = dest_cpu; + req->force = force; list_add(&req->list, &rq->migration_queue); return 1; @@ -3170,7 +3172,7 @@ static void sched_migrate_task(struct task_struct *p, int dest_cpu) goto out; /* force the process onto the specified CPU */ - if (migrate_task(p, dest_cpu, &req)) { + if (migrate_task(p, dest_cpu, &req, false)) { /* Need to wait for migration thread (might exit: take ref). */ struct task_struct *mt = rq->migration_thread; @@ -7090,34 +7092,19 @@ static inline void sched_init_granularity(void) * 7) we wake up and the migration is done. */ -/* - * Change a given task's CPU affinity. Migrate the thread to a - * proper CPU and schedule it away if the CPU it's executing on - * is removed from the allowed bitmask. - * - * NOTE: the caller must have a valid reference to the task, the - * task must not exit() & deallocate itself prematurely. The - * call is not atomic; no spinlocks may be held. - */ -int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) +static inline int __set_cpus_allowed(struct task_struct *p, + const struct cpumask *new_mask, + struct rq *rq, unsigned long *flags, + bool force) { struct migration_req req; - unsigned long flags; - struct rq *rq; int ret = 0; - rq = task_rq_lock(p, &flags); if (!cpumask_intersects(new_mask, cpu_online_mask)) { ret = -EINVAL; goto out; } - if (unlikely((p->flags & PF_THREAD_BOUND) && p != current && - !cpumask_equal(&p->cpus_allowed, new_mask))) { - ret = -EINVAL; - goto out; - } - if (p->sched_class->set_cpus_allowed) p->sched_class->set_cpus_allowed(p, new_mask); else { @@ -7129,12 +7116,13 @@ int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) if (cpumask_test_cpu(task_cpu(p), new_mask)) goto out; - if (migrate_task(p, cpumask_any_and(cpu_online_mask, new_mask), &req)) { + if (migrate_task(p, cpumask_any_and(cpu_online_mask, new_mask), &req, + force)) { /* Need help from migration thread: drop lock and wait. */ struct task_struct *mt = rq->migration_thread; get_task_struct(mt); - task_rq_unlock(rq, &flags); + task_rq_unlock(rq, flags); wake_up_process(rq->migration_thread); put_task_struct(mt); wait_for_completion(&req.done); @@ -7142,12 +7130,53 @@ int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) return 0; } out: - task_rq_unlock(rq, &flags); + task_rq_unlock(rq, flags); return ret; } + +/* + * Change a given task's CPU affinity. Migrate the thread to a + * proper CPU and schedule it away if the CPU it's executing on + * is removed from the allowed bitmask. + * + * NOTE: the caller must have a valid reference to the task, the + * task must not exit() & deallocate itself prematurely. The + * call is not atomic; no spinlocks may be held. + */ +int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) +{ + unsigned long flags; + struct rq *rq; + + rq = task_rq_lock(p, &flags); + + if (unlikely((p->flags & PF_THREAD_BOUND) && p != current && + !cpumask_equal(&p->cpus_allowed, new_mask))) { + task_rq_unlock(rq, &flags); + return -EINVAL; + } + + return __set_cpus_allowed(p, new_mask, rq, &flags, false); +} EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr); +/* + * Similar to set_cpus_allowed_ptr() but bypasses PF_THREAD_BOUND + * check and ignores cpu_active() status as long as the cpu is online. + * The caller is responsible for guaranteeing that the destination + * cpus don't go down until this function finishes and in general + * ensuring things don't go bonkers. + */ +int force_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) +{ + unsigned long flags; + struct rq *rq; + + rq = task_rq_lock(p, &flags); + return __set_cpus_allowed(p, new_mask, rq, &flags, true); +} + /* * Move (not current) task off this cpu, onto dest cpu. We're doing * this because either it can't run here any more (set_cpus_allowed() @@ -7159,12 +7188,13 @@ EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr); * * Returns non-zero if task was successfully migrated. */ -static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu) +static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu, + bool force) { struct rq *rq_dest, *rq_src; int ret = 0, on_rq; - if (unlikely(!cpu_active(dest_cpu))) + if (!force && unlikely(!cpu_active(dest_cpu))) return ret; rq_src = cpu_rq(src_cpu); @@ -7243,7 +7273,8 @@ static int migration_thread(void *data) if (req->task != NULL) { spin_unlock(&rq->lock); - __migrate_task(req->task, cpu, req->dest_cpu); + __migrate_task(req->task, cpu, req->dest_cpu, + req->force); } else if (likely(cpu == (badcpu = smp_processor_id()))) { req->dest_cpu = RCU_MIGRATION_GOT_QS; spin_unlock(&rq->lock); @@ -7268,7 +7299,7 @@ static int __migrate_task_irq(struct task_struct *p, int src_cpu, int dest_cpu) int ret; local_irq_disable(); - ret = __migrate_task(p, src_cpu, dest_cpu); + ret = __migrate_task(p, src_cpu, dest_cpu, false); local_irq_enable(); return ret; } -- cgit v1.2.3