From 7c68af6e32c73992bad24107311f3433c89016e2 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Sat, 19 Sep 2009 09:40:22 +0300 Subject: core, x86: Add user return notifiers Add a general per-cpu notifier that is called whenever the kernel is about to return to userspace. The notifier uses a thread_info flag and existing checks, so there is no impact on user return or context switch fast paths. This will be used initially to speed up KVM task switching by lazily updating MSRs. Signed-off-by: Avi Kivity LKML-Reference: <1253342422-13811-1-git-send-email-avi@redhat.com> Signed-off-by: H. Peter Anvin --- kernel/user-return-notifier.c | 46 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 kernel/user-return-notifier.c (limited to 'kernel/user-return-notifier.c') diff --git a/kernel/user-return-notifier.c b/kernel/user-return-notifier.c new file mode 100644 index 000000000000..530ccb816513 --- /dev/null +++ b/kernel/user-return-notifier.c @@ -0,0 +1,46 @@ + +#include +#include +#include +#include + +static DEFINE_PER_CPU(struct hlist_head, return_notifier_list); + +#define URN_LIST_HEAD per_cpu(return_notifier_list, raw_smp_processor_id()) + +/* + * Request a notification when the current cpu returns to userspace. Must be + * called in atomic context. The notifier will also be called in atomic + * context. + */ +void user_return_notifier_register(struct user_return_notifier *urn) +{ + set_tsk_thread_flag(current, TIF_USER_RETURN_NOTIFY); + hlist_add_head(&urn->link, &URN_LIST_HEAD); +} +EXPORT_SYMBOL_GPL(user_return_notifier_register); + +/* + * Removes a registered user return notifier. Must be called from atomic + * context, and from the same cpu registration occured in. + */ +void user_return_notifier_unregister(struct user_return_notifier *urn) +{ + hlist_del(&urn->link); + if (hlist_empty(&URN_LIST_HEAD)) + clear_tsk_thread_flag(current, TIF_USER_RETURN_NOTIFY); +} +EXPORT_SYMBOL_GPL(user_return_notifier_unregister); + +/* Calls registered user return notifiers */ +void fire_user_return_notifiers(void) +{ + struct user_return_notifier *urn; + struct hlist_node *tmp1, *tmp2; + struct hlist_head *head; + + head = &get_cpu_var(return_notifier_list); + hlist_for_each_entry_safe(urn, tmp1, tmp2, head, link) + urn->on_user_return(urn); + put_cpu_var(); +} -- cgit v1.2.3 From 3c912b6edaac56cb451e7571c95c15cbb6bd0c81 Mon Sep 17 00:00:00 2001 From: Stephen Rothwell Date: Mon, 2 Nov 2009 16:17:22 +1100 Subject: x86: Fix user return notifier put_cpu_var() invocation Today's linux-next build (x86_64 allmodconfig) failed like this: kernel/user-return-notifier.c: In function 'fire_user_return_notifiers': kernel/user-return-notifier.c:45: error: expected expression before ')' token Introduced by commit 7c68af6e32c73992bad24107311f3433c89016e2 ("core, x86: Add user return notifiers") from the tip and kvm trees but revealed by commit e0fdb0e050eae331046385643618f12452aa7e73 ("percpu: add __percpu for sparse") from the percpu tree. Before that percpu tree commit, "put_cpu_var()" would compile without error (even though it really needs a parameter). Signed-off-by: Stephen Rothwell Cc: Avi Kivity Cc: Peter Zijlstra Cc: Tejun Heo Cc: Rusty Russell Cc: Christoph Lameter LKML-Reference: <20091102161722.eea4358d.sfr@canb.auug.org.au> Signed-off-by: Ingo Molnar --- kernel/user-return-notifier.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel/user-return-notifier.c') diff --git a/kernel/user-return-notifier.c b/kernel/user-return-notifier.c index 530ccb816513..03e2d6fd9b18 100644 --- a/kernel/user-return-notifier.c +++ b/kernel/user-return-notifier.c @@ -42,5 +42,5 @@ void fire_user_return_notifiers(void) head = &get_cpu_var(return_notifier_list); hlist_for_each_entry_safe(urn, tmp1, tmp2, head, link) urn->on_user_return(urn); - put_cpu_var(); + put_cpu_var(return_notifier_list); } -- cgit v1.2.3 From 1786bf009f18f722afbb62143c8541e7e60a4e92 Mon Sep 17 00:00:00 2001 From: Avi Kivity Date: Mon, 30 Nov 2009 12:54:51 +0200 Subject: core: Clean up user return notifers use of per_cpu Instead of using per_cpu(..., raw_smp_processor_id()), use __get_cpu_var(...). Signed-off-by: Avi Kivity LKML-Reference: <1259578491-4589-1-git-send-email-avi@redhat.com> Signed-off-by: Ingo Molnar --- kernel/user-return-notifier.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'kernel/user-return-notifier.c') diff --git a/kernel/user-return-notifier.c b/kernel/user-return-notifier.c index 03e2d6fd9b18..eb27fd3430a2 100644 --- a/kernel/user-return-notifier.c +++ b/kernel/user-return-notifier.c @@ -6,8 +6,6 @@ static DEFINE_PER_CPU(struct hlist_head, return_notifier_list); -#define URN_LIST_HEAD per_cpu(return_notifier_list, raw_smp_processor_id()) - /* * Request a notification when the current cpu returns to userspace. Must be * called in atomic context. The notifier will also be called in atomic @@ -16,7 +14,7 @@ static DEFINE_PER_CPU(struct hlist_head, return_notifier_list); void user_return_notifier_register(struct user_return_notifier *urn) { set_tsk_thread_flag(current, TIF_USER_RETURN_NOTIFY); - hlist_add_head(&urn->link, &URN_LIST_HEAD); + hlist_add_head(&urn->link, &__get_cpu_var(return_notifier_list)); } EXPORT_SYMBOL_GPL(user_return_notifier_register); @@ -27,7 +25,7 @@ EXPORT_SYMBOL_GPL(user_return_notifier_register); void user_return_notifier_unregister(struct user_return_notifier *urn) { hlist_del(&urn->link); - if (hlist_empty(&URN_LIST_HEAD)) + if (hlist_empty(&__get_cpu_var(return_notifier_list))) clear_tsk_thread_flag(current, TIF_USER_RETURN_NOTIFY); } EXPORT_SYMBOL_GPL(user_return_notifier_unregister); -- cgit v1.2.3