From 978b8409eab15aa733ae3a79c9b5158d34cd3fb7 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Tue, 20 Feb 2024 19:36:14 -1000 Subject: workqueue: Factor out work_grab_pending() from __cancel_work_sync() The planned disable/enable support will need the same logic. Let's factor it out. No functional changes. v2: Update function comment to include @irq_flags. Signed-off-by: Tejun Heo Reviewed-by: Lai Jiangshan --- kernel/workqueue.c | 132 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 80 insertions(+), 52 deletions(-) (limited to 'kernel/workqueue.c') diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 7c6915e23c5c..3b606eb5d6e3 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -484,6 +484,12 @@ static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS]; /* I: attributes used when instantiating ordered pools on demand */ static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS]; +/* + * Used to synchronize multiple cancel_sync attempts on the same work item. See + * work_grab_pending() and __cancel_work_sync(). + */ +static DECLARE_WAIT_QUEUE_HEAD(wq_cancel_waitq); + /* * I: kthread_worker to release pwq's. pwq release needs to be bounced to a * process context while holding a pool lock. Bounce to a dedicated kthread @@ -2147,6 +2153,75 @@ fail: return -EAGAIN; } +struct cwt_wait { + wait_queue_entry_t wait; + struct work_struct *work; +}; + +static int cwt_wakefn(wait_queue_entry_t *wait, unsigned mode, int sync, void *key) +{ + struct cwt_wait *cwait = container_of(wait, struct cwt_wait, wait); + + if (cwait->work != key) + return 0; + return autoremove_wake_function(wait, mode, sync, key); +} + +/** + * work_grab_pending - steal work item from worklist and disable irq + * @work: work item to steal + * @cflags: %WORK_CANCEL_ flags + * @irq_flags: place to store IRQ state + * + * Grab PENDING bit of @work. @work can be in any stable state - idle, on timer + * or on worklist. + * + * Must be called in process context. IRQ is disabled on return with IRQ state + * stored in *@irq_flags. The caller is responsible for re-enabling it using + * local_irq_restore(). + * + * Returns %true if @work was pending. %false if idle. + */ +static bool work_grab_pending(struct work_struct *work, u32 cflags, + unsigned long *irq_flags) +{ + struct cwt_wait cwait; + int ret; + + might_sleep(); +repeat: + ret = try_to_grab_pending(work, cflags, irq_flags); + if (likely(ret >= 0)) + return ret; + if (ret != -ENOENT) + goto repeat; + + /* + * Someone is already canceling. Wait for it to finish. flush_work() + * doesn't work for PREEMPT_NONE because we may get woken up between + * @work's completion and the other canceling task resuming and clearing + * CANCELING - flush_work() will return false immediately as @work is no + * longer busy, try_to_grab_pending() will return -ENOENT as @work is + * still being canceled and the other canceling task won't be able to + * clear CANCELING as we're hogging the CPU. + * + * Let's wait for completion using a waitqueue. As this may lead to the + * thundering herd problem, use a custom wake function which matches + * @work along with exclusive wait and wakeup. + */ + init_wait(&cwait.wait); + cwait.wait.func = cwt_wakefn; + cwait.work = work; + + prepare_to_wait_exclusive(&wq_cancel_waitq, &cwait.wait, + TASK_UNINTERRUPTIBLE); + if (work_is_canceling(work)) + schedule(); + finish_wait(&wq_cancel_waitq, &cwait.wait); + + goto repeat; +} + /** * insert_work - insert a work into a pool * @pwq: pwq @work belongs to @@ -4125,60 +4200,13 @@ static bool __cancel_work(struct work_struct *work, u32 cflags) return ret; } -struct cwt_wait { - wait_queue_entry_t wait; - struct work_struct *work; -}; - -static int cwt_wakefn(wait_queue_entry_t *wait, unsigned mode, int sync, void *key) -{ - struct cwt_wait *cwait = container_of(wait, struct cwt_wait, wait); - - if (cwait->work != key) - return 0; - return autoremove_wake_function(wait, mode, sync, key); -} - static bool __cancel_work_sync(struct work_struct *work, u32 cflags) { - static DECLARE_WAIT_QUEUE_HEAD(cancel_waitq); unsigned long irq_flags; - int ret; - - do { - ret = try_to_grab_pending(work, cflags, &irq_flags); - /* - * If someone else is already canceling, wait for it to - * finish. flush_work() doesn't work for PREEMPT_NONE - * because we may get scheduled between @work's completion - * and the other canceling task resuming and clearing - * CANCELING - flush_work() will return false immediately - * as @work is no longer busy, try_to_grab_pending() will - * return -ENOENT as @work is still being canceled and the - * other canceling task won't be able to clear CANCELING as - * we're hogging the CPU. - * - * Let's wait for completion using a waitqueue. As this - * may lead to the thundering herd problem, use a custom - * wake function which matches @work along with exclusive - * wait and wakeup. - */ - if (unlikely(ret == -ENOENT)) { - struct cwt_wait cwait; - - init_wait(&cwait.wait); - cwait.wait.func = cwt_wakefn; - cwait.work = work; - - prepare_to_wait_exclusive(&cancel_waitq, &cwait.wait, - TASK_UNINTERRUPTIBLE); - if (work_is_canceling(work)) - schedule(); - finish_wait(&cancel_waitq, &cwait.wait); - } - } while (unlikely(ret < 0)); + bool ret; - /* tell other tasks trying to grab @work to back off */ + /* claim @work and tell other tasks trying to grab @work to back off */ + ret = work_grab_pending(work, cflags, &irq_flags); mark_work_canceling(work); local_irq_restore(irq_flags); @@ -4197,8 +4225,8 @@ static bool __cancel_work_sync(struct work_struct *work, u32 cflags) * visible there. */ smp_mb(); - if (waitqueue_active(&cancel_waitq)) - __wake_up(&cancel_waitq, TASK_NORMAL, 1, work); + if (waitqueue_active(&wq_cancel_waitq)) + __wake_up(&wq_cancel_waitq, TASK_NORMAL, 1, work); return ret; } -- cgit v1.2.3