diff options
| author | Peter Zijlstra <peterz@infradead.org> | 2024-05-23 12:26:06 +0200 |
|---|---|---|
| committer | Peter Zijlstra <peterz@infradead.org> | 2024-08-17 11:06:44 +0200 |
| commit | 54a58a78779169f9c92a51facf6de7ce94962328 (patch) | |
| tree | 55d5622d6c8b753a0f17c927ed8ae12f006b38e5 | |
| parent | 152e11f6df293e816a6a37c69757033cdc72667d (diff) | |
sched/fair: Implement DELAY_ZERO
'Extend' DELAY_DEQUEUE by noting that since we wanted to dequeued them
at the 0-lag point, truncate lag (eg. don't let them earn positive
lag).
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Valentin Schneider <vschneid@redhat.com>
Tested-by: Valentin Schneider <vschneid@redhat.com>
Link: https://lkml.kernel.org/r/20240727105030.403750550@infradead.org
| -rw-r--r-- | kernel/sched/fair.c | 20 | ||||
| -rw-r--r-- | kernel/sched/features.h | 3 |
2 files changed, 21 insertions, 2 deletions
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index da5065a226ee..1a593393e308 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -5447,8 +5447,11 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) != DEQUEUE_SAVE) update_min_vruntime(cfs_rq); - if (flags & DEQUEUE_DELAYED) + if (flags & DEQUEUE_DELAYED) { se->sched_delayed = 0; + if (sched_feat(DELAY_ZERO) && se->vlag > 0) + se->vlag = 0; + } if (cfs_rq->nr_running == 0) update_idle_cfs_rq_clock_pelt(cfs_rq); @@ -5527,7 +5530,6 @@ pick_next_entity(struct rq *rq, struct cfs_rq *cfs_rq) dequeue_entities(rq, se, DEQUEUE_SLEEP | DEQUEUE_DELAYED); SCHED_WARN_ON(se->sched_delayed); SCHED_WARN_ON(se->on_rq); - return NULL; } return se; @@ -6825,6 +6827,20 @@ requeue_delayed_entity(struct sched_entity *se) SCHED_WARN_ON(!se->sched_delayed); SCHED_WARN_ON(!se->on_rq); + if (sched_feat(DELAY_ZERO)) { + update_entity_lag(cfs_rq, se); + if (se->vlag > 0) { + cfs_rq->nr_running--; + if (se != cfs_rq->curr) + __dequeue_entity(cfs_rq, se); + se->vlag = 0; + place_entity(cfs_rq, se, 0); + if (se != cfs_rq->curr) + __enqueue_entity(cfs_rq, se); + cfs_rq->nr_running++; + } + } + se->sched_delayed = 0; } diff --git a/kernel/sched/features.h b/kernel/sched/features.h index 1feaa7bbc278..7fdeb5576188 100644 --- a/kernel/sched/features.h +++ b/kernel/sched/features.h @@ -34,8 +34,11 @@ SCHED_FEAT(CACHE_HOT_BUDDY, true) * By delaying the dequeue for non-eligible tasks, they remain in the * competition and can burn off their negative lag. When they get selected * they'll have positive lag by definition. + * + * DELAY_ZERO clips the lag on dequeue (or wakeup) to 0. */ SCHED_FEAT(DELAY_DEQUEUE, true) +SCHED_FEAT(DELAY_ZERO, true) /* * Allow wakeup-time preemption of the current task: |