admin/linux
1
0
Fork 0
mirror of https://github.com/torvalds/linux.git synced 2025-12-07 20:06:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

sched_ext: Factor out move_task_between_dsqs() from scx_dispatch_from_dsq()

Browse Source 
Pure reorganization. No functional changes.

Signed-off-by: Tejun Heo <tj@kernel.org>
This commit is contained in:
Tejun Heo
2024-09-24 14:08:52 -10:00
parent 95b873693a
commit 8427acb6b5
1 changed files with 75 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

116
kernel/sched/ext.c
View File

@@ -2369,6 +2369,73 @@ static inline bool task_can_run_on_remote_rq(struct task_struct *p, struct rq *r
static inline bool consume_remote_task(struct rq *this_rq, struct task_struct *p, struct scx_dispatch_q *dsq, struct rq *task_rq) { return false; }
#endif /* CONFIG_SMP */
/**
* move_task_between_dsqs() - Move a task from one DSQ to another
* @p: target task
* @enq_flags: %SCX_ENQ_*
* @src_dsq: DSQ @p is currently on, must not be a local DSQ
* @dst_dsq: DSQ @p is being moved to, can be any DSQ
*
* Must be called with @p's task_rq and @src_dsq locked. If @dst_dsq is a local
* DSQ and @p is on a different CPU, @p will be migrated and thus its task_rq
* will change. As @p's task_rq is locked, this function doesn't need to use the
* holding_cpu mechanism.
*
* On return, @src_dsq is unlocked and only @p's new task_rq, which is the
* return value, is locked.
*/
static struct rq *move_task_between_dsqs(struct task_struct *p, u64 enq_flags,
struct scx_dispatch_q *src_dsq,
struct scx_dispatch_q *dst_dsq)
{
struct rq *src_rq = task_rq(p), *dst_rq;
BUG_ON(src_dsq->id == SCX_DSQ_LOCAL);
lockdep_assert_held(&src_dsq->lock);
lockdep_assert_rq_held(src_rq);
if (dst_dsq->id == SCX_DSQ_LOCAL) {
dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq);
if (!task_can_run_on_remote_rq(p, dst_rq, true)) {
dst_dsq = find_global_dsq(p);
dst_rq = src_rq;
}
} else {
/* no need to migrate if destination is a non-local DSQ */
dst_rq = src_rq;
}
/*
* Move @p into $dst_dsq. If $dst_dsq is the local DSQ of a different
* CPU, @p will be migrated.
*/
if (dst_dsq->id == SCX_DSQ_LOCAL) {
/* @p is going from a non-local DSQ to a local DSQ */
if (src_rq == dst_rq) {
task_unlink_from_dsq(p, src_dsq);
move_local_task_to_local_dsq(p, enq_flags,
src_dsq, dst_rq);
raw_spin_unlock(&src_dsq->lock);
} else {
raw_spin_unlock(&src_dsq->lock);
move_remote_task_to_local_dsq(p, enq_flags,
src_rq, dst_rq);
}
} else {
/*
* @p is going from a non-local DSQ to a non-local DSQ. As
* $src_dsq is already locked, do an abbreviated dequeue.
*/
task_unlink_from_dsq(p, src_dsq);
p->scx.dsq = NULL;
raw_spin_unlock(&src_dsq->lock);
dispatch_enqueue(dst_dsq, p, enq_flags);
}
return dst_rq;
}
static bool consume_dispatch_q(struct rq *rq, struct scx_dispatch_q *dsq)
{
struct task_struct *p;
@@ -6033,7 +6100,7 @@ static bool scx_dispatch_from_dsq(struct bpf_iter_scx_dsq_kern *kit,
u64 enq_flags)
{
struct scx_dispatch_q *src_dsq = kit->dsq, *dst_dsq;
struct rq *this_rq, *src_rq, *dst_rq, *locked_rq;
struct rq *this_rq, *src_rq, *locked_rq;
bool dispatched = false;
bool in_balance;
unsigned long flags;
@@ -6079,51 +6146,18 @@ static bool scx_dispatch_from_dsq(struct bpf_iter_scx_dsq_kern *kit,
/* @p is still on $src_dsq and stable, determine the destination */
dst_dsq = find_dsq_for_dispatch(this_rq, dsq_id, p);
if (dst_dsq->id == SCX_DSQ_LOCAL) {
dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq);
if (!task_can_run_on_remote_rq(p, dst_rq, true)) {
dst_dsq = find_global_dsq(p);
dst_rq = src_rq;
}
} else {
/* no need to migrate if destination is a non-local DSQ */
dst_rq = src_rq;
}
/*
* Move @p into $dst_dsq. If $dst_dsq is the local DSQ of a different
* CPU, @p will be migrated.
* Apply vtime and slice updates before moving so that the new time is
* visible before inserting into $dst_dsq. @p is still on $src_dsq but
* this is safe as we're locking it.
*/
if (dst_dsq->id == SCX_DSQ_LOCAL) {
/* @p is going from a non-local DSQ to a local DSQ */
if (src_rq == dst_rq) {
task_unlink_from_dsq(p, src_dsq);
move_local_task_to_local_dsq(p, enq_flags,
src_dsq, dst_rq);
raw_spin_unlock(&src_dsq->lock);
} else {
raw_spin_unlock(&src_dsq->lock);
move_remote_task_to_local_dsq(p, enq_flags,
src_rq, dst_rq);
locked_rq = dst_rq;
}
} else {
/*
* @p is going from a non-local DSQ to a non-local DSQ. As
* $src_dsq is already locked, do an abbreviated dequeue.
*/
task_unlink_from_dsq(p, src_dsq);
p->scx.dsq = NULL;
raw_spin_unlock(&src_dsq->lock);
if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_VTIME)
p->scx.dsq_vtime = kit->vtime;
dispatch_enqueue(dst_dsq, p, enq_flags);
}
if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_VTIME)
p->scx.dsq_vtime = kit->vtime;
if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_SLICE)
p->scx.slice = kit->slice;
/* execute move */
locked_rq = move_task_between_dsqs(p, enq_flags, src_dsq, dst_dsq);
dispatched = true;
out:
if (in_balance) {