582 lines
18 KiB
Diff
582 lines
18 KiB
Diff
Subject: sched: Add support for lazy preemption
|
|
From: Thomas Gleixner <tglx@linutronix.de>
|
|
Date: Fri, 26 Oct 2012 18:50:54 +0100
|
|
|
|
It has become an obsession to mitigate the determinism vs. throughput
|
|
loss of RT. Looking at the mainline semantics of preemption points
|
|
gives a hint why RT sucks throughput wise for ordinary SCHED_OTHER
|
|
tasks. One major issue is the wakeup of tasks which are right away
|
|
preempting the waking task while the waking task holds a lock on which
|
|
the woken task will block right after having preempted the wakee. In
|
|
mainline this is prevented due to the implicit preemption disable of
|
|
spin/rw_lock held regions. On RT this is not possible due to the fully
|
|
preemptible nature of sleeping spinlocks.
|
|
|
|
Though for a SCHED_OTHER task preempting another SCHED_OTHER task this
|
|
is really not a correctness issue. RT folks are concerned about
|
|
SCHED_FIFO/RR tasks preemption and not about the purely fairness
|
|
driven SCHED_OTHER preemption latencies.
|
|
|
|
So I introduced a lazy preemption mechanism which only applies to
|
|
SCHED_OTHER tasks preempting another SCHED_OTHER task. Aside of the
|
|
existing preempt_count each tasks sports now a preempt_lazy_count
|
|
which is manipulated on lock acquiry and release. This is slightly
|
|
incorrect as for lazyness reasons I coupled this on
|
|
migrate_disable/enable so some other mechanisms get the same treatment
|
|
(e.g. get_cpu_light).
|
|
|
|
Now on the scheduler side instead of setting NEED_RESCHED this sets
|
|
NEED_RESCHED_LAZY in case of a SCHED_OTHER/SCHED_OTHER preemption and
|
|
therefor allows to exit the waking task the lock held region before
|
|
the woken task preempts. That also works better for cross CPU wakeups
|
|
as the other side can stay in the adaptive spinning loop.
|
|
|
|
For RT class preemption there is no change. This simply sets
|
|
NEED_RESCHED and forgoes the lazy preemption counter.
|
|
|
|
Initial test do not expose any observable latency increasement, but
|
|
history shows that I've been proven wrong before :)
|
|
|
|
The lazy preemption mode is per default on, but with
|
|
CONFIG_SCHED_DEBUG enabled it can be disabled via:
|
|
|
|
# echo NO_PREEMPT_LAZY >/sys/kernel/debug/sched_features
|
|
|
|
and reenabled via
|
|
|
|
# echo PREEMPT_LAZY >/sys/kernel/debug/sched_features
|
|
|
|
The test results so far are very machine and workload dependent, but
|
|
there is a clear trend that it enhances the non RT workload
|
|
performance.
|
|
|
|
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
|
|
---
|
|
include/linux/ftrace_event.h | 1
|
|
include/linux/preempt.h | 38 ++++++++++++++++++++++++++-
|
|
include/linux/sched.h | 51 ++++++++++++++++++++++++++++++++----
|
|
kernel/Kconfig.preempt | 6 ++++
|
|
kernel/sched/core.c | 60 ++++++++++++++++++++++++++++++++++++++++++-
|
|
kernel/sched/fair.c | 16 +++++------
|
|
kernel/sched/features.h | 3 ++
|
|
kernel/sched/sched.h | 9 ++++++
|
|
kernel/trace/trace.c | 41 +++++++++++++++++------------
|
|
kernel/trace/trace.h | 2 +
|
|
kernel/trace/trace_output.c | 13 +++++++--
|
|
11 files changed, 206 insertions(+), 34 deletions(-)
|
|
|
|
--- a/include/linux/ftrace_event.h
|
|
+++ b/include/linux/ftrace_event.h
|
|
@@ -51,6 +51,7 @@ struct trace_entry {
|
|
int pid;
|
|
unsigned short migrate_disable;
|
|
unsigned short padding;
|
|
+ unsigned char preempt_lazy_count;
|
|
};
|
|
|
|
#define FTRACE_MAX_EVENT \
|
|
--- a/include/linux/preempt.h
|
|
+++ b/include/linux/preempt.h
|
|
@@ -23,15 +23,38 @@
|
|
|
|
#define preempt_count() (current_thread_info()->preempt_count)
|
|
|
|
+#ifdef CONFIG_PREEMPT_LAZY
|
|
+#define add_preempt_lazy_count(val) do { preempt_lazy_count() += (val); } while (0)
|
|
+#define sub_preempt_lazy_count(val) do { preempt_lazy_count() -= (val); } while (0)
|
|
+#define inc_preempt_lazy_count() add_preempt_lazy_count(1)
|
|
+#define dec_preempt_lazy_count() sub_preempt_lazy_count(1)
|
|
+#define preempt_lazy_count() (current_thread_info()->preempt_lazy_count)
|
|
+#else
|
|
+#define add_preempt_lazy_count(val) do { } while (0)
|
|
+#define sub_preempt_lazy_count(val) do { } while (0)
|
|
+#define inc_preempt_lazy_count() do { } while (0)
|
|
+#define dec_preempt_lazy_count() do { } while (0)
|
|
+#define preempt_lazy_count() (0)
|
|
+#endif
|
|
+
|
|
#ifdef CONFIG_PREEMPT
|
|
|
|
asmlinkage void preempt_schedule(void);
|
|
|
|
+# ifdef CONFIG_PREEMPT_LAZY
|
|
+#define preempt_check_resched() \
|
|
+do { \
|
|
+ if (unlikely(test_thread_flag(TIF_NEED_RESCHED) || \
|
|
+ test_thread_flag(TIF_NEED_RESCHED_LAZY))) \
|
|
+ preempt_schedule(); \
|
|
+} while (0)
|
|
+# else
|
|
#define preempt_check_resched() \
|
|
do { \
|
|
- if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) \
|
|
+ if (unlikely(test_thread_flag(TIF_NEED_RESCHED))) \
|
|
preempt_schedule(); \
|
|
} while (0)
|
|
+# endif
|
|
|
|
#else /* !CONFIG_PREEMPT */
|
|
|
|
@@ -48,6 +71,12 @@ do { \
|
|
barrier(); \
|
|
} while (0)
|
|
|
|
+#define preempt_lazy_disable() \
|
|
+do { \
|
|
+ inc_preempt_lazy_count(); \
|
|
+ barrier(); \
|
|
+} while (0)
|
|
+
|
|
#define sched_preempt_enable_no_resched() \
|
|
do { \
|
|
barrier(); \
|
|
@@ -68,6 +97,13 @@ do { \
|
|
barrier(); \
|
|
preempt_check_resched(); \
|
|
} while (0)
|
|
+
|
|
+#define preempt_lazy_enable() \
|
|
+do { \
|
|
+ dec_preempt_lazy_count(); \
|
|
+ barrier(); \
|
|
+ preempt_check_resched(); \
|
|
+} while (0)
|
|
|
|
/* For debugging and tracer internals only! */
|
|
#define add_preempt_count_notrace(val) \
|
|
--- a/include/linux/sched.h
|
|
+++ b/include/linux/sched.h
|
|
@@ -2692,6 +2692,52 @@ static inline int test_tsk_need_resched(
|
|
return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED));
|
|
}
|
|
|
|
+#ifdef CONFIG_PREEMPT_LAZY
|
|
+static inline void set_tsk_need_resched_lazy(struct task_struct *tsk)
|
|
+{
|
|
+ set_tsk_thread_flag(tsk,TIF_NEED_RESCHED_LAZY);
|
|
+}
|
|
+
|
|
+static inline void clear_tsk_need_resched_lazy(struct task_struct *tsk)
|
|
+{
|
|
+ clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED_LAZY);
|
|
+}
|
|
+
|
|
+static inline int test_tsk_need_resched_lazy(struct task_struct *tsk)
|
|
+{
|
|
+ return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED_LAZY));
|
|
+}
|
|
+
|
|
+static inline int need_resched_lazy(void)
|
|
+{
|
|
+ return test_thread_flag(TIF_NEED_RESCHED_LAZY);
|
|
+}
|
|
+
|
|
+static inline int need_resched_now(void)
|
|
+{
|
|
+ return test_thread_flag(TIF_NEED_RESCHED);
|
|
+}
|
|
+
|
|
+static inline int need_resched(void)
|
|
+{
|
|
+ return test_thread_flag(TIF_NEED_RESCHED) ||
|
|
+ test_thread_flag(TIF_NEED_RESCHED_LAZY);
|
|
+}
|
|
+#else
|
|
+static inline void clear_tsk_need_resched_lazy(struct task_struct *tsk) { }
|
|
+static inline int need_resched_lazy(void) { return 0; }
|
|
+
|
|
+static inline int need_resched_now(void)
|
|
+{
|
|
+ return test_thread_flag(TIF_NEED_RESCHED);
|
|
+}
|
|
+
|
|
+static inline int need_resched(void)
|
|
+{
|
|
+ return test_thread_flag(TIF_NEED_RESCHED);
|
|
+}
|
|
+#endif
|
|
+
|
|
static inline int restart_syscall(void)
|
|
{
|
|
set_tsk_thread_flag(current, TIF_SIGPENDING);
|
|
@@ -2723,11 +2769,6 @@ static inline int signal_pending_state(l
|
|
return (state & TASK_INTERRUPTIBLE) || __fatal_signal_pending(p);
|
|
}
|
|
|
|
-static inline int need_resched(void)
|
|
-{
|
|
- return unlikely(test_thread_flag(TIF_NEED_RESCHED));
|
|
-}
|
|
-
|
|
/*
|
|
* cond_resched() and cond_resched_lock(): latency reduction via
|
|
* explicit rescheduling in places that are safe. The return
|
|
--- a/kernel/Kconfig.preempt
|
|
+++ b/kernel/Kconfig.preempt
|
|
@@ -6,6 +6,12 @@ config PREEMPT_RT_BASE
|
|
bool
|
|
select PREEMPT
|
|
|
|
+config HAVE_PREEMPT_LAZY
|
|
+ bool
|
|
+
|
|
+config PREEMPT_LAZY
|
|
+ def_bool y if HAVE_PREEMPT_LAZY && PREEMPT_RT_FULL
|
|
+
|
|
choice
|
|
prompt "Preemption Model"
|
|
default PREEMPT_NONE
|
|
--- a/kernel/sched/core.c
|
|
+++ b/kernel/sched/core.c
|
|
@@ -543,6 +543,37 @@ void resched_task(struct task_struct *p)
|
|
smp_send_reschedule(cpu);
|
|
}
|
|
|
|
+#ifdef CONFIG_PREEMPT_LAZY
|
|
+void resched_task_lazy(struct task_struct *p)
|
|
+{
|
|
+ int cpu;
|
|
+
|
|
+ if (!sched_feat(PREEMPT_LAZY)) {
|
|
+ resched_task(p);
|
|
+ return;
|
|
+ }
|
|
+
|
|
+ assert_raw_spin_locked(&task_rq(p)->lock);
|
|
+
|
|
+ if (test_tsk_need_resched(p))
|
|
+ return;
|
|
+
|
|
+ if (test_tsk_need_resched_lazy(p))
|
|
+ return;
|
|
+
|
|
+ set_tsk_need_resched_lazy(p);
|
|
+
|
|
+ cpu = task_cpu(p);
|
|
+ if (cpu == smp_processor_id())
|
|
+ return;
|
|
+
|
|
+ /* NEED_RESCHED_LAZY must be visible before we test polling */
|
|
+ smp_mb();
|
|
+ if (!tsk_is_polling(p))
|
|
+ smp_send_reschedule(cpu);
|
|
+}
|
|
+#endif
|
|
+
|
|
void resched_cpu(int cpu)
|
|
{
|
|
struct rq *rq = cpu_rq(cpu);
|
|
@@ -659,6 +690,17 @@ void resched_task(struct task_struct *p)
|
|
assert_raw_spin_locked(&task_rq(p)->lock);
|
|
set_tsk_need_resched(p);
|
|
}
|
|
+#ifdef CONFIG_PREEMPT_LAZY
|
|
+void resched_task_lazy(struct task_struct *p)
|
|
+{
|
|
+ if (!sched_feat(PREEMPT_LAZY)) {
|
|
+ resched_task(p);
|
|
+ return;
|
|
+ }
|
|
+ assert_raw_spin_locked(&task_rq(p)->lock);
|
|
+ set_tsk_need_resched_lazy(p);
|
|
+}
|
|
+#endif
|
|
#endif /* CONFIG_SMP */
|
|
|
|
#if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \
|
|
@@ -1720,6 +1762,9 @@ void sched_fork(struct task_struct *p)
|
|
/* Want to start with kernel preemption disabled. */
|
|
task_thread_info(p)->preempt_count = 1;
|
|
#endif
|
|
+#ifdef CONFIG_HAVE_PREEMPT_LAZY
|
|
+ task_thread_info(p)->preempt_lazy_count = 0;
|
|
+#endif
|
|
#ifdef CONFIG_SMP
|
|
plist_node_init(&p->pushable_tasks, MAX_PRIO);
|
|
#endif
|
|
@@ -2928,6 +2973,7 @@ void migrate_disable(void)
|
|
return;
|
|
}
|
|
|
|
+ preempt_lazy_disable();
|
|
pin_current_cpu();
|
|
p->migrate_disable = 1;
|
|
preempt_enable();
|
|
@@ -2983,6 +3029,7 @@ void migrate_enable(void)
|
|
|
|
unpin_current_cpu();
|
|
preempt_enable();
|
|
+ preempt_lazy_enable();
|
|
}
|
|
EXPORT_SYMBOL(migrate_enable);
|
|
#else
|
|
@@ -3119,6 +3166,7 @@ need_resched:
|
|
put_prev_task(rq, prev);
|
|
next = pick_next_task(rq);
|
|
clear_tsk_need_resched(prev);
|
|
+ clear_tsk_need_resched_lazy(prev);
|
|
rq->skip_clock_update = 0;
|
|
|
|
if (likely(prev != next)) {
|
|
@@ -3255,6 +3303,14 @@ asmlinkage void __sched notrace preempt_
|
|
if (likely(ti->preempt_count || irqs_disabled()))
|
|
return;
|
|
|
|
+#ifdef CONFIG_PREEMPT_LAZY
|
|
+ /*
|
|
+ * Check for lazy preemption
|
|
+ */
|
|
+ if (ti->preempt_lazy_count && !test_thread_flag(TIF_NEED_RESCHED))
|
|
+ return;
|
|
+#endif
|
|
+
|
|
do {
|
|
add_preempt_count_notrace(PREEMPT_ACTIVE);
|
|
/*
|
|
@@ -4866,7 +4922,9 @@ void __cpuinit init_idle(struct task_str
|
|
|
|
/* Set the preempt count _outside_ the spinlocks! */
|
|
task_thread_info(idle)->preempt_count = 0;
|
|
-
|
|
+#ifdef CONFIG_HAVE_PREEMPT_LAZY
|
|
+ task_thread_info(idle)->preempt_lazy_count = 0;
|
|
+#endif
|
|
/*
|
|
* The idle tasks have their own, simple scheduling class:
|
|
*/
|
|
--- a/kernel/sched/fair.c
|
|
+++ b/kernel/sched/fair.c
|
|
@@ -1827,7 +1827,7 @@ check_preempt_tick(struct cfs_rq *cfs_rq
|
|
ideal_runtime = sched_slice(cfs_rq, curr);
|
|
delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime;
|
|
if (delta_exec > ideal_runtime) {
|
|
- resched_task(rq_of(cfs_rq)->curr);
|
|
+ resched_task_lazy(rq_of(cfs_rq)->curr);
|
|
/*
|
|
* The current task ran long enough, ensure it doesn't get
|
|
* re-elected due to buddy favours.
|
|
@@ -1851,7 +1851,7 @@ check_preempt_tick(struct cfs_rq *cfs_rq
|
|
return;
|
|
|
|
if (delta > ideal_runtime)
|
|
- resched_task(rq_of(cfs_rq)->curr);
|
|
+ resched_task_lazy(rq_of(cfs_rq)->curr);
|
|
}
|
|
|
|
static void
|
|
@@ -1971,7 +1971,7 @@ entity_tick(struct cfs_rq *cfs_rq, struc
|
|
* validating it and just reschedule.
|
|
*/
|
|
if (queued) {
|
|
- resched_task(rq_of(cfs_rq)->curr);
|
|
+ resched_task_lazy(rq_of(cfs_rq)->curr);
|
|
return;
|
|
}
|
|
/*
|
|
@@ -2160,7 +2160,7 @@ static void __account_cfs_rq_runtime(str
|
|
* hierarchy can be throttled
|
|
*/
|
|
if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr))
|
|
- resched_task(rq_of(cfs_rq)->curr);
|
|
+ resched_task_lazy(rq_of(cfs_rq)->curr);
|
|
}
|
|
|
|
static __always_inline
|
|
@@ -2745,7 +2745,7 @@ static void hrtick_start_fair(struct rq
|
|
|
|
if (delta < 0) {
|
|
if (rq->curr == p)
|
|
- resched_task(p);
|
|
+ resched_task_lazy(p);
|
|
return;
|
|
}
|
|
|
|
@@ -3577,7 +3577,7 @@ static void check_preempt_wakeup(struct
|
|
return;
|
|
|
|
preempt:
|
|
- resched_task(curr);
|
|
+ resched_task_lazy(curr);
|
|
/*
|
|
* Only set the backward buddy when the current task is still
|
|
* on the rq. This can happen when a wakeup gets interleaved
|
|
@@ -5772,7 +5772,7 @@ static void task_fork_fair(struct task_s
|
|
* 'current' within the tree based on its new key value.
|
|
*/
|
|
swap(curr->vruntime, se->vruntime);
|
|
- resched_task(rq->curr);
|
|
+ resched_task_lazy(rq->curr);
|
|
}
|
|
|
|
se->vruntime -= cfs_rq->min_vruntime;
|
|
@@ -5797,7 +5797,7 @@ prio_changed_fair(struct rq *rq, struct
|
|
*/
|
|
if (rq->curr == p) {
|
|
if (p->prio > oldprio)
|
|
- resched_task(rq->curr);
|
|
+ resched_task_lazy(rq->curr);
|
|
} else
|
|
check_preempt_curr(rq, p, 0);
|
|
}
|
|
--- a/kernel/sched/features.h
|
|
+++ b/kernel/sched/features.h
|
|
@@ -65,6 +65,9 @@ SCHED_FEAT(NONTASK_POWER, true)
|
|
SCHED_FEAT(TTWU_QUEUE, true)
|
|
#else
|
|
SCHED_FEAT(TTWU_QUEUE, false)
|
|
+# ifdef CONFIG_PREEMPT_LAZY
|
|
+SCHED_FEAT(PREEMPT_LAZY, true)
|
|
+# endif
|
|
#endif
|
|
|
|
SCHED_FEAT(FORCE_SD_OVERLAP, false)
|
|
--- a/kernel/sched/sched.h
|
|
+++ b/kernel/sched/sched.h
|
|
@@ -897,6 +897,15 @@ extern void init_sched_fair_class(void);
|
|
extern void resched_task(struct task_struct *p);
|
|
extern void resched_cpu(int cpu);
|
|
|
|
+#ifdef CONFIG_PREEMPT_LAZY
|
|
+extern void resched_task_lazy(struct task_struct *tsk);
|
|
+#else
|
|
+static inline void resched_task_lazy(struct task_struct *tsk)
|
|
+{
|
|
+ resched_task(tsk);
|
|
+}
|
|
+#endif
|
|
+
|
|
extern struct rt_bandwidth def_rt_bandwidth;
|
|
extern void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime);
|
|
|
|
--- a/kernel/trace/trace.c
|
|
+++ b/kernel/trace/trace.c
|
|
@@ -1167,6 +1167,7 @@ tracing_generic_entry_update(struct trac
|
|
struct task_struct *tsk = current;
|
|
|
|
entry->preempt_count = pc & 0xff;
|
|
+ entry->preempt_lazy_count = preempt_lazy_count();
|
|
entry->pid = (tsk) ? tsk->pid : 0;
|
|
entry->padding = 0;
|
|
entry->flags =
|
|
@@ -1177,7 +1178,8 @@ tracing_generic_entry_update(struct trac
|
|
#endif
|
|
((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
|
|
((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
|
|
- (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
|
|
+ (need_resched_now() ? TRACE_FLAG_NEED_RESCHED : 0) |
|
|
+ (need_resched_lazy() ? TRACE_FLAG_NEED_RESCHED_LAZY : 0);
|
|
|
|
entry->migrate_disable = (tsk) ? __migrate_disabled(tsk) & 0xFF : 0;
|
|
}
|
|
@@ -2032,15 +2034,17 @@ get_total_entries(struct trace_array *tr
|
|
|
|
static void print_lat_help_header(struct seq_file *m)
|
|
{
|
|
- seq_puts(m, "# _------=> CPU# \n");
|
|
- seq_puts(m, "# / _-----=> irqs-off \n");
|
|
- seq_puts(m, "# | / _----=> need-resched \n");
|
|
- seq_puts(m, "# || / _---=> hardirq/softirq \n");
|
|
- seq_puts(m, "# ||| / _--=> preempt-depth \n");
|
|
- seq_puts(m, "# |||| / _--=> migrate-disable\n");
|
|
- seq_puts(m, "# ||||| / delay \n");
|
|
- seq_puts(m, "# cmd pid |||||| time | caller \n");
|
|
- seq_puts(m, "# \\ / ||||| \\ | / \n");
|
|
+ seq_puts(m, "# _--------=> CPU# \n");
|
|
+ seq_puts(m, "# / _-------=> irqs-off \n");
|
|
+ seq_puts(m, "# | / _------=> need-resched \n");
|
|
+ seq_puts(m, "# || / _-----=> need-resched_lazy \n");
|
|
+ seq_puts(m, "# ||| / _----=> hardirq/softirq \n");
|
|
+ seq_puts(m, "# |||| / _---=> preempt-depth \n");
|
|
+ seq_puts(m, "# ||||| / _--=> preempt-lazy-depth\n");
|
|
+ seq_puts(m, "# |||||| / _-=> migrate-disable \n");
|
|
+ seq_puts(m, "# ||||||| / delay \n");
|
|
+ seq_puts(m, "# cmd pid |||||||| time | caller \n");
|
|
+ seq_puts(m, "# \\ / |||||||| \\ | / \n");
|
|
}
|
|
|
|
static void print_event_info(struct trace_array *tr, struct seq_file *m)
|
|
@@ -2064,13 +2068,16 @@ static void print_func_help_header(struc
|
|
static void print_func_help_header_irq(struct trace_array *tr, struct seq_file *m)
|
|
{
|
|
print_event_info(tr, m);
|
|
- seq_puts(m, "# _-----=> irqs-off\n");
|
|
- seq_puts(m, "# / _----=> need-resched\n");
|
|
- seq_puts(m, "# | / _---=> hardirq/softirq\n");
|
|
- seq_puts(m, "# || / _--=> preempt-depth\n");
|
|
- seq_puts(m, "# ||| / delay\n");
|
|
- seq_puts(m, "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n");
|
|
- seq_puts(m, "# | | | |||| | |\n");
|
|
+ seq_puts(m, "# _-------=> irqs-off \n");
|
|
+ seq_puts(m, "# / _------=> need-resched \n");
|
|
+ seq_puts(m, "# |/ _-----=> need-resched_lazy \n");
|
|
+ seq_puts(m, "# ||/ _----=> hardirq/softirq \n");
|
|
+ seq_puts(m, "# |||/ _---=> preempt-depth \n");
|
|
+ seq_puts(m, "# ||||/ _--=> preempt-lazy-depth\n");
|
|
+ seq_puts(m, "# ||||| / _-=> migrate-disable \n");
|
|
+ seq_puts(m, "# |||||| / delay\n");
|
|
+ seq_puts(m, "# TASK-PID CPU# ||||||| TIMESTAMP FUNCTION\n");
|
|
+ seq_puts(m, "# | | | ||||||| | |\n");
|
|
}
|
|
|
|
void
|
|
--- a/kernel/trace/trace.h
|
|
+++ b/kernel/trace/trace.h
|
|
@@ -116,6 +116,7 @@ struct uprobe_trace_entry_head {
|
|
* NEED_RESCHED - reschedule is requested
|
|
* HARDIRQ - inside an interrupt handler
|
|
* SOFTIRQ - inside a softirq handler
|
|
+ * NEED_RESCHED_LAZY - lazy reschedule is requested
|
|
*/
|
|
enum trace_flag_type {
|
|
TRACE_FLAG_IRQS_OFF = 0x01,
|
|
@@ -123,6 +124,7 @@ enum trace_flag_type {
|
|
TRACE_FLAG_NEED_RESCHED = 0x04,
|
|
TRACE_FLAG_HARDIRQ = 0x08,
|
|
TRACE_FLAG_SOFTIRQ = 0x10,
|
|
+ TRACE_FLAG_NEED_RESCHED_LAZY = 0x20,
|
|
};
|
|
|
|
#define TRACE_BUF_SIZE 1024
|
|
--- a/kernel/trace/trace_output.c
|
|
+++ b/kernel/trace/trace_output.c
|
|
@@ -564,6 +564,7 @@ int trace_print_lat_fmt(struct trace_seq
|
|
{
|
|
char hardsoft_irq;
|
|
char need_resched;
|
|
+ char need_resched_lazy;
|
|
char irqs_off;
|
|
int hardirq;
|
|
int softirq;
|
|
@@ -578,14 +579,17 @@ int trace_print_lat_fmt(struct trace_seq
|
|
'.';
|
|
need_resched =
|
|
(entry->flags & TRACE_FLAG_NEED_RESCHED) ? 'N' : '.';
|
|
+ need_resched_lazy =
|
|
+ (entry->flags & TRACE_FLAG_NEED_RESCHED_LAZY) ? 'L' : '.';
|
|
hardsoft_irq =
|
|
(hardirq && softirq) ? 'H' :
|
|
hardirq ? 'h' :
|
|
softirq ? 's' :
|
|
'.';
|
|
|
|
- if (!trace_seq_printf(s, "%c%c%c",
|
|
- irqs_off, need_resched, hardsoft_irq))
|
|
+ if (!trace_seq_printf(s, "%c%c%c%c",
|
|
+ irqs_off, need_resched, need_resched_lazy,
|
|
+ hardsoft_irq))
|
|
return 0;
|
|
|
|
if (entry->preempt_count)
|
|
@@ -593,6 +597,11 @@ int trace_print_lat_fmt(struct trace_seq
|
|
else
|
|
ret = trace_seq_putc(s, '.');
|
|
|
|
+ if (entry->preempt_lazy_count)
|
|
+ ret = trace_seq_printf(s, "%x", entry->preempt_lazy_count);
|
|
+ else
|
|
+ ret = trace_seq_putc(s, '.');
|
|
+
|
|
if (entry->migrate_disable)
|
|
ret = trace_seq_printf(s, "%x", entry->migrate_disable);
|
|
else
|