|
|
|
@@ -458,46 +458,10 @@ int rcu_read_lock_bh_held(void);
|
|
|
|
|
* If CONFIG_DEBUG_LOCK_ALLOC is selected, returns nonzero iff in an
|
|
|
|
|
* RCU-sched read-side critical section. In absence of
|
|
|
|
|
* CONFIG_DEBUG_LOCK_ALLOC, this assumes we are in an RCU-sched read-side
|
|
|
|
|
* critical section unless it can prove otherwise. Note that disabling
|
|
|
|
|
* of preemption (including disabling irqs) counts as an RCU-sched
|
|
|
|
|
* read-side critical section. This is useful for debug checks in functions
|
|
|
|
|
* that required that they be called within an RCU-sched read-side
|
|
|
|
|
* critical section.
|
|
|
|
|
*
|
|
|
|
|
* Check debug_lockdep_rcu_enabled() to prevent false positives during boot
|
|
|
|
|
* and while lockdep is disabled.
|
|
|
|
|
*
|
|
|
|
|
* Note that if the CPU is in the idle loop from an RCU point of
|
|
|
|
|
* view (ie: that we are in the section between rcu_idle_enter() and
|
|
|
|
|
* rcu_idle_exit()) then rcu_read_lock_held() returns false even if the CPU
|
|
|
|
|
* did an rcu_read_lock(). The reason for this is that RCU ignores CPUs
|
|
|
|
|
* that are in such a section, considering these as in extended quiescent
|
|
|
|
|
* state, so such a CPU is effectively never in an RCU read-side critical
|
|
|
|
|
* section regardless of what RCU primitives it invokes. This state of
|
|
|
|
|
* affairs is required --- we need to keep an RCU-free window in idle
|
|
|
|
|
* where the CPU may possibly enter into low power mode. This way we can
|
|
|
|
|
* notice an extended quiescent state to other CPUs that started a grace
|
|
|
|
|
* period. Otherwise we would delay any grace period as long as we run in
|
|
|
|
|
* the idle task.
|
|
|
|
|
*
|
|
|
|
|
* Similarly, we avoid claiming an SRCU read lock held if the current
|
|
|
|
|
* CPU is offline.
|
|
|
|
|
* critical section unless it can prove otherwise.
|
|
|
|
|
*/
|
|
|
|
|
#ifdef CONFIG_PREEMPT_COUNT
|
|
|
|
|
static inline int rcu_read_lock_sched_held(void)
|
|
|
|
|
{
|
|
|
|
|
int lockdep_opinion = 0;
|
|
|
|
|
|
|
|
|
|
if (!debug_lockdep_rcu_enabled())
|
|
|
|
|
return 1;
|
|
|
|
|
if (!rcu_is_watching())
|
|
|
|
|
return 0;
|
|
|
|
|
if (!rcu_lockdep_current_cpu_online())
|
|
|
|
|
return 0;
|
|
|
|
|
if (debug_locks)
|
|
|
|
|
lockdep_opinion = lock_is_held(&rcu_sched_lock_map);
|
|
|
|
|
return lockdep_opinion || preempt_count() != 0 || irqs_disabled();
|
|
|
|
|
}
|
|
|
|
|
int rcu_read_lock_sched_held(void);
|
|
|
|
|
#else /* #ifdef CONFIG_PREEMPT_COUNT */
|
|
|
|
|
static inline int rcu_read_lock_sched_held(void)
|
|
|
|
|
{
|
|
|
|
@@ -537,14 +501,14 @@ static inline int rcu_read_lock_sched_held(void)
|
|
|
|
|
#ifdef CONFIG_PROVE_RCU
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* rcu_lockdep_assert - emit lockdep splat if specified condition not met
|
|
|
|
|
* RCU_LOCKDEP_WARN - emit lockdep splat if specified condition is met
|
|
|
|
|
* @c: condition to check
|
|
|
|
|
* @s: informative message
|
|
|
|
|
*/
|
|
|
|
|
#define rcu_lockdep_assert(c, s) \
|
|
|
|
|
#define RCU_LOCKDEP_WARN(c, s) \
|
|
|
|
|
do { \
|
|
|
|
|
static bool __section(.data.unlikely) __warned; \
|
|
|
|
|
if (debug_lockdep_rcu_enabled() && !__warned && !(c)) { \
|
|
|
|
|
if (debug_lockdep_rcu_enabled() && !__warned && (c)) { \
|
|
|
|
|
__warned = true; \
|
|
|
|
|
lockdep_rcu_suspicious(__FILE__, __LINE__, s); \
|
|
|
|
|
} \
|
|
|
|
@@ -553,7 +517,7 @@ static inline int rcu_read_lock_sched_held(void)
|
|
|
|
|
#if defined(CONFIG_PROVE_RCU) && !defined(CONFIG_PREEMPT_RCU)
|
|
|
|
|
static inline void rcu_preempt_sleep_check(void)
|
|
|
|
|
{
|
|
|
|
|
rcu_lockdep_assert(!lock_is_held(&rcu_lock_map),
|
|
|
|
|
RCU_LOCKDEP_WARN(lock_is_held(&rcu_lock_map),
|
|
|
|
|
"Illegal context switch in RCU read-side critical section");
|
|
|
|
|
}
|
|
|
|
|
#else /* #ifdef CONFIG_PROVE_RCU */
|
|
|
|
@@ -565,15 +529,15 @@ static inline void rcu_preempt_sleep_check(void)
|
|
|
|
|
#define rcu_sleep_check() \
|
|
|
|
|
do { \
|
|
|
|
|
rcu_preempt_sleep_check(); \
|
|
|
|
|
rcu_lockdep_assert(!lock_is_held(&rcu_bh_lock_map), \
|
|
|
|
|
RCU_LOCKDEP_WARN(lock_is_held(&rcu_bh_lock_map), \
|
|
|
|
|
"Illegal context switch in RCU-bh read-side critical section"); \
|
|
|
|
|
rcu_lockdep_assert(!lock_is_held(&rcu_sched_lock_map), \
|
|
|
|
|
RCU_LOCKDEP_WARN(lock_is_held(&rcu_sched_lock_map), \
|
|
|
|
|
"Illegal context switch in RCU-sched read-side critical section"); \
|
|
|
|
|
} while (0)
|
|
|
|
|
|
|
|
|
|
#else /* #ifdef CONFIG_PROVE_RCU */
|
|
|
|
|
|
|
|
|
|
#define rcu_lockdep_assert(c, s) do { } while (0)
|
|
|
|
|
#define RCU_LOCKDEP_WARN(c, s) do { } while (0)
|
|
|
|
|
#define rcu_sleep_check() do { } while (0)
|
|
|
|
|
|
|
|
|
|
#endif /* #else #ifdef CONFIG_PROVE_RCU */
|
|
|
|
@@ -604,13 +568,13 @@ static inline void rcu_preempt_sleep_check(void)
|
|
|
|
|
({ \
|
|
|
|
|
/* Dependency order vs. p above. */ \
|
|
|
|
|
typeof(*p) *________p1 = (typeof(*p) *__force)lockless_dereference(p); \
|
|
|
|
|
rcu_lockdep_assert(c, "suspicious rcu_dereference_check() usage"); \
|
|
|
|
|
RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_check() usage"); \
|
|
|
|
|
rcu_dereference_sparse(p, space); \
|
|
|
|
|
((typeof(*p) __force __kernel *)(________p1)); \
|
|
|
|
|
})
|
|
|
|
|
#define __rcu_dereference_protected(p, c, space) \
|
|
|
|
|
({ \
|
|
|
|
|
rcu_lockdep_assert(c, "suspicious rcu_dereference_protected() usage"); \
|
|
|
|
|
RCU_LOCKDEP_WARN(!(c), "suspicious rcu_dereference_protected() usage"); \
|
|
|
|
|
rcu_dereference_sparse(p, space); \
|
|
|
|
|
((typeof(*p) __force __kernel *)(p)); \
|
|
|
|
|
})
|
|
|
|
@@ -834,7 +798,7 @@ static inline void rcu_read_lock(void)
|
|
|
|
|
__rcu_read_lock();
|
|
|
|
|
__acquire(RCU);
|
|
|
|
|
rcu_lock_acquire(&rcu_lock_map);
|
|
|
|
|
rcu_lockdep_assert(rcu_is_watching(),
|
|
|
|
|
RCU_LOCKDEP_WARN(!rcu_is_watching(),
|
|
|
|
|
"rcu_read_lock() used illegally while idle");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -885,7 +849,7 @@ static inline void rcu_read_lock(void)
|
|
|
|
|
*/
|
|
|
|
|
static inline void rcu_read_unlock(void)
|
|
|
|
|
{
|
|
|
|
|
rcu_lockdep_assert(rcu_is_watching(),
|
|
|
|
|
RCU_LOCKDEP_WARN(!rcu_is_watching(),
|
|
|
|
|
"rcu_read_unlock() used illegally while idle");
|
|
|
|
|
__release(RCU);
|
|
|
|
|
__rcu_read_unlock();
|
|
|
|
@@ -914,7 +878,7 @@ static inline void rcu_read_lock_bh(void)
|
|
|
|
|
local_bh_disable();
|
|
|
|
|
__acquire(RCU_BH);
|
|
|
|
|
rcu_lock_acquire(&rcu_bh_lock_map);
|
|
|
|
|
rcu_lockdep_assert(rcu_is_watching(),
|
|
|
|
|
RCU_LOCKDEP_WARN(!rcu_is_watching(),
|
|
|
|
|
"rcu_read_lock_bh() used illegally while idle");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -925,7 +889,7 @@ static inline void rcu_read_lock_bh(void)
|
|
|
|
|
*/
|
|
|
|
|
static inline void rcu_read_unlock_bh(void)
|
|
|
|
|
{
|
|
|
|
|
rcu_lockdep_assert(rcu_is_watching(),
|
|
|
|
|
RCU_LOCKDEP_WARN(!rcu_is_watching(),
|
|
|
|
|
"rcu_read_unlock_bh() used illegally while idle");
|
|
|
|
|
rcu_lock_release(&rcu_bh_lock_map);
|
|
|
|
|
__release(RCU_BH);
|
|
|
|
@@ -950,7 +914,7 @@ static inline void rcu_read_lock_sched(void)
|
|
|
|
|
preempt_disable();
|
|
|
|
|
__acquire(RCU_SCHED);
|
|
|
|
|
rcu_lock_acquire(&rcu_sched_lock_map);
|
|
|
|
|
rcu_lockdep_assert(rcu_is_watching(),
|
|
|
|
|
RCU_LOCKDEP_WARN(!rcu_is_watching(),
|
|
|
|
|
"rcu_read_lock_sched() used illegally while idle");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -968,7 +932,7 @@ static inline notrace void rcu_read_lock_sched_notrace(void)
|
|
|
|
|
*/
|
|
|
|
|
static inline void rcu_read_unlock_sched(void)
|
|
|
|
|
{
|
|
|
|
|
rcu_lockdep_assert(rcu_is_watching(),
|
|
|
|
|
RCU_LOCKDEP_WARN(!rcu_is_watching(),
|
|
|
|
|
"rcu_read_unlock_sched() used illegally while idle");
|
|
|
|
|
rcu_lock_release(&rcu_sched_lock_map);
|
|
|
|
|
__release(RCU_SCHED);
|
|
|
|
|