* [PATCH RT 01/31] gpio: omap: use raw locks for locking
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
@ 2015-03-13 15:14 ` Steven Rostedt
2015-03-13 15:14 ` [PATCH RT 02/31] create-rt-enqueue Steven Rostedt
` (28 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:14 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker, stable-rt
[-- Attachment #1: 0001-gpio-omap-use-raw-locks-for-locking.patch --]
[-- Type: text/plain, Size: 10365 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
This patch converts gpio_bank.lock from a spin_lock into a
raw_spin_lock. The call path is to access this lock is always under a
raw_spin_lock, for instance
- __setup_irq() holds &desc->lock with irq off
+ __irq_set_trigger()
+ omap_gpio_irq_type()
- handle_level_irq() (runs with irqs off therefore raw locks)
+ mask_ack_irq()
+ omap_gpio_mask_irq()
This fixes the obvious backtrace on -RT. However the locking vs context
is not and this is not limited to -RT:
- omap_gpio_irq_type() is called with IRQ off and has an conditional
call to pm_runtime_get_sync() which may sleep. Either it may happen or
it may not happen but pm_runtime_get_sync() should not be called with
irqs off.
- omap_gpio_debounce() is holding the lock with IRQs off.
+ omap2_set_gpio_debounce()
+ clk_prepare_enable()
+ clk_prepare() this one might sleep.
The number of users of gpiod_set_debounce() / gpio_set_debounce()
looks low but still this is not good.
Cc: stable-rt@vger.kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
drivers/gpio/gpio-omap.c | 70 ++++++++++++++++++++++++------------------------
1 file changed, 35 insertions(+), 35 deletions(-)
diff --git a/drivers/gpio/gpio-omap.c b/drivers/gpio/gpio-omap.c
index 54052125859a..0fb30e1d1cff 100644
--- a/drivers/gpio/gpio-omap.c
+++ b/drivers/gpio/gpio-omap.c
@@ -59,7 +59,7 @@ struct gpio_bank {
u32 saved_datain;
u32 level_mask;
u32 toggle_mask;
- spinlock_t lock;
+ raw_spinlock_t lock;
struct gpio_chip chip;
struct clk *dbck;
u32 mod_usage;
@@ -503,19 +503,19 @@ static int gpio_irq_type(struct irq_data *d, unsigned type)
(type & (IRQ_TYPE_LEVEL_LOW|IRQ_TYPE_LEVEL_HIGH)))
return -EINVAL;
- spin_lock_irqsave(&bank->lock, flags);
+ raw_spin_lock_irqsave(&bank->lock, flags);
offset = GPIO_INDEX(bank, gpio);
retval = _set_gpio_triggering(bank, offset, type);
if (!LINE_USED(bank->mod_usage, offset)) {
_enable_gpio_module(bank, offset);
_set_gpio_direction(bank, offset, 1);
} else if (!gpio_is_input(bank, 1 << offset)) {
- spin_unlock_irqrestore(&bank->lock, flags);
+ raw_spin_unlock_irqrestore(&bank->lock, flags);
return -EINVAL;
}
bank->irq_usage |= 1 << GPIO_INDEX(bank, gpio);
- spin_unlock_irqrestore(&bank->lock, flags);
+ raw_spin_unlock_irqrestore(&bank->lock, flags);
if (type & (IRQ_TYPE_LEVEL_LOW | IRQ_TYPE_LEVEL_HIGH))
__irq_set_handler_locked(d->irq, handle_level_irq);
@@ -632,14 +632,14 @@ static int _set_gpio_wakeup(struct gpio_bank *bank, int gpio, int enable)
return -EINVAL;
}
- spin_lock_irqsave(&bank->lock, flags);
+ raw_spin_lock_irqsave(&bank->lock, flags);
if (enable)
bank->context.wake_en |= gpio_bit;
else
bank->context.wake_en &= ~gpio_bit;
__raw_writel(bank->context.wake_en, bank->base + bank->regs->wkup_en);
- spin_unlock_irqrestore(&bank->lock, flags);
+ raw_spin_unlock_irqrestore(&bank->lock, flags);
return 0;
}
@@ -674,7 +674,7 @@ static int omap_gpio_request(struct gpio_chip *chip, unsigned offset)
if (!BANK_USED(bank))
pm_runtime_get_sync(bank->dev);
- spin_lock_irqsave(&bank->lock, flags);
+ raw_spin_lock_irqsave(&bank->lock, flags);
/* Set trigger to none. You need to enable the desired trigger with
* request_irq() or set_irq_type(). Only do this if the IRQ line has
* not already been requested.
@@ -684,7 +684,7 @@ static int omap_gpio_request(struct gpio_chip *chip, unsigned offset)
_enable_gpio_module(bank, offset);
}
bank->mod_usage |= 1 << offset;
- spin_unlock_irqrestore(&bank->lock, flags);
+ raw_spin_unlock_irqrestore(&bank->lock, flags);
return 0;
}
@@ -694,11 +694,11 @@ static void omap_gpio_free(struct gpio_chip *chip, unsigned offset)
struct gpio_bank *bank = container_of(chip, struct gpio_bank, chip);
unsigned long flags;
- spin_lock_irqsave(&bank->lock, flags);
+ raw_spin_lock_irqsave(&bank->lock, flags);
bank->mod_usage &= ~(1 << offset);
_disable_gpio_module(bank, offset);
_reset_gpio(bank, bank->chip.base + offset);
- spin_unlock_irqrestore(&bank->lock, flags);
+ raw_spin_unlock_irqrestore(&bank->lock, flags);
/*
* If this is the last gpio to be freed in the bank,
@@ -796,11 +796,11 @@ static void gpio_irq_shutdown(struct irq_data *d)
unsigned long flags;
unsigned offset = GPIO_INDEX(bank, gpio);
- spin_lock_irqsave(&bank->lock, flags);
+ raw_spin_lock_irqsave(&bank->lock, flags);
bank->irq_usage &= ~(1 << offset);
_disable_gpio_module(bank, offset);
_reset_gpio(bank, gpio);
- spin_unlock_irqrestore(&bank->lock, flags);
+ raw_spin_unlock_irqrestore(&bank->lock, flags);
/*
* If this is the last IRQ to be freed in the bank,
@@ -824,10 +824,10 @@ static void gpio_mask_irq(struct irq_data *d)
unsigned int gpio = irq_to_gpio(bank, d->hwirq);
unsigned long flags;
- spin_lock_irqsave(&bank->lock, flags);
+ raw_spin_lock_irqsave(&bank->lock, flags);
_set_gpio_irqenable(bank, gpio, 0);
_set_gpio_triggering(bank, GPIO_INDEX(bank, gpio), IRQ_TYPE_NONE);
- spin_unlock_irqrestore(&bank->lock, flags);
+ raw_spin_unlock_irqrestore(&bank->lock, flags);
}
static void gpio_unmask_irq(struct irq_data *d)
@@ -838,7 +838,7 @@ static void gpio_unmask_irq(struct irq_data *d)
u32 trigger = irqd_get_trigger_type(d);
unsigned long flags;
- spin_lock_irqsave(&bank->lock, flags);
+ raw_spin_lock_irqsave(&bank->lock, flags);
if (trigger)
_set_gpio_triggering(bank, GPIO_INDEX(bank, gpio), trigger);
@@ -850,7 +850,7 @@ static void gpio_unmask_irq(struct irq_data *d)
}
_set_gpio_irqenable(bank, gpio, 1);
- spin_unlock_irqrestore(&bank->lock, flags);
+ raw_spin_unlock_irqrestore(&bank->lock, flags);
}
static struct irq_chip gpio_irq_chip = {
@@ -873,9 +873,9 @@ static int omap_mpuio_suspend_noirq(struct device *dev)
OMAP_MPUIO_GPIO_MASKIT / bank->stride;
unsigned long flags;
- spin_lock_irqsave(&bank->lock, flags);
+ raw_spin_lock_irqsave(&bank->lock, flags);
__raw_writel(0xffff & ~bank->context.wake_en, mask_reg);
- spin_unlock_irqrestore(&bank->lock, flags);
+ raw_spin_unlock_irqrestore(&bank->lock, flags);
return 0;
}
@@ -888,9 +888,9 @@ static int omap_mpuio_resume_noirq(struct device *dev)
OMAP_MPUIO_GPIO_MASKIT / bank->stride;
unsigned long flags;
- spin_lock_irqsave(&bank->lock, flags);
+ raw_spin_lock_irqsave(&bank->lock, flags);
__raw_writel(bank->context.wake_en, mask_reg);
- spin_unlock_irqrestore(&bank->lock, flags);
+ raw_spin_unlock_irqrestore(&bank->lock, flags);
return 0;
}
@@ -933,9 +933,9 @@ static int gpio_input(struct gpio_chip *chip, unsigned offset)
unsigned long flags;
bank = container_of(chip, struct gpio_bank, chip);
- spin_lock_irqsave(&bank->lock, flags);
+ raw_spin_lock_irqsave(&bank->lock, flags);
_set_gpio_direction(bank, offset, 1);
- spin_unlock_irqrestore(&bank->lock, flags);
+ raw_spin_unlock_irqrestore(&bank->lock, flags);
return 0;
}
@@ -960,7 +960,7 @@ static int gpio_output(struct gpio_chip *chip, unsigned offset, int value)
int retval = 0;
bank = container_of(chip, struct gpio_bank, chip);
- spin_lock_irqsave(&bank->lock, flags);
+ raw_spin_lock_irqsave(&bank->lock, flags);
if (LINE_USED(bank->irq_usage, offset)) {
retval = -EINVAL;
@@ -971,7 +971,7 @@ static int gpio_output(struct gpio_chip *chip, unsigned offset, int value)
_set_gpio_direction(bank, offset, 0);
exit:
- spin_unlock_irqrestore(&bank->lock, flags);
+ raw_spin_unlock_irqrestore(&bank->lock, flags);
return retval;
}
@@ -983,9 +983,9 @@ static int gpio_debounce(struct gpio_chip *chip, unsigned offset,
bank = container_of(chip, struct gpio_bank, chip);
- spin_lock_irqsave(&bank->lock, flags);
+ raw_spin_lock_irqsave(&bank->lock, flags);
_set_gpio_debounce(bank, offset, debounce);
- spin_unlock_irqrestore(&bank->lock, flags);
+ raw_spin_unlock_irqrestore(&bank->lock, flags);
return 0;
}
@@ -996,9 +996,9 @@ static void gpio_set(struct gpio_chip *chip, unsigned offset, int value)
unsigned long flags;
bank = container_of(chip, struct gpio_bank, chip);
- spin_lock_irqsave(&bank->lock, flags);
+ raw_spin_lock_irqsave(&bank->lock, flags);
bank->set_dataout(bank, offset, value);
- spin_unlock_irqrestore(&bank->lock, flags);
+ raw_spin_unlock_irqrestore(&bank->lock, flags);
}
/*---------------------------------------------------------------------*/
@@ -1210,7 +1210,7 @@ static int omap_gpio_probe(struct platform_device *pdev)
else
bank->set_dataout = _set_gpio_dataout_mask;
- spin_lock_init(&bank->lock);
+ raw_spin_lock_init(&bank->lock);
/* Static mapping, never released */
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
@@ -1267,7 +1267,7 @@ static int omap_gpio_runtime_suspend(struct device *dev)
unsigned long flags;
u32 wake_low, wake_hi;
- spin_lock_irqsave(&bank->lock, flags);
+ raw_spin_lock_irqsave(&bank->lock, flags);
/*
* Only edges can generate a wakeup event to the PRCM.
@@ -1320,7 +1320,7 @@ update_gpio_context_count:
bank->get_context_loss_count(bank->dev);
_gpio_dbck_disable(bank);
- spin_unlock_irqrestore(&bank->lock, flags);
+ raw_spin_unlock_irqrestore(&bank->lock, flags);
return 0;
}
@@ -1335,7 +1335,7 @@ static int omap_gpio_runtime_resume(struct device *dev)
unsigned long flags;
int c;
- spin_lock_irqsave(&bank->lock, flags);
+ raw_spin_lock_irqsave(&bank->lock, flags);
/*
* On the first resume during the probe, the context has not
@@ -1371,14 +1371,14 @@ static int omap_gpio_runtime_resume(struct device *dev)
if (c != bank->context_loss_count) {
omap_gpio_restore_context(bank);
} else {
- spin_unlock_irqrestore(&bank->lock, flags);
+ raw_spin_unlock_irqrestore(&bank->lock, flags);
return 0;
}
}
}
if (!bank->workaround_enabled) {
- spin_unlock_irqrestore(&bank->lock, flags);
+ raw_spin_unlock_irqrestore(&bank->lock, flags);
return 0;
}
@@ -1433,7 +1433,7 @@ static int omap_gpio_runtime_resume(struct device *dev)
}
bank->workaround_enabled = false;
- spin_unlock_irqrestore(&bank->lock, flags);
+ raw_spin_unlock_irqrestore(&bank->lock, flags);
return 0;
}
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 02/31] create-rt-enqueue
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
2015-03-13 15:14 ` [PATCH RT 01/31] gpio: omap: use raw locks for locking Steven Rostedt
@ 2015-03-13 15:14 ` Steven Rostedt
2015-03-13 15:14 ` [PATCH RT 03/31] rtmutex: Simplify rtmutex_slowtrylock() Steven Rostedt
` (27 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:14 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker
[-- Attachment #1: 0002-create-rt-enqueue.patch --]
[-- Type: text/plain, Size: 5532 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: Steven Rostedt <rostedt@goodmis.org>
To ease backporting patches, replace the plist functions with
rt_mutex_enqueue{_pi}() and rt_mutex_dequeue{_pi}() like upstream -rt does.
This will lower the conflicts in backporting patches.
This also makes sure that every time a waiter is added to the pi_list it
matches the priority of the waiter list.
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
kernel/rtmutex.c | 58 ++++++++++++++++++++++++++++++++++++++------------------
1 file changed, 40 insertions(+), 18 deletions(-)
diff --git a/kernel/rtmutex.c b/kernel/rtmutex.c
index 2671e5532534..7cd751bb891a 100644
--- a/kernel/rtmutex.c
+++ b/kernel/rtmutex.c
@@ -155,6 +155,31 @@ static inline bool unlock_rt_mutex_safe(struct rt_mutex *lock)
}
#endif
+static inline void
+rt_mutex_enqueue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter)
+{
+ plist_add(&waiter->list_entry, &lock->wait_list);
+}
+
+static inline void
+rt_mutex_dequeue(struct rt_mutex *lock, struct rt_mutex_waiter *waiter)
+{
+ plist_del(&waiter->list_entry, &lock->wait_list);
+}
+
+static inline void
+rt_mutex_enqueue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)
+{
+ waiter->pi_list_entry.prio = waiter->list_entry.prio;
+ plist_add(&waiter->pi_list_entry, &task->pi_waiters);
+}
+
+static inline void
+rt_mutex_dequeue_pi(struct task_struct *task, struct rt_mutex_waiter *waiter)
+{
+ plist_del(&waiter->pi_list_entry, &task->pi_waiters);
+}
+
static inline void init_lists(struct rt_mutex *lock)
{
if (unlikely(!lock->wait_list.node_list.prev))
@@ -363,9 +388,9 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
top_waiter = rt_mutex_top_waiter(lock);
/* Requeue the waiter */
- plist_del(&waiter->list_entry, &lock->wait_list);
+ rt_mutex_dequeue(lock, waiter);
waiter->list_entry.prio = task->prio;
- plist_add(&waiter->list_entry, &lock->wait_list);
+ rt_mutex_enqueue(lock, waiter);
/* Release the task */
raw_spin_unlock_irqrestore(&task->pi_lock, flags);
@@ -391,17 +416,15 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
if (waiter == rt_mutex_top_waiter(lock)) {
/* Boost the owner */
- plist_del(&top_waiter->pi_list_entry, &task->pi_waiters);
- waiter->pi_list_entry.prio = waiter->list_entry.prio;
- plist_add(&waiter->pi_list_entry, &task->pi_waiters);
+ rt_mutex_dequeue_pi(task, top_waiter);
+ rt_mutex_enqueue_pi(task, waiter);
__rt_mutex_adjust_prio(task);
} else if (top_waiter == waiter) {
/* Deboost the owner */
- plist_del(&waiter->pi_list_entry, &task->pi_waiters);
+ rt_mutex_dequeue_pi(task, waiter);
waiter = rt_mutex_top_waiter(lock);
- waiter->pi_list_entry.prio = waiter->list_entry.prio;
- plist_add(&waiter->pi_list_entry, &task->pi_waiters);
+ rt_mutex_enqueue_pi(task, waiter);
__rt_mutex_adjust_prio(task);
}
@@ -517,7 +540,7 @@ __try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
/* remove the queued waiter. */
if (waiter) {
- plist_del(&waiter->list_entry, &lock->wait_list);
+ rt_mutex_dequeue(lock, waiter);
task->pi_blocked_on = NULL;
}
@@ -527,8 +550,7 @@ __try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
*/
if (rt_mutex_has_waiters(lock)) {
top = rt_mutex_top_waiter(lock);
- top->pi_list_entry.prio = top->list_entry.prio;
- plist_add(&top->pi_list_entry, &task->pi_waiters);
+ rt_mutex_enqueue_pi(task, top);
}
raw_spin_unlock_irqrestore(&task->pi_lock, flags);
}
@@ -606,7 +628,7 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
/* Get the top priority waiter on the lock */
if (rt_mutex_has_waiters(lock))
top_waiter = rt_mutex_top_waiter(lock);
- plist_add(&waiter->list_entry, &lock->wait_list);
+ rt_mutex_enqueue(lock, waiter);
task->pi_blocked_on = waiter;
@@ -617,8 +639,8 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
raw_spin_lock_irqsave(&owner->pi_lock, flags);
if (waiter == rt_mutex_top_waiter(lock)) {
- plist_del(&top_waiter->pi_list_entry, &owner->pi_waiters);
- plist_add(&waiter->pi_list_entry, &owner->pi_waiters);
+ rt_mutex_dequeue_pi(owner, top_waiter);
+ rt_mutex_enqueue_pi(owner, waiter);
__rt_mutex_adjust_prio(owner);
if (rt_mutex_real_waiter(owner->pi_blocked_on))
@@ -679,7 +701,7 @@ static void wakeup_next_waiter(struct rt_mutex *lock)
* boosted mode and go back to normal after releasing
* lock->wait_lock.
*/
- plist_del(&waiter->pi_list_entry, ¤t->pi_waiters);
+ rt_mutex_dequeue_pi(current, waiter);
/*
* As we are waking up the top waiter, and the waiter stays
@@ -716,7 +738,7 @@ static void remove_waiter(struct rt_mutex *lock,
unsigned long flags;
raw_spin_lock_irqsave(¤t->pi_lock, flags);
- plist_del(&waiter->list_entry, &lock->wait_list);
+ rt_mutex_dequeue(lock, waiter);
current->pi_blocked_on = NULL;
raw_spin_unlock_irqrestore(¤t->pi_lock, flags);
@@ -727,13 +749,13 @@ static void remove_waiter(struct rt_mutex *lock,
raw_spin_lock_irqsave(&owner->pi_lock, flags);
- plist_del(&waiter->pi_list_entry, &owner->pi_waiters);
+ rt_mutex_dequeue_pi(owner, waiter);
if (rt_mutex_has_waiters(lock)) {
struct rt_mutex_waiter *next;
next = rt_mutex_top_waiter(lock);
- plist_add(&next->pi_list_entry, &owner->pi_waiters);
+ rt_mutex_enqueue_pi(owner, next);
}
__rt_mutex_adjust_prio(owner);
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 03/31] rtmutex: Simplify rtmutex_slowtrylock()
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
2015-03-13 15:14 ` [PATCH RT 01/31] gpio: omap: use raw locks for locking Steven Rostedt
2015-03-13 15:14 ` [PATCH RT 02/31] create-rt-enqueue Steven Rostedt
@ 2015-03-13 15:14 ` Steven Rostedt
2015-03-13 15:14 ` [PATCH RT 04/31] rtmutex: Simplify and document try_to_take_rtmutex() Steven Rostedt
` (26 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:14 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker, Oleg Nesterov, Lai Jiangshan
[-- Attachment #1: 0003-rtmutex-Simplify-rtmutex_slowtrylock.patch --]
[-- Type: text/plain, Size: 2456 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: Thomas Gleixner <tglx@linutronix.de>
upstream-commit: 88f2b4c15e561bb5c28709d666364f273bf54b98
Oleg noticed that rtmutex_slowtrylock() has a pointless check for
rt_mutex_owner(lock) != current.
To avoid calling try_to_take_rtmutex() we really want to check whether
the lock has an owner at all or whether the trylock failed because the
owner is NULL, but the RT_MUTEX_HAS_WAITERS bit is set. This covers
the lock is owned by caller situation as well.
We can actually do this check lockless. trylock is taking a chance
whether we take lock->wait_lock to do the check or not.
Add comments to the function while at it.
Reported-by: Oleg Nesterov <oleg@redhat.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Steven Rostedt <rostedt@goodmis.org>
Reviewed-by: Lai Jiangshan <laijs@cn.fujitsu.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Conflicts:
kernel/rtmutex.c
---
kernel/rtmutex.c | 33 +++++++++++++++++++++------------
1 file changed, 21 insertions(+), 12 deletions(-)
diff --git a/kernel/rtmutex.c b/kernel/rtmutex.c
index 7cd751bb891a..c9989ae1cfa5 100644
--- a/kernel/rtmutex.c
+++ b/kernel/rtmutex.c
@@ -1255,24 +1255,33 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state,
/*
* Slow path try-lock function:
*/
-static inline int
-rt_mutex_slowtrylock(struct rt_mutex *lock)
+static inline int rt_mutex_slowtrylock(struct rt_mutex *lock)
{
- int ret = 0;
+ int ret;
+ /*
+ * If the lock already has an owner we fail to get the lock.
+ * This can be done without taking the @lock->wait_lock as
+ * it is only being read, and this is a trylock anyway.
+ */
+ if (rt_mutex_owner(lock))
+ return 0;
+
+ /*
+ * The mutex has currently no owner. Lock the wait lock and
+ * try to acquire the lock.
+ */
if (!raw_spin_trylock(&lock->wait_lock))
- return ret;
+ return 0;
init_lists(lock);
- if (likely(rt_mutex_owner(lock) != current)) {
+ ret = try_to_take_rt_mutex(lock, current, NULL);
- ret = try_to_take_rt_mutex(lock, current, NULL);
- /*
- * try_to_take_rt_mutex() sets the lock waiters
- * bit unconditionally. Clean this up.
- */
- fixup_rt_mutex_waiters(lock);
- }
+ /*
+ * try_to_take_rt_mutex() sets the lock waiters bit
+ * unconditionally. Clean this up.
+ */
+ fixup_rt_mutex_waiters(lock);
raw_spin_unlock(&lock->wait_lock);
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 04/31] rtmutex: Simplify and document try_to_take_rtmutex()
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
` (2 preceding siblings ...)
2015-03-13 15:14 ` [PATCH RT 03/31] rtmutex: Simplify rtmutex_slowtrylock() Steven Rostedt
@ 2015-03-13 15:14 ` Steven Rostedt
2015-03-13 15:14 ` [PATCH RT 05/31] rtmutex: No need to keep task ref for lock owner check Steven Rostedt
` (25 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:14 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker
[-- Attachment #1: 0004-rtmutex-Simplify-and-document-try_to_take_rtmutex.patch --]
[-- Type: text/plain, Size: 6423 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: Thomas Gleixner <tglx@linutronix.de>
upstream commit: 358c331f391f3e0432f4f96f25017d12ac8d10b1
The current implementation of try_to_take_rtmutex() is correct, but
requires more than a single brain twist to understand the clever
encoded conditionals.
Untangle it and document the cases proper.
Looks less efficient at the first glance, but actually reduces the
binary code size on x8664 by 80 bytes.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Conflicts:
kernel/rtmutex.c
---
kernel/rtmutex.c | 134 ++++++++++++++++++++++++++++++++++++-------------------
1 file changed, 89 insertions(+), 45 deletions(-)
diff --git a/kernel/rtmutex.c b/kernel/rtmutex.c
index c9989ae1cfa5..1052a337fa2c 100644
--- a/kernel/rtmutex.c
+++ b/kernel/rtmutex.c
@@ -485,78 +485,122 @@ static inline int lock_is_stealable(struct task_struct *task,
*
* Must be called with lock->wait_lock held.
*
- * @lock: the lock to be acquired.
- * @task: the task which wants to acquire the lock
- * @waiter: the waiter that is queued to the lock's wait list. (could be NULL)
+ * @lock: The lock to be acquired.
+ * @task: The task which wants to acquire the lock
+ * @waiter: The waiter that is queued to the lock's wait list if the
+ * callsite called task_blocked_on_lock(), otherwise NULL
*/
static int
__try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
struct rt_mutex_waiter *waiter, int mode)
{
+ unsigned long flags;
+
/*
- * We have to be careful here if the atomic speedups are
- * enabled, such that, when
- * - no other waiter is on the lock
- * - the lock has been released since we did the cmpxchg
- * the lock can be released or taken while we are doing the
- * checks and marking the lock with RT_MUTEX_HAS_WAITERS.
+ * Before testing whether we can acquire @lock, we set the
+ * RT_MUTEX_HAS_WAITERS bit in @lock->owner. This forces all
+ * other tasks which try to modify @lock into the slow path
+ * and they serialize on @lock->wait_lock.
*
- * The atomic acquire/release aware variant of
- * mark_rt_mutex_waiters uses a cmpxchg loop. After setting
- * the WAITERS bit, the atomic release / acquire can not
- * happen anymore and lock->wait_lock protects us from the
- * non-atomic case.
+ * The RT_MUTEX_HAS_WAITERS bit can have a transitional state
+ * as explained at the top of this file if and only if:
*
- * Note, that this might set lock->owner =
- * RT_MUTEX_HAS_WAITERS in the case the lock is not contended
- * any more. This is fixed up when we take the ownership.
- * This is the transitional state explained at the top of this file.
+ * - There is a lock owner. The caller must fixup the
+ * transient state if it does a trylock or leaves the lock
+ * function due to a signal or timeout.
+ *
+ * - @task acquires the lock and there are no other
+ * waiters. This is undone in rt_mutex_set_owner(@task) at
+ * the end of this function.
*/
mark_rt_mutex_waiters(lock);
+ /*
+ * If @lock has an owner, give up.
+ */
if (rt_mutex_owner(lock))
return 0;
/*
- * It will get the lock because of one of these conditions:
- * 1) there is no waiter
- * 2) higher priority than waiters
- * 3) it is top waiter
+ * If @waiter != NULL, @task has already enqueued the waiter
+ * into @lock waiter list. If @waiter == NULL then this is a
+ * trylock attempt.
*/
- if (rt_mutex_has_waiters(lock)) {
- struct task_struct *pown = rt_mutex_top_waiter(lock)->task;
-
- if (task != pown && !lock_is_stealable(task, pown, mode))
+ if (waiter) {
+ /*
+ * If waiter is not the highest priority waiter of
+ * @lock, give up.
+ */
+ if (waiter != rt_mutex_top_waiter(lock))
return 0;
- }
-
- /* We got the lock. */
- if (waiter || rt_mutex_has_waiters(lock)) {
- unsigned long flags;
- struct rt_mutex_waiter *top;
-
- raw_spin_lock_irqsave(&task->pi_lock, flags);
-
- /* remove the queued waiter. */
- if (waiter) {
- rt_mutex_dequeue(lock, waiter);
- task->pi_blocked_on = NULL;
- }
+ /*
+ * We can acquire the lock. Remove the waiter from the
+ * lock waiters list.
+ */
+ rt_mutex_dequeue(lock, waiter);
+ } else {
/*
- * We have to enqueue the top waiter(if it exists) into
- * task->pi_waiters list.
+ * If the lock has waiters already we check whether @task is
+ * eligible to take over the lock.
+ *
+ * If there are no other waiters, @task can acquire
+ * the lock. @task->pi_blocked_on is NULL, so it does
+ * not need to be dequeued.
*/
if (rt_mutex_has_waiters(lock)) {
- top = rt_mutex_top_waiter(lock);
- rt_mutex_enqueue_pi(task, top);
+ struct task_struct *pown = rt_mutex_top_waiter(lock)->task;
+
+ /*
+ * If @task->prio is greater than or equal to
+ * the top waiter priority (kernel view),
+ * @task lost.
+ */
+ if (task != pown && !lock_is_stealable(task, pown, mode))
+ return 0;
+
+ /*
+ * The current top waiter stays enqueued. We
+ * don't have to change anything in the lock
+ * waiters order.
+ */
+ } else {
+ /*
+ * No waiters. Take the lock without the
+ * pi_lock dance.@task->pi_blocked_on is NULL
+ * and we have no waiters to enqueue in @task
+ * pi waiters list.
+ */
+ goto takeit;
}
- raw_spin_unlock_irqrestore(&task->pi_lock, flags);
}
+ /*
+ * Clear @task->pi_blocked_on. Requires protection by
+ * @task->pi_lock. Redundant operation for the @waiter == NULL
+ * case, but conditionals are more expensive than a redundant
+ * store.
+ */
+ raw_spin_lock_irqsave(&task->pi_lock, flags);
+ task->pi_blocked_on = NULL;
+ /*
+ * Finish the lock acquisition. @task is the new owner. If
+ * other waiters exist we have to insert the highest priority
+ * waiter into @task->pi_waiters list.
+ */
+ if (rt_mutex_has_waiters(lock))
+ rt_mutex_enqueue_pi(task, rt_mutex_top_waiter(lock));
+ raw_spin_unlock_irqrestore(&task->pi_lock, flags);
+
+takeit:
+ /* We got the lock. */
debug_rt_mutex_lock(lock);
+ /*
+ * This either preserves the RT_MUTEX_HAS_WAITERS bit if there
+ * are still waiters or clears it.
+ */
rt_mutex_set_owner(lock, task);
rt_mutex_deadlock_account_lock(lock, task);
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 05/31] rtmutex: No need to keep task ref for lock owner check
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
` (3 preceding siblings ...)
2015-03-13 15:14 ` [PATCH RT 04/31] rtmutex: Simplify and document try_to_take_rtmutex() Steven Rostedt
@ 2015-03-13 15:14 ` Steven Rostedt
2015-03-13 15:15 ` [PATCH RT 06/31] rtmutex: Clarify the boost/deboost part Steven Rostedt
` (24 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:14 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker, Lai Jiangshan
[-- Attachment #1: 0005-rtmutex-No-need-to-keep-task-ref-for-lock-owner-chec.patch --]
[-- Type: text/plain, Size: 1416 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: Thomas Gleixner <tglx@linutronix.de>
upstream commit: 2ffa5a5cd2fe792b6399c903d5172adf088d8ff7
There is no point to keep the task ref across the check for lock
owner. Drop the ref before that, so the protection context is clear.
Found while documenting the chain walk.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Steven Rostedt <rostedt@goodmis.org>
Reviewed-by: Lai Jiangshan <laijs@cn.fujitsu.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
kernel/rtmutex.c | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/kernel/rtmutex.c b/kernel/rtmutex.c
index 1052a337fa2c..eb748b542095 100644
--- a/kernel/rtmutex.c
+++ b/kernel/rtmutex.c
@@ -394,6 +394,8 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
/* Release the task */
raw_spin_unlock_irqrestore(&task->pi_lock, flags);
+ put_task_struct(task);
+
if (!rt_mutex_owner(lock)) {
struct rt_mutex_waiter *lock_top_waiter;
@@ -405,9 +407,8 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
if (top_waiter != lock_top_waiter)
rt_mutex_wake_waiter(lock_top_waiter);
raw_spin_unlock(&lock->wait_lock);
- goto out_put_task;
+ return 0;
}
- put_task_struct(task);
/* Grab the next task */
task = rt_mutex_owner(lock);
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 06/31] rtmutex: Clarify the boost/deboost part
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
` (4 preceding siblings ...)
2015-03-13 15:14 ` [PATCH RT 05/31] rtmutex: No need to keep task ref for lock owner check Steven Rostedt
@ 2015-03-13 15:15 ` Steven Rostedt
2015-03-13 15:15 ` Steven Rostedt
` (23 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker
[-- Attachment #1: 0006-rtmutex-Clarify-the-boost-deboost-part.patch --]
[-- Type: text/plain, Size: 4763 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: Thomas Gleixner <tglx@linutronix.de>
upstream commit: a57594a13a446d1a6ab1dcd48339f799ce586843
Add a separate local variable for the boost/deboost logic to make the
code more readable. Add comments where appropriate.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Conflicts:
kernel/rtmutex.c
---
kernel/rtmutex.c | 57 +++++++++++++++++++++++++++++++++++++++++++++++---------
1 file changed, 48 insertions(+), 9 deletions(-)
diff --git a/kernel/rtmutex.c b/kernel/rtmutex.c
index eb748b542095..4384a96bd0a0 100644
--- a/kernel/rtmutex.c
+++ b/kernel/rtmutex.c
@@ -276,9 +276,10 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
struct rt_mutex_waiter *orig_waiter,
struct task_struct *top_task)
{
- struct rt_mutex *lock;
struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter;
+ struct rt_mutex_waiter *prerequeue_top_waiter;
int detect_deadlock, ret = 0, depth = 0;
+ struct rt_mutex *lock;
unsigned long flags;
detect_deadlock = debug_rt_mutex_detect_deadlock(orig_waiter,
@@ -385,9 +386,14 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
goto out_unlock_pi;
}
- top_waiter = rt_mutex_top_waiter(lock);
+ /*
+ * Store the current top waiter before doing the requeue
+ * operation on @lock. We need it for the boost/deboost
+ * decision below.
+ */
+ prerequeue_top_waiter = rt_mutex_top_waiter(lock);
- /* Requeue the waiter */
+ /* Requeue the waiter in the lock waiter list. */
rt_mutex_dequeue(lock, waiter);
waiter->list_entry.prio = task->prio;
rt_mutex_enqueue(lock, waiter);
@@ -396,6 +402,11 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
raw_spin_unlock_irqrestore(&task->pi_lock, flags);
put_task_struct(task);
+ /*
+ * We must abort the chain walk if there is no lock owner even
+ * in the dead lock detection case, as we have nothing to
+ * follow here. This is the end of the chain we are walking.
+ */
if (!rt_mutex_owner(lock)) {
struct rt_mutex_waiter *lock_top_waiter;
@@ -404,29 +415,48 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
* to wake the new top waiter up to try to get the lock.
*/
lock_top_waiter = rt_mutex_top_waiter(lock);
- if (top_waiter != lock_top_waiter)
+ if (prerequeue_top_waiter != lock_top_waiter)
rt_mutex_wake_waiter(lock_top_waiter);
raw_spin_unlock(&lock->wait_lock);
return 0;
}
- /* Grab the next task */
+ /* Grab the next task, i.e. the owner of @lock */
task = rt_mutex_owner(lock);
get_task_struct(task);
raw_spin_lock_irqsave(&task->pi_lock, flags);
if (waiter == rt_mutex_top_waiter(lock)) {
- /* Boost the owner */
- rt_mutex_dequeue_pi(task, top_waiter);
+ /*
+ * The waiter became the new top (highest priority)
+ * waiter on the lock. Replace the previous top waiter
+ * in the owner tasks pi waiters list with this waiter
+ * and adjust the priority of the owner.
+ */
+ rt_mutex_dequeue_pi(task, prerequeue_top_waiter);
rt_mutex_enqueue_pi(task, waiter);
__rt_mutex_adjust_prio(task);
- } else if (top_waiter == waiter) {
- /* Deboost the owner */
+ } else if (prerequeue_top_waiter == waiter) {
+ /*
+ * The waiter was the top waiter on the lock, but is
+ * no longer the top prority waiter. Replace waiter in
+ * the owner tasks pi waiters list with the new top
+ * (highest priority) waiter and adjust the priority
+ * of the owner.
+ * The new top waiter is stored in @waiter so that
+ * @waiter == @top_waiter evaluates to true below and
+ * we continue to deboost the rest of the chain.
+ */
rt_mutex_dequeue_pi(task, waiter);
waiter = rt_mutex_top_waiter(lock);
rt_mutex_enqueue_pi(task, waiter);
__rt_mutex_adjust_prio(task);
+ } else {
+ /*
+ * Nothing changed. No need to do any priority
+ * adjustment.
+ */
}
/*
@@ -439,6 +469,10 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
raw_spin_unlock_irqrestore(&task->pi_lock, flags);
+ /*
+ * Store the top waiter of @lock for the end of chain walk
+ * decision below.
+ */
top_waiter = rt_mutex_top_waiter(lock);
raw_spin_unlock(&lock->wait_lock);
@@ -449,6 +483,11 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
if (!next_lock)
goto out_put_task;
+ /*
+ * If the current waiter is not the top waiter on the lock,
+ * then we can stop the chain walk here if we are not in full
+ * deadlock detection mode.
+ */
if (!detect_deadlock && waiter != top_waiter)
goto out_put_task;
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 07/31] rtmutex: Document pi chain walk
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
@ 2015-03-13 15:15 ` Steven Rostedt
2015-03-13 15:14 ` [PATCH RT 02/31] create-rt-enqueue Steven Rostedt
` (28 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker
[-- Attachment #1: 0007-rtmutex-Document-pi-chain-walk.patch --]
[-- Type: text/plain, Size: 7324 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: Thomas Gleixner <tglx@linutronix.de>
upstream commit: 3eb65aeadf701976b084e9171e16bb7d1e83fbb0
Add commentry to document the chain walk and the protection mechanisms
and their scope.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
kernel/rtmutex.c | 100 ++++++++++++++++++++++++++++++++++++++++++++++++++-----
1 file changed, 91 insertions(+), 9 deletions(-)
diff --git a/kernel/rtmutex.c b/kernel/rtmutex.c
index 4384a96bd0a0..b6c48cb7f337 100644
--- a/kernel/rtmutex.c
+++ b/kernel/rtmutex.c
@@ -268,6 +268,48 @@ static inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p)
* Adjust the priority chain. Also used for deadlock detection.
* Decreases task's usage by one - may thus free the task.
* Returns 0 or -EDEADLK.
+ *
+ * Chain walk basics and protection scope
+ *
+ * [R] refcount on task
+ * [P] task->pi_lock held
+ * [L] rtmutex->wait_lock held
+ *
+ * Step Description Protected by
+ * function arguments:
+ * @task [R]
+ * @orig_lock if != NULL @top_task is blocked on it
+ * @next_lock Unprotected. Cannot be
+ * dereferenced. Only used for
+ * comparison.
+ * @orig_waiter if != NULL @top_task is blocked on it
+ * @top_task current, or in case of proxy
+ * locking protected by calling
+ * code
+ * again:
+ * loop_sanity_check();
+ * retry:
+ * [1] lock(task->pi_lock); [R] acquire [P]
+ * [2] waiter = task->pi_blocked_on; [P]
+ * [3] check_exit_conditions_1(); [P]
+ * [4] lock = waiter->lock; [P]
+ * [5] if (!try_lock(lock->wait_lock)) { [P] try to acquire [L]
+ * unlock(task->pi_lock); release [P]
+ * goto retry;
+ * }
+ * [6] check_exit_conditions_2(); [P] + [L]
+ * [7] requeue_lock_waiter(lock, waiter); [P] + [L]
+ * [8] unlock(task->pi_lock); release [P]
+ * put_task_struct(task); release [R]
+ * [9] check_exit_conditions_3(); [L]
+ * [10] task = owner(lock); [L]
+ * get_task_struct(task); [L] acquire [R]
+ * lock(task->pi_lock); [L] acquire [P]
+ * [11] requeue_pi_waiter(tsk, waiters(lock));[P] + [L]
+ * [12] check_exit_conditions_4(); [P] + [L]
+ * [13] unlock(task->pi_lock); release [P]
+ * unlock(lock->wait_lock); release [L]
+ * goto again;
*/
static int rt_mutex_adjust_prio_chain(struct task_struct *task,
int deadlock_detect,
@@ -292,6 +334,9 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
* carefully whether things change under us.
*/
again:
+ /*
+ * We limit the lock chain length for each invocation.
+ */
if (++depth > max_lock_depth) {
static int prev_max;
@@ -309,13 +354,28 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
return -EDEADLK;
}
+
+ /*
+ * We are fully preemptible here and only hold the refcount on
+ * @task. So everything can have changed under us since the
+ * caller or our own code below (goto retry/again) dropped all
+ * locks.
+ */
retry:
/*
- * Task can not go away as we did a get_task() before !
+ * [1] Task cannot go away as we did a get_task() before !
*/
raw_spin_lock_irqsave(&task->pi_lock, flags);
+ /*
+ * [2] Get the waiter on which @task is blocked on.
+ */
waiter = task->pi_blocked_on;
+
+ /*
+ * [3] check_exit_conditions_1() protected by task->pi_lock.
+ */
+
/*
* Check whether the end of the boosting chain has been
* reached or the state of the chain has changed while we
@@ -366,7 +426,15 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
if (!detect_deadlock && waiter->list_entry.prio == task->prio)
goto out_unlock_pi;
+ /*
+ * [4] Get the next lock
+ */
lock = waiter->lock;
+ /*
+ * [5] We need to trylock here as we are holding task->pi_lock,
+ * which is the reverse lock order versus the other rtmutex
+ * operations.
+ */
if (!raw_spin_trylock(&lock->wait_lock)) {
raw_spin_unlock_irqrestore(&task->pi_lock, flags);
cpu_relax();
@@ -374,6 +442,9 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
}
/*
+ * [6] check_exit_conditions_2() protected by task->pi_lock and
+ * lock->wait_lock.
+ *
* Deadlock detection. If the lock is the same as the original
* lock which caused us to walk the lock chain or if the
* current lock is owned by the task which initiated the chain
@@ -393,16 +464,18 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
*/
prerequeue_top_waiter = rt_mutex_top_waiter(lock);
- /* Requeue the waiter in the lock waiter list. */
+ /* [7] Requeue the waiter in the lock waiter list. */
rt_mutex_dequeue(lock, waiter);
waiter->list_entry.prio = task->prio;
rt_mutex_enqueue(lock, waiter);
- /* Release the task */
+ /* [8] Release the task */
raw_spin_unlock_irqrestore(&task->pi_lock, flags);
put_task_struct(task);
/*
+ * [9] check_exit_conditions_3 protected by lock->wait_lock.
+ *
* We must abort the chain walk if there is no lock owner even
* in the dead lock detection case, as we have nothing to
* follow here. This is the end of the chain we are walking.
@@ -411,8 +484,9 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
struct rt_mutex_waiter *lock_top_waiter;
/*
- * If the requeue above changed the top waiter, then we need
- * to wake the new top waiter up to try to get the lock.
+ * If the requeue [7] above changed the top waiter,
+ * then we need to wake the new top waiter up to try
+ * to get the lock.
*/
lock_top_waiter = rt_mutex_top_waiter(lock);
if (prerequeue_top_waiter != lock_top_waiter)
@@ -421,11 +495,12 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
return 0;
}
- /* Grab the next task, i.e. the owner of @lock */
+ /* [10] Grab the next task, i.e. the owner of @lock */
task = rt_mutex_owner(lock);
get_task_struct(task);
raw_spin_lock_irqsave(&task->pi_lock, flags);
+ /* [11] requeue the pi waiters if necessary */
if (waiter == rt_mutex_top_waiter(lock)) {
/*
* The waiter became the new top (highest priority)
@@ -460,23 +535,30 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
}
/*
+ * [12] check_exit_conditions_4() protected by task->pi_lock
+ * and lock->wait_lock. The actual decisions are made after we
+ * dropped the locks.
+ *
* Check whether the task which owns the current lock is pi
* blocked itself. If yes we store a pointer to the lock for
* the lock chain change detection above. After we dropped
* task->pi_lock next_lock cannot be dereferenced anymore.
*/
next_lock = task_blocked_on_lock(task);
-
- raw_spin_unlock_irqrestore(&task->pi_lock, flags);
-
/*
* Store the top waiter of @lock for the end of chain walk
* decision below.
*/
top_waiter = rt_mutex_top_waiter(lock);
+
+ /* [13] Drop the locks */
+ raw_spin_unlock_irqrestore(&task->pi_lock, flags);
raw_spin_unlock(&lock->wait_lock);
/*
+ * Make the actual exit decisions [12], based on the stored
+ * values.
+ *
* We reached the end of the lock chain. Stop right here. No
* point to go back just to figure that out.
*/
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 07/31] rtmutex: Document pi chain walk
@ 2015-03-13 15:15 ` Steven Rostedt
0 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker
[-- Attachment #1: 0007-rtmutex-Document-pi-chain-walk.patch --]
[-- Type: text/plain, Size: 6852 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: Thomas Gleixner <tglx@linutronix.de>
upstream commit: 3eb65aeadf701976b084e9171e16bb7d1e83fbb0
Add commentry to document the chain walk and the protection mechanisms
and their scope.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
kernel/rtmutex.c | 100 ++++++++++++++++++++++++++++++++++++++++++++++++++-----
1 file changed, 91 insertions(+), 9 deletions(-)
diff --git a/kernel/rtmutex.c b/kernel/rtmutex.c
index 4384a96bd0a0..b6c48cb7f337 100644
--- a/kernel/rtmutex.c
+++ b/kernel/rtmutex.c
@@ -268,6 +268,48 @@ static inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p)
* Adjust the priority chain. Also used for deadlock detection.
* Decreases task's usage by one - may thus free the task.
* Returns 0 or -EDEADLK.
+ *
+ * Chain walk basics and protection scope
+ *
+ * [R] refcount on task
+ * [P] task->pi_lock held
+ * [L] rtmutex->wait_lock held
+ *
+ * Step Description Protected by
+ * function arguments:
+ * @task [R]
+ * @orig_lock if != NULL @top_task is blocked on it
+ * @next_lock Unprotected. Cannot be
+ * dereferenced. Only used for
+ * comparison.
+ * @orig_waiter if != NULL @top_task is blocked on it
+ * @top_task current, or in case of proxy
+ * locking protected by calling
+ * code
+ * again:
+ * loop_sanity_check();
+ * retry:
+ * [1] lock(task->pi_lock); [R] acquire [P]
+ * [2] waiter = task->pi_blocked_on; [P]
+ * [3] check_exit_conditions_1(); [P]
+ * [4] lock = waiter->lock; [P]
+ * [5] if (!try_lock(lock->wait_lock)) { [P] try to acquire [L]
+ * unlock(task->pi_lock); release [P]
+ * goto retry;
+ * }
+ * [6] check_exit_conditions_2(); [P] + [L]
+ * [7] requeue_lock_waiter(lock, waiter); [P] + [L]
+ * [8] unlock(task->pi_lock); release [P]
+ * put_task_struct(task); release [R]
+ * [9] check_exit_conditions_3(); [L]
+ * [10] task = owner(lock); [L]
+ * get_task_struct(task); [L] acquire [R]
+ * lock(task->pi_lock); [L] acquire [P]
+ * [11] requeue_pi_waiter(tsk, waiters(lock));[P] + [L]
+ * [12] check_exit_conditions_4(); [P] + [L]
+ * [13] unlock(task->pi_lock); release [P]
+ * unlock(lock->wait_lock); release [L]
+ * goto again;
*/
static int rt_mutex_adjust_prio_chain(struct task_struct *task,
int deadlock_detect,
@@ -292,6 +334,9 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
* carefully whether things change under us.
*/
again:
+ /*
+ * We limit the lock chain length for each invocation.
+ */
if (++depth > max_lock_depth) {
static int prev_max;
@@ -309,13 +354,28 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
return -EDEADLK;
}
+
+ /*
+ * We are fully preemptible here and only hold the refcount on
+ * @task. So everything can have changed under us since the
+ * caller or our own code below (goto retry/again) dropped all
+ * locks.
+ */
retry:
/*
- * Task can not go away as we did a get_task() before !
+ * [1] Task cannot go away as we did a get_task() before !
*/
raw_spin_lock_irqsave(&task->pi_lock, flags);
+ /*
+ * [2] Get the waiter on which @task is blocked on.
+ */
waiter = task->pi_blocked_on;
+
+ /*
+ * [3] check_exit_conditions_1() protected by task->pi_lock.
+ */
+
/*
* Check whether the end of the boosting chain has been
* reached or the state of the chain has changed while we
@@ -366,7 +426,15 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
if (!detect_deadlock && waiter->list_entry.prio == task->prio)
goto out_unlock_pi;
+ /*
+ * [4] Get the next lock
+ */
lock = waiter->lock;
+ /*
+ * [5] We need to trylock here as we are holding task->pi_lock,
+ * which is the reverse lock order versus the other rtmutex
+ * operations.
+ */
if (!raw_spin_trylock(&lock->wait_lock)) {
raw_spin_unlock_irqrestore(&task->pi_lock, flags);
cpu_relax();
@@ -374,6 +442,9 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
}
/*
+ * [6] check_exit_conditions_2() protected by task->pi_lock and
+ * lock->wait_lock.
+ *
* Deadlock detection. If the lock is the same as the original
* lock which caused us to walk the lock chain or if the
* current lock is owned by the task which initiated the chain
@@ -393,16 +464,18 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
*/
prerequeue_top_waiter = rt_mutex_top_waiter(lock);
- /* Requeue the waiter in the lock waiter list. */
+ /* [7] Requeue the waiter in the lock waiter list. */
rt_mutex_dequeue(lock, waiter);
waiter->list_entry.prio = task->prio;
rt_mutex_enqueue(lock, waiter);
- /* Release the task */
+ /* [8] Release the task */
raw_spin_unlock_irqrestore(&task->pi_lock, flags);
put_task_struct(task);
/*
+ * [9] check_exit_conditions_3 protected by lock->wait_lock.
+ *
* We must abort the chain walk if there is no lock owner even
* in the dead lock detection case, as we have nothing to
* follow here. This is the end of the chain we are walking.
@@ -411,8 +484,9 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
struct rt_mutex_waiter *lock_top_waiter;
/*
- * If the requeue above changed the top waiter, then we need
- * to wake the new top waiter up to try to get the lock.
+ * If the requeue [7] above changed the top waiter,
+ * then we need to wake the new top waiter up to try
+ * to get the lock.
*/
lock_top_waiter = rt_mutex_top_waiter(lock);
if (prerequeue_top_waiter != lock_top_waiter)
@@ -421,11 +495,12 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
return 0;
}
- /* Grab the next task, i.e. the owner of @lock */
+ /* [10] Grab the next task, i.e. the owner of @lock */
task = rt_mutex_owner(lock);
get_task_struct(task);
raw_spin_lock_irqsave(&task->pi_lock, flags);
+ /* [11] requeue the pi waiters if necessary */
if (waiter == rt_mutex_top_waiter(lock)) {
/*
* The waiter became the new top (highest priority)
@@ -460,23 +535,30 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
}
/*
+ * [12] check_exit_conditions_4() protected by task->pi_lock
+ * and lock->wait_lock. The actual decisions are made after we
+ * dropped the locks.
+ *
* Check whether the task which owns the current lock is pi
* blocked itself. If yes we store a pointer to the lock for
* the lock chain change detection above. After we dropped
* task->pi_lock next_lock cannot be dereferenced anymore.
*/
next_lock = task_blocked_on_lock(task);
-
- raw_spin_unlock_irqrestore(&task->pi_lock, flags);
^ permalink raw reply related [flat|nested] 33+ messages in thread
* [PATCH RT 08/31] rtmutex: Simplify remove_waiter()
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
` (6 preceding siblings ...)
2015-03-13 15:15 ` Steven Rostedt
@ 2015-03-13 15:15 ` Steven Rostedt
2015-03-13 15:15 ` [PATCH RT 09/31] rtmutex: Confine deadlock logic to futex Steven Rostedt
` (21 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker, Lai Jiangshan
[-- Attachment #1: 0008-rtmutex-Simplify-remove_waiter.patch --]
[-- Type: text/plain, Size: 2583 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: Thomas Gleixner <tglx@linutronix.de>
upstream commit: 1ca7b86062ec8473d03c5cdfd336abc8b1c8098c
Exit right away, when the removed waiter was not the top priority
waiter on the lock. Get rid of the extra indent level.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Steven Rostedt <rostedt@goodmis.org>
Reviewed-by: Lai Jiangshan <laijs@cn.fujitsu.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Conflicts:
kernel/rtmutex.c
---
kernel/rtmutex.c | 33 ++++++++++++++++++---------------
1 file changed, 18 insertions(+), 15 deletions(-)
diff --git a/kernel/rtmutex.c b/kernel/rtmutex.c
index b6c48cb7f337..823f782b1847 100644
--- a/kernel/rtmutex.c
+++ b/kernel/rtmutex.c
@@ -898,7 +898,7 @@ static void wakeup_next_waiter(struct rt_mutex *lock)
static void remove_waiter(struct rt_mutex *lock,
struct rt_mutex_waiter *waiter)
{
- int first = (waiter == rt_mutex_top_waiter(lock));
+ bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock));
struct task_struct *owner = rt_mutex_owner(lock);
struct rt_mutex *next_lock = NULL;
unsigned long flags;
@@ -908,31 +908,34 @@ static void remove_waiter(struct rt_mutex *lock,
current->pi_blocked_on = NULL;
raw_spin_unlock_irqrestore(¤t->pi_lock, flags);
- if (!owner)
+ /*
+ * Only update priority if the waiter was the highest priority
+ * waiter of the lock and there is an owner to update.
+ */
+ if (!owner || !is_top_waiter)
return;
- if (first) {
-
- raw_spin_lock_irqsave(&owner->pi_lock, flags);
+ raw_spin_lock_irqsave(&owner->pi_lock, flags);
- rt_mutex_dequeue_pi(owner, waiter);
+ rt_mutex_dequeue_pi(owner, waiter);
- if (rt_mutex_has_waiters(lock)) {
- struct rt_mutex_waiter *next;
+ if (rt_mutex_has_waiters(lock))
+ rt_mutex_enqueue_pi(owner, rt_mutex_top_waiter(lock));
- next = rt_mutex_top_waiter(lock);
- rt_mutex_enqueue_pi(owner, next);
- }
- __rt_mutex_adjust_prio(owner);
+ __rt_mutex_adjust_prio(owner);
- /* Store the lock on which owner is blocked or NULL */
+ /* Store the lock on which owner is blocked or NULL */
+ if (rt_mutex_real_waiter(owner->pi_blocked_on))
next_lock = task_blocked_on_lock(owner);
- raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
- }
+ raw_spin_unlock_irqrestore(&owner->pi_lock, flags);
WARN_ON(!plist_node_empty(&waiter->pi_list_entry));
+ /*
+ * Don't walk the chain, if the owner task is not blocked
+ * itself.
+ */
if (!next_lock)
return;
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 09/31] rtmutex: Confine deadlock logic to futex
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
` (7 preceding siblings ...)
2015-03-13 15:15 ` [PATCH RT 08/31] rtmutex: Simplify remove_waiter() Steven Rostedt
@ 2015-03-13 15:15 ` Steven Rostedt
2015-03-13 15:15 ` [PATCH RT 10/31] rtmutex: Cleanup deadlock detector debug logic Steven Rostedt
` (20 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker
[-- Attachment #1: 0009-rtmutex-Confine-deadlock-logic-to-futex.patch --]
[-- Type: text/plain, Size: 11133 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: Thomas Gleixner <tglx@linutronix.de>
upstream commit: c051b21f71d1ffdfd7ad406a1ef5ede5e5f974c5
The deadlock logic is only required for futexes.
Remove the extra arguments for the public functions and also for the
futex specific ones which get always called with deadlock detection
enabled.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Steven Rostedt <rostedt@goodmis.org>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Conflicts:
include/linux/rtmutex.h
kernel/rtmutex.c
---
include/linux/rtmutex.h | 8 +++-----
kernel/futex.c | 10 ++++-----
kernel/rt.c | 8 ++++----
kernel/rtmutex-tester.c | 4 ++--
kernel/rtmutex.c | 54 ++++++++++++++++++++++++-------------------------
kernel/rtmutex_common.h | 7 +++----
6 files changed, 43 insertions(+), 48 deletions(-)
diff --git a/include/linux/rtmutex.h b/include/linux/rtmutex.h
index 5ebd0bbb6eaa..9340b866ce21 100644
--- a/include/linux/rtmutex.h
+++ b/include/linux/rtmutex.h
@@ -109,12 +109,10 @@ extern void __rt_mutex_init(struct rt_mutex *lock, const char *name);
extern void rt_mutex_destroy(struct rt_mutex *lock);
extern void rt_mutex_lock(struct rt_mutex *lock);
-extern int rt_mutex_lock_interruptible(struct rt_mutex *lock,
- int detect_deadlock);
-extern int rt_mutex_lock_killable(struct rt_mutex *lock, int detect_deadlock);
+extern int rt_mutex_lock_interruptible(struct rt_mutex *lock);
+extern int rt_mutex_lock_killable(struct rt_mutex *lock);
extern int rt_mutex_timed_lock(struct rt_mutex *lock,
- struct hrtimer_sleeper *timeout,
- int detect_deadlock);
+ struct hrtimer_sleeper *timeout);
extern int rt_mutex_trylock(struct rt_mutex *lock);
diff --git a/kernel/futex.c b/kernel/futex.c
index ae1068fdd786..75f386961a8d 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -1568,7 +1568,7 @@ retry_private:
this->pi_state = pi_state;
ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex,
this->rt_waiter,
- this->task, 1);
+ this->task);
if (ret == 1) {
/* We got the lock. */
requeue_pi_wake_futex(this, &key2, hb2);
@@ -2184,9 +2184,9 @@ retry_private:
/*
* Block on the PI mutex:
*/
- if (!trylock)
- ret = rt_mutex_timed_lock(&q.pi_state->pi_mutex, to, 1);
- else {
+ if (!trylock) {
+ ret = rt_mutex_timed_futex_lock(&q.pi_state->pi_mutex, to);
+ } else {
ret = rt_mutex_trylock(&q.pi_state->pi_mutex);
/* Fixup the trylock return value: */
ret = ret ? 0 : -EWOULDBLOCK;
@@ -2551,7 +2551,7 @@ static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags,
*/
WARN_ON(!q.pi_state);
pi_mutex = &q.pi_state->pi_mutex;
- ret = rt_mutex_finish_proxy_lock(pi_mutex, to, &rt_waiter, 1);
+ ret = rt_mutex_finish_proxy_lock(pi_mutex, to, &rt_waiter);
debug_rt_mutex_free_waiter(&rt_waiter);
spin_lock(&hb2->lock);
diff --git a/kernel/rt.c b/kernel/rt.c
index 433ae4236181..1b3497d3b848 100644
--- a/kernel/rt.c
+++ b/kernel/rt.c
@@ -98,7 +98,7 @@ int __lockfunc _mutex_lock_interruptible(struct mutex *lock)
int ret;
mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_);
- ret = rt_mutex_lock_interruptible(&lock->lock, 0);
+ ret = rt_mutex_lock_interruptible(&lock->lock);
if (ret)
mutex_release(&lock->dep_map, 1, _RET_IP_);
return ret;
@@ -110,7 +110,7 @@ int __lockfunc _mutex_lock_killable(struct mutex *lock)
int ret;
mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_);
- ret = rt_mutex_lock_killable(&lock->lock, 0);
+ ret = rt_mutex_lock_killable(&lock->lock);
if (ret)
mutex_release(&lock->dep_map, 1, _RET_IP_);
return ret;
@@ -137,7 +137,7 @@ int __lockfunc _mutex_lock_interruptible_nested(struct mutex *lock, int subclass
int ret;
mutex_acquire_nest(&lock->dep_map, subclass, 0, NULL, _RET_IP_);
- ret = rt_mutex_lock_interruptible(&lock->lock, 0);
+ ret = rt_mutex_lock_interruptible(&lock->lock);
if (ret)
mutex_release(&lock->dep_map, 1, _RET_IP_);
return ret;
@@ -149,7 +149,7 @@ int __lockfunc _mutex_lock_killable_nested(struct mutex *lock, int subclass)
int ret;
mutex_acquire(&lock->dep_map, subclass, 0, _RET_IP_);
- ret = rt_mutex_lock_killable(&lock->lock, 0);
+ ret = rt_mutex_lock_killable(&lock->lock);
if (ret)
mutex_release(&lock->dep_map, 1, _RET_IP_);
return ret;
diff --git a/kernel/rtmutex-tester.c b/kernel/rtmutex-tester.c
index 1d96dd0d93c1..a6b605b51b97 100644
--- a/kernel/rtmutex-tester.c
+++ b/kernel/rtmutex-tester.c
@@ -16,7 +16,7 @@
#include <linux/freezer.h>
#include <linux/stat.h>
-#include "rtmutex.h"
+#include "rtmutex_common.h"
#define MAX_RT_TEST_THREADS 8
#define MAX_RT_TEST_MUTEXES 8
@@ -107,7 +107,7 @@ static int handle_op(struct test_thread_data *td, int lockwakeup)
td->mutexes[id] = 1;
td->event = atomic_add_return(1, &rttest_event);
- ret = rt_mutex_lock_interruptible(&mutexes[id], 0);
+ ret = rt_mutex_lock_interruptible(&mutexes[id]);
td->event = atomic_add_return(1, &rttest_event);
td->mutexes[id] = ret ? 0 : 4;
return ret ? -EINTR : 0;
diff --git a/kernel/rtmutex.c b/kernel/rtmutex.c
index 823f782b1847..21f8990ac37c 100644
--- a/kernel/rtmutex.c
+++ b/kernel/rtmutex.c
@@ -1528,16 +1528,15 @@ rt_mutex_slowunlock(struct rt_mutex *lock)
*/
static inline int
rt_mutex_fastlock(struct rt_mutex *lock, int state,
- int detect_deadlock,
int (*slowfn)(struct rt_mutex *lock, int state,
struct hrtimer_sleeper *timeout,
int detect_deadlock))
{
- if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) {
+ if (likely(rt_mutex_cmpxchg(lock, NULL, current))) {
rt_mutex_deadlock_account_lock(lock, current);
return 0;
} else
- return slowfn(lock, state, NULL, detect_deadlock);
+ return slowfn(lock, state, NULL, 0);
}
static inline int
@@ -1584,7 +1583,7 @@ void __sched rt_mutex_lock(struct rt_mutex *lock)
{
might_sleep();
- rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, 0, rt_mutex_slowlock);
+ rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, rt_mutex_slowlock);
}
EXPORT_SYMBOL_GPL(rt_mutex_lock);
@@ -1592,41 +1591,46 @@ EXPORT_SYMBOL_GPL(rt_mutex_lock);
* rt_mutex_lock_interruptible - lock a rt_mutex interruptible
*
* @lock: the rt_mutex to be locked
- * @detect_deadlock: deadlock detection on/off
*
* Returns:
* 0 on success
* -EINTR when interrupted by a signal
- * -EDEADLK when the lock would deadlock (when deadlock detection is on)
*/
-int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock,
- int detect_deadlock)
+int __sched rt_mutex_lock_interruptible(struct rt_mutex *lock)
{
might_sleep();
- return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE,
- detect_deadlock, rt_mutex_slowlock);
+ return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE, rt_mutex_slowlock);
}
EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible);
+/*
+ * Futex variant with full deadlock detection.
+ */
+int rt_mutex_timed_futex_lock(struct rt_mutex *lock,
+ struct hrtimer_sleeper *timeout)
+{
+ might_sleep();
+
+ return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout, 1,
+ rt_mutex_slowlock);
+}
+
/**
* rt_mutex_lock_killable - lock a rt_mutex killable
*
* @lock: the rt_mutex to be locked
- * @detect_deadlock: deadlock detection on/off
*
* Returns:
* 0 on success
* -EINTR when interrupted by a signal
* -EDEADLK when the lock would deadlock (when deadlock detection is on)
*/
-int __sched rt_mutex_lock_killable(struct rt_mutex *lock,
- int detect_deadlock)
+int __sched rt_mutex_lock_killable(struct rt_mutex *lock)
{
might_sleep();
- return rt_mutex_fastlock(lock, TASK_KILLABLE,
- detect_deadlock, rt_mutex_slowlock);
+ return rt_mutex_fastlock(lock, TASK_KILLABLE, rt_mutex_slowlock);
}
EXPORT_SYMBOL_GPL(rt_mutex_lock_killable);
@@ -1637,22 +1641,19 @@ EXPORT_SYMBOL_GPL(rt_mutex_lock_killable);
*
* @lock: the rt_mutex to be locked
* @timeout: timeout structure or NULL (no timeout)
- * @detect_deadlock: deadlock detection on/off
*
* Returns:
* 0 on success
* -EINTR when interrupted by a signal
* -ETIMEDOUT when the timeout expired
- * -EDEADLK when the lock would deadlock (when deadlock detection is on)
*/
int
-rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout,
- int detect_deadlock)
+rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout)
{
might_sleep();
- return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,
- detect_deadlock, rt_mutex_slowlock);
+ return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout, 0,
+ rt_mutex_slowlock);
}
EXPORT_SYMBOL_GPL(rt_mutex_timed_lock);
@@ -1756,7 +1757,6 @@ void rt_mutex_proxy_unlock(struct rt_mutex *lock,
* @lock: the rt_mutex to take
* @waiter: the pre-initialized rt_mutex_waiter
* @task: the task to prepare
- * @detect_deadlock: perform deadlock detection (1) or not (0)
*
* Returns:
* 0 - task blocked on lock
@@ -1767,7 +1767,7 @@ void rt_mutex_proxy_unlock(struct rt_mutex *lock,
*/
int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
struct rt_mutex_waiter *waiter,
- struct task_struct *task, int detect_deadlock)
+ struct task_struct *task)
{
int ret;
@@ -1854,22 +1854,20 @@ struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock)
* rt_mutex_finish_proxy_lock() - Complete lock acquisition
* @lock: the rt_mutex we were woken on
* @to: the timeout, null if none. hrtimer should already have
- * been started.
+ * been started.
* @waiter: the pre-initialized rt_mutex_waiter
- * @detect_deadlock: perform deadlock detection (1) or not (0)
*
* Complete the lock acquisition started our behalf by another thread.
*
* Returns:
* 0 - success
- * <0 - error, one of -EINTR, -ETIMEDOUT, or -EDEADLK
+ * <0 - error, one of -EINTR, -ETIMEDOUT
*
* Special API call for PI-futex requeue support
*/
int rt_mutex_finish_proxy_lock(struct rt_mutex *lock,
struct hrtimer_sleeper *to,
- struct rt_mutex_waiter *waiter,
- int detect_deadlock)
+ struct rt_mutex_waiter *waiter)
{
int ret;
diff --git a/kernel/rtmutex_common.h b/kernel/rtmutex_common.h
index 6ec3dc1eab10..efde34108ebc 100644
--- a/kernel/rtmutex_common.h
+++ b/kernel/rtmutex_common.h
@@ -114,12 +114,11 @@ extern void rt_mutex_proxy_unlock(struct rt_mutex *lock,
struct task_struct *proxy_owner);
extern int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
struct rt_mutex_waiter *waiter,
- struct task_struct *task,
- int detect_deadlock);
+ struct task_struct *task);
extern int rt_mutex_finish_proxy_lock(struct rt_mutex *lock,
struct hrtimer_sleeper *to,
- struct rt_mutex_waiter *waiter,
- int detect_deadlock);
+ struct rt_mutex_waiter *waiter);
+extern int rt_mutex_timed_futex_lock(struct rt_mutex *l, struct hrtimer_sleeper *to);
#ifdef CONFIG_DEBUG_RT_MUTEXES
# include "rtmutex-debug.h"
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 10/31] rtmutex: Cleanup deadlock detector debug logic
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
` (8 preceding siblings ...)
2015-03-13 15:15 ` [PATCH RT 09/31] rtmutex: Confine deadlock logic to futex Steven Rostedt
@ 2015-03-13 15:15 ` Steven Rostedt
2015-03-13 15:15 ` [PATCH RT 11/31] rtmutex: Avoid pointless requeueing in the deadlock detection chain walk Steven Rostedt
` (19 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker, Peter Zijlstra, Lai Jiangshan
[-- Attachment #1: 0010-rtmutex-Cleanup-deadlock-detector-debug-logic.patch --]
[-- Type: text/plain, Size: 12023 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: Thomas Gleixner <tglx@linutronix.de>
upstream commit: 8930ed80f970a90a795239e7415c9b0e6f964649
The conditions under which deadlock detection is conducted are unclear
and undocumented.
Add constants instead of using 0/1 and provide a selection function
which hides the additional debug dependency from the calling code.
Add comments where needed.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Reviewed-by: Steven Rostedt <rostedt@goodmis.org>
Cc: Lai Jiangshan <laijs@cn.fujitsu.com>
Link: http://lkml.kernel.org/r/20140522031949.947264874@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Conflicts:
kernel/rtmutex.c
---
kernel/rtmutex-debug.c | 5 ++--
kernel/rtmutex-debug.h | 7 +++--
kernel/rtmutex.c | 78 ++++++++++++++++++++++++++++++++++---------------
kernel/rtmutex.h | 7 ++++-
kernel/rtmutex_common.h | 15 ++++++++++
5 files changed, 82 insertions(+), 30 deletions(-)
diff --git a/kernel/rtmutex-debug.c b/kernel/rtmutex-debug.c
index 13b243a323fa..2035a46ca489 100644
--- a/kernel/rtmutex-debug.c
+++ b/kernel/rtmutex-debug.c
@@ -66,12 +66,13 @@ void rt_mutex_debug_task_free(struct task_struct *task)
* the deadlock. We print when we return. act_waiter can be NULL in
* case of a remove waiter operation.
*/
-void debug_rt_mutex_deadlock(int detect, struct rt_mutex_waiter *act_waiter,
+void debug_rt_mutex_deadlock(enum rtmutex_chainwalk chwalk,
+ struct rt_mutex_waiter *act_waiter,
struct rt_mutex *lock)
{
struct task_struct *task;
- if (!debug_locks || detect || !act_waiter)
+ if (!debug_locks || chwalk == RT_MUTEX_FULL_CHAINWALK || !act_waiter)
return;
task = rt_mutex_owner(act_waiter->lock);
diff --git a/kernel/rtmutex-debug.h b/kernel/rtmutex-debug.h
index ab29b6a22669..d0519c3432b6 100644
--- a/kernel/rtmutex-debug.h
+++ b/kernel/rtmutex-debug.h
@@ -20,14 +20,15 @@ extern void debug_rt_mutex_unlock(struct rt_mutex *lock);
extern void debug_rt_mutex_proxy_lock(struct rt_mutex *lock,
struct task_struct *powner);
extern void debug_rt_mutex_proxy_unlock(struct rt_mutex *lock);
-extern void debug_rt_mutex_deadlock(int detect, struct rt_mutex_waiter *waiter,
+extern void debug_rt_mutex_deadlock(enum rtmutex_chainwalk chwalk,
+ struct rt_mutex_waiter *waiter,
struct rt_mutex *lock);
extern void debug_rt_mutex_print_deadlock(struct rt_mutex_waiter *waiter);
# define debug_rt_mutex_reset_waiter(w) \
do { (w)->deadlock_lock = NULL; } while (0)
-static inline int debug_rt_mutex_detect_deadlock(struct rt_mutex_waiter *waiter,
- int detect)
+static inline bool debug_rt_mutex_detect_deadlock(struct rt_mutex_waiter *waiter,
+ enum rtmutex_chainwalk walk)
{
return (waiter != NULL);
}
diff --git a/kernel/rtmutex.c b/kernel/rtmutex.c
index 21f8990ac37c..31555c11c58a 100644
--- a/kernel/rtmutex.c
+++ b/kernel/rtmutex.c
@@ -253,6 +253,32 @@ static void rt_mutex_wake_waiter(struct rt_mutex_waiter *waiter)
}
/*
+ * Deadlock detection is conditional:
+ *
+ * If CONFIG_DEBUG_RT_MUTEXES=n, deadlock detection is only conducted
+ * if the detect argument is == RT_MUTEX_FULL_CHAINWALK.
+ *
+ * If CONFIG_DEBUG_RT_MUTEXES=y, deadlock detection is always
+ * conducted independent of the detect argument.
+ *
+ * If the waiter argument is NULL this indicates the deboost path and
+ * deadlock detection is disabled independent of the detect argument
+ * and the config settings.
+ */
+static bool rt_mutex_cond_detect_deadlock(struct rt_mutex_waiter *waiter,
+ enum rtmutex_chainwalk chwalk)
+{
+ /*
+ * This is just a wrapper function for the following call,
+ * because debug_rt_mutex_detect_deadlock() smells like a magic
+ * debug feature and I wanted to keep the cond function in the
+ * main source file along with the comments instead of having
+ * two of the same in the headers.
+ */
+ return debug_rt_mutex_detect_deadlock(waiter, chwalk);
+}
+
+/*
* Max number of times we'll walk the boosting chain:
*/
int max_lock_depth = 1024;
@@ -312,7 +338,7 @@ static inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p)
* goto again;
*/
static int rt_mutex_adjust_prio_chain(struct task_struct *task,
- int deadlock_detect,
+ enum rtmutex_chainwalk chwalk,
struct rt_mutex *orig_lock,
struct rt_mutex *next_lock,
struct rt_mutex_waiter *orig_waiter,
@@ -320,12 +346,12 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
{
struct rt_mutex_waiter *waiter, *top_waiter = orig_waiter;
struct rt_mutex_waiter *prerequeue_top_waiter;
- int detect_deadlock, ret = 0, depth = 0;
+ int ret = 0, depth = 0;
struct rt_mutex *lock;
+ bool detect_deadlock;
unsigned long flags;
- detect_deadlock = debug_rt_mutex_detect_deadlock(orig_waiter,
- deadlock_detect);
+ detect_deadlock = rt_mutex_cond_detect_deadlock(orig_waiter, chwalk);
/*
* The (de)boosting is a step by step approach with a lot of
@@ -451,7 +477,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
* walk, we detected a deadlock.
*/
if (lock == orig_lock || rt_mutex_owner(lock) == top_task) {
- debug_rt_mutex_deadlock(deadlock_detect, orig_waiter, lock);
+ debug_rt_mutex_deadlock(chwalk, orig_waiter, lock);
raw_spin_unlock(&lock->wait_lock);
ret = -EDEADLK;
goto out_unlock_pi;
@@ -747,7 +773,7 @@ try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task,
static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
struct rt_mutex_waiter *waiter,
struct task_struct *task,
- int detect_deadlock)
+ enum rtmutex_chainwalk chwalk)
{
struct task_struct *owner = rt_mutex_owner(lock);
struct rt_mutex_waiter *top_waiter = waiter;
@@ -811,7 +837,7 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
__rt_mutex_adjust_prio(owner);
if (rt_mutex_real_waiter(owner->pi_blocked_on))
chain_walk = 1;
- } else if (debug_rt_mutex_detect_deadlock(waiter, detect_deadlock)) {
+ } else if (rt_mutex_cond_detect_deadlock(waiter, chwalk)) {
chain_walk = 1;
}
@@ -836,7 +862,7 @@ static int task_blocks_on_rt_mutex(struct rt_mutex *lock,
raw_spin_unlock(&lock->wait_lock);
- res = rt_mutex_adjust_prio_chain(owner, detect_deadlock, lock,
+ res = rt_mutex_adjust_prio_chain(owner, chwalk, lock,
next_lock, waiter, task);
raw_spin_lock(&lock->wait_lock);
@@ -944,7 +970,8 @@ static void remove_waiter(struct rt_mutex *lock,
raw_spin_unlock(&lock->wait_lock);
- rt_mutex_adjust_prio_chain(owner, 0, lock, next_lock, NULL, current);
+ rt_mutex_adjust_prio_chain(owner, RT_MUTEX_MIN_CHAINWALK, lock,
+ next_lock, NULL, current);
raw_spin_lock(&lock->wait_lock);
}
@@ -973,8 +1000,8 @@ void rt_mutex_adjust_pi(struct task_struct *task)
/* gets dropped in rt_mutex_adjust_prio_chain()! */
get_task_struct(task);
raw_spin_unlock_irqrestore(&task->pi_lock, flags);
-
- rt_mutex_adjust_prio_chain(task, 0, NULL, next_lock, NULL, task);
+ rt_mutex_adjust_prio_chain(task, RT_MUTEX_MIN_CHAINWALK, NULL,
+ next_lock, NULL, task);
}
#ifdef CONFIG_PREEMPT_RT_FULL
@@ -1367,7 +1394,7 @@ static void rt_mutex_handle_deadlock(int res, int detect_deadlock,
static int __sched
rt_mutex_slowlock(struct rt_mutex *lock, int state,
struct hrtimer_sleeper *timeout,
- int detect_deadlock)
+ enum rtmutex_chainwalk chwalk)
{
struct rt_mutex_waiter waiter;
int ret = 0;
@@ -1392,7 +1419,7 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state,
timeout->task = NULL;
}
- ret = task_blocks_on_rt_mutex(lock, &waiter, current, detect_deadlock);
+ ret = task_blocks_on_rt_mutex(lock, &waiter, current, chwalk);
if (likely(!ret))
ret = __rt_mutex_slowlock(lock, state, timeout, &waiter);
@@ -1401,7 +1428,7 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state,
if (unlikely(ret)) {
remove_waiter(lock, &waiter);
- rt_mutex_handle_deadlock(ret, detect_deadlock, &waiter);
+ rt_mutex_handle_deadlock(ret, chwalk, &waiter);
}
/*
@@ -1530,27 +1557,29 @@ static inline int
rt_mutex_fastlock(struct rt_mutex *lock, int state,
int (*slowfn)(struct rt_mutex *lock, int state,
struct hrtimer_sleeper *timeout,
- int detect_deadlock))
+ enum rtmutex_chainwalk chwalk))
{
if (likely(rt_mutex_cmpxchg(lock, NULL, current))) {
rt_mutex_deadlock_account_lock(lock, current);
return 0;
} else
- return slowfn(lock, state, NULL, 0);
+ return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK);
}
static inline int
rt_mutex_timed_fastlock(struct rt_mutex *lock, int state,
- struct hrtimer_sleeper *timeout, int detect_deadlock,
+ struct hrtimer_sleeper *timeout,
+ enum rtmutex_chainwalk chwalk,
int (*slowfn)(struct rt_mutex *lock, int state,
struct hrtimer_sleeper *timeout,
int detect_deadlock))
{
- if (!detect_deadlock && likely(rt_mutex_cmpxchg(lock, NULL, current))) {
+ if (chwalk == RT_MUTEX_MIN_CHAINWALK &&
+ likely(rt_mutex_cmpxchg(lock, NULL, current))) {
rt_mutex_deadlock_account_lock(lock, current);
return 0;
} else
- return slowfn(lock, state, timeout, detect_deadlock);
+ return slowfn(lock, state, timeout, chwalk);
}
static inline int
@@ -1612,8 +1641,8 @@ int rt_mutex_timed_futex_lock(struct rt_mutex *lock,
{
might_sleep();
- return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout, 1,
- rt_mutex_slowlock);
+ return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,
+ RT_MUTEX_FULL_CHAINWALK, rt_mutex_slowlock);
}
/**
@@ -1652,8 +1681,8 @@ rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout)
{
might_sleep();
- return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout, 0,
- rt_mutex_slowlock);
+ return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout,
+ RT_MUTEX_MIN_CHAINWALK, rt_mutex_slowlock);
}
EXPORT_SYMBOL_GPL(rt_mutex_timed_lock);
@@ -1808,7 +1837,8 @@ int rt_mutex_start_proxy_lock(struct rt_mutex *lock,
#endif
/* We enforce deadlock detection for futexes */
- ret = task_blocks_on_rt_mutex(lock, waiter, task, 1);
+ ret = task_blocks_on_rt_mutex(lock, waiter, task,
+ RT_MUTEX_FULL_CHAINWALK);
if (ret && !rt_mutex_owner(lock)) {
/*
diff --git a/kernel/rtmutex.h b/kernel/rtmutex.h
index f6a1f3c133b1..c4060584c407 100644
--- a/kernel/rtmutex.h
+++ b/kernel/rtmutex.h
@@ -22,10 +22,15 @@
#define debug_rt_mutex_init(m, n) do { } while (0)
#define debug_rt_mutex_deadlock(d, a ,l) do { } while (0)
#define debug_rt_mutex_print_deadlock(w) do { } while (0)
-#define debug_rt_mutex_detect_deadlock(w,d) (d)
#define debug_rt_mutex_reset_waiter(w) do { } while (0)
static inline void rt_mutex_print_deadlock(struct rt_mutex_waiter *w)
{
WARN(1, "rtmutex deadlock detected\n");
}
+
+static inline bool debug_rt_mutex_detect_deadlock(struct rt_mutex_waiter *w,
+ enum rtmutex_chainwalk walk)
+{
+ return walk == RT_MUTEX_FULL_CHAINWALK;
+}
diff --git a/kernel/rtmutex_common.h b/kernel/rtmutex_common.h
index efde34108ebc..213c7bf3c0d9 100644
--- a/kernel/rtmutex_common.h
+++ b/kernel/rtmutex_common.h
@@ -102,6 +102,21 @@ static inline struct task_struct *rt_mutex_owner(struct rt_mutex *lock)
}
/*
+ * Constants for rt mutex functions which have a selectable deadlock
+ * detection.
+ *
+ * RT_MUTEX_MIN_CHAINWALK: Stops the lock chain walk when there are
+ * no further PI adjustments to be made.
+ *
+ * RT_MUTEX_FULL_CHAINWALK: Invoke deadlock detection with a full
+ * walk of the lock chain.
+ */
+enum rtmutex_chainwalk {
+ RT_MUTEX_MIN_CHAINWALK,
+ RT_MUTEX_FULL_CHAINWALK,
+};
+
+/*
* PI-futex support (proxy locking functions, etc.):
*/
#define PI_WAKEUP_INPROGRESS ((struct rt_mutex_waiter *) 1)
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 11/31] rtmutex: Avoid pointless requeueing in the deadlock detection chain walk
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
` (9 preceding siblings ...)
2015-03-13 15:15 ` [PATCH RT 10/31] rtmutex: Cleanup deadlock detector debug logic Steven Rostedt
@ 2015-03-13 15:15 ` Steven Rostedt
2015-03-13 15:15 ` Steven Rostedt
` (18 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker, Peter Zijlstra, Lai Jiangshan
[-- Attachment #1: 0011-rtmutex-Avoid-pointless-requeueing-in-the-deadlock-d.patch --]
[-- Type: text/plain, Size: 4358 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: Thomas Gleixner <tglx@linutronix.de>
upstream commit: 67792e2cabadbadd1a93f6790fa7bcbd47eca7c3
In case the dead lock detector is enabled we follow the lock chain to
the end in rt_mutex_adjust_prio_chain, even if we could stop earlier
due to the priority/waiter constellation.
But once we are no longer the top priority waiter in a certain step
or the task holding the lock has already the same priority then there
is no point in dequeing and enqueing along the lock chain as there is
no change at all.
So stop the queueing at this point.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Reviewed-by: Steven Rostedt <rostedt@goodmis.org>
Cc: Lai Jiangshan <laijs@cn.fujitsu.com>
Link: http://lkml.kernel.org/r/20140522031950.280830190@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
kernel/rtmutex.c | 77 ++++++++++++++++++++++++++++++++++++++++++++++++++------
1 file changed, 70 insertions(+), 7 deletions(-)
diff --git a/kernel/rtmutex.c b/kernel/rtmutex.c
index 31555c11c58a..b7b1de2acf0b 100644
--- a/kernel/rtmutex.c
+++ b/kernel/rtmutex.c
@@ -350,6 +350,7 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
struct rt_mutex *lock;
bool detect_deadlock;
unsigned long flags;
+ bool requeue = true;
detect_deadlock = rt_mutex_cond_detect_deadlock(orig_waiter, chwalk);
@@ -439,18 +440,31 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
goto out_unlock_pi;
/*
* If deadlock detection is off, we stop here if we
- * are not the top pi waiter of the task.
+ * are not the top pi waiter of the task. If deadlock
+ * detection is enabled we continue, but stop the
+ * requeueing in the chain walk.
*/
- if (!detect_deadlock && top_waiter != task_top_pi_waiter(task))
- goto out_unlock_pi;
+ if (top_waiter != task_top_pi_waiter(task)) {
+ if (!detect_deadlock)
+ goto out_unlock_pi;
+ else
+ requeue = false;
+ }
}
/*
- * When deadlock detection is off then we check, if further
- * priority adjustment is necessary.
+ * If the waiter priority is the same as the task priority
+ * then there is no further priority adjustment necessary. If
+ * deadlock detection is off, we stop the chain walk. If its
+ * enabled we continue, but stop the requeueing in the chain
+ * walk.
*/
- if (!detect_deadlock && waiter->list_entry.prio == task->prio)
- goto out_unlock_pi;
+ if (waiter->list_entry.prio == task->prio) {
+ if (!detect_deadlock)
+ goto out_unlock_pi;
+ else
+ requeue = false;
+ }
/*
* [4] Get the next lock
@@ -484,6 +498,55 @@ static int rt_mutex_adjust_prio_chain(struct task_struct *task,
}
/*
+ * If we just follow the lock chain for deadlock detection, no
+ * need to do all the requeue operations. To avoid a truckload
+ * of conditionals around the various places below, just do the
+ * minimum chain walk checks.
+ */
+ if (!requeue) {
+ /*
+ * No requeue[7] here. Just release @task [8]
+ */
+ raw_spin_unlock_irqrestore(&task->pi_lock, flags);
+ put_task_struct(task);
+
+ /*
+ * [9] check_exit_conditions_3 protected by lock->wait_lock.
+ * If there is no owner of the lock, end of chain.
+ */
+ if (!rt_mutex_owner(lock)) {
+ raw_spin_unlock(&lock->wait_lock);
+ return 0;
+ }
+
+ /* [10] Grab the next task, i.e. owner of @lock */
+ task = rt_mutex_owner(lock);
+ get_task_struct(task);
+ raw_spin_lock_irqsave(&task->pi_lock, flags);
+
+ /*
+ * No requeue [11] here. We just do deadlock detection.
+ *
+ * [12] Store whether owner is blocked
+ * itself. Decision is made after dropping the locks
+ */
+ next_lock = task_blocked_on_lock(task);
+ /*
+ * Get the top waiter for the next iteration
+ */
+ top_waiter = rt_mutex_top_waiter(lock);
+
+ /* [13] Drop locks */
+ raw_spin_unlock_irqrestore(&task->pi_lock, flags);
+ raw_spin_unlock(&lock->wait_lock);
+
+ /* If owner is not blocked, end of chain. */
+ if (!next_lock)
+ goto out_put_task;
+ goto again;
+ }
+
+ /*
* Store the current top waiter before doing the requeue
* operation on @lock. We need it for the boost/deboost
* decision below.
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 12/31] futex: Make unlock_pi more robust
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
@ 2015-03-13 15:15 ` Steven Rostedt
2015-03-13 15:14 ` [PATCH RT 02/31] create-rt-enqueue Steven Rostedt
` (28 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker, Peter Zijlstra, Darren Hart,
Davidlohr Bueso, Kees Cook, wad
[-- Attachment #1: 0012-futex-Make-unlock_pi-more-robust.patch --]
[-- Type: text/plain, Size: 4934 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: Thomas Gleixner <tglx@linutronix.de>
upstream commit: ccf9e6a80d9e1b9df69c98e6b9745cf49869ee15
The kernel tries to atomically unlock the futex without checking
whether there is kernel state associated to the futex.
So if user space manipulated the user space value, this will leave
kernel internal state around associated to the owner task.
For robustness sake, lookup first whether there are waiters on the
futex. If there are waiters, wake the top priority waiter with all the
proper sanity checks applied.
If there are no waiters, do the atomic release. We do not have to
preserve the waiters bit in this case, because a potentially incoming
waiter is blocked on the hb->lock and will acquire the futex
atomically. We neither have to preserve the owner died bit. The caller
is the owner and it was supposed to cleanup the mess.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Darren Hart <darren@dvhart.com>
Cc: Davidlohr Bueso <davidlohr@hp.com>
Cc: Kees Cook <kees@outflux.net>
Cc: wad@chromium.org
Link: http://lkml.kernel.org/r/20140611204237.016987332@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
kernel/futex.c | 80 +++++++++++++++++++---------------------------------------
1 file changed, 26 insertions(+), 54 deletions(-)
diff --git a/kernel/futex.c b/kernel/futex.c
index 75f386961a8d..d9189e82f3a2 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -1039,22 +1039,6 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this)
return 0;
}
-static int unlock_futex_pi(u32 __user *uaddr, u32 uval)
-{
- u32 uninitialized_var(oldval);
-
- /*
- * There is no waiter, so we unlock the futex. The owner died
- * bit has not to be preserved here. We are the owner:
- */
- if (cmpxchg_futex_value_locked(&oldval, uaddr, uval, 0))
- return -EFAULT;
- if (oldval != uval)
- return -EAGAIN;
-
- return 0;
-}
-
/*
* Express the locking dependencies for lockdep:
*/
@@ -2248,11 +2232,10 @@ uaddr_faulted:
*/
static int futex_unlock_pi(u32 __user *uaddr, unsigned int flags)
{
- struct futex_hash_bucket *hb;
- struct futex_q *this, *next;
- struct plist_head *head;
+ u32 uninitialized_var(curval), uval, vpid = task_pid_vnr(current);
union futex_key key = FUTEX_KEY_INIT;
- u32 uval, vpid = task_pid_vnr(current);
+ struct futex_hash_bucket *hb;
+ struct futex_q *match;
int ret;
retry:
@@ -2265,61 +2248,50 @@ retry:
return -EPERM;
ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, VERIFY_WRITE);
- if (unlikely(ret != 0))
- goto out;
+ if (ret)
+ return ret;
hb = hash_futex(&key);
spin_lock(&hb->lock);
/*
- * To avoid races, try to do the TID -> 0 atomic transition
- * again. If it succeeds then we can return without waking
- * anyone else up. We only try this if neither the waiters nor
- * the owner died bit are set.
+ * Check waiters first. We do not trust user space values at
+ * all and we at least want to know if user space fiddled
+ * with the futex value instead of blindly unlocking.
*/
- if (!(uval & ~FUTEX_TID_MASK) &&
- cmpxchg_futex_value_locked(&uval, uaddr, vpid, 0))
- goto pi_faulted;
- /*
- * Rare case: we managed to release the lock atomically,
- * no need to wake anyone else up:
- */
- if (unlikely(uval == vpid))
- goto out_unlock;
-
- /*
- * Ok, other tasks may need to be woken up - check waiters
- * and do the wakeup if necessary:
- */
- head = &hb->chain;
-
- plist_for_each_entry_safe(this, next, head, list) {
- if (!match_futex (&this->key, &key))
- continue;
- ret = wake_futex_pi(uaddr, uval, this);
+ match = futex_top_waiter(hb, &key);
+ if (match) {
+ ret = wake_futex_pi(uaddr, uval, match);
/*
- * The atomic access to the futex value
- * generated a pagefault, so retry the
- * user-access and the wakeup:
+ * The atomic access to the futex value generated a
+ * pagefault, so retry the user-access and the wakeup:
*/
if (ret == -EFAULT)
goto pi_faulted;
goto out_unlock;
}
+
/*
- * No waiters - kernel unlocks the futex:
+ * We have no kernel internal state, i.e. no waiters in the
+ * kernel. Waiters which are about to queue themselves are stuck
+ * on hb->lock. So we can safely ignore them. We do neither
+ * preserve the WAITERS bit not the OWNER_DIED one. We are the
+ * owner.
*/
- ret = unlock_futex_pi(uaddr, uval);
- if (ret == -EFAULT)
+ if (cmpxchg_futex_value_locked(&curval, uaddr, uval, 0))
goto pi_faulted;
+ /*
+ * If uval has changed, let user space handle it.
+ */
+ ret = (curval == uval) ? 0 : -EAGAIN;
+
out_unlock:
spin_unlock(&hb->lock);
put_futex_key(&key);
-
-out:
return ret;
+
pi_faulted:
spin_unlock(&hb->lock);
put_futex_key(&key);
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 12/31] futex: Make unlock_pi more robust
@ 2015-03-13 15:15 ` Steven Rostedt
0 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker, Peter Zijlstra, Darren Hart,
Davidlohr Bueso, Kees Cook, wad
[-- Attachment #1: 0012-futex-Make-unlock_pi-more-robust.patch --]
[-- Type: text/plain, Size: 4834 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: Thomas Gleixner <tglx@linutronix.de>
upstream commit: ccf9e6a80d9e1b9df69c98e6b9745cf49869ee15
The kernel tries to atomically unlock the futex without checking
whether there is kernel state associated to the futex.
So if user space manipulated the user space value, this will leave
kernel internal state around associated to the owner task.
For robustness sake, lookup first whether there are waiters on the
futex. If there are waiters, wake the top priority waiter with all the
proper sanity checks applied.
If there are no waiters, do the atomic release. We do not have to
preserve the waiters bit in this case, because a potentially incoming
waiter is blocked on the hb->lock and will acquire the futex
atomically. We neither have to preserve the owner died bit. The caller
is the owner and it was supposed to cleanup the mess.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Darren Hart <darren@dvhart.com>
Cc: Davidlohr Bueso <davidlohr@hp.com>
Cc: Kees Cook <kees@outflux.net>
Cc: wad@chromium.org
Link: http://lkml.kernel.org/r/20140611204237.016987332@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
kernel/futex.c | 80 +++++++++++++++++++---------------------------------------
1 file changed, 26 insertions(+), 54 deletions(-)
diff --git a/kernel/futex.c b/kernel/futex.c
index 75f386961a8d..d9189e82f3a2 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -1039,22 +1039,6 @@ static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this)
return 0;
}
-static int unlock_futex_pi(u32 __user *uaddr, u32 uval)
-{
- u32 uninitialized_var(oldval);
-
- /*
- * There is no waiter, so we unlock the futex. The owner died
- * bit has not to be preserved here. We are the owner:
- */
- if (cmpxchg_futex_value_locked(&oldval, uaddr, uval, 0))
- return -EFAULT;
- if (oldval != uval)
- return -EAGAIN;
-
- return 0;
-}
-
/*
* Express the locking dependencies for lockdep:
*/
@@ -2248,11 +2232,10 @@ uaddr_faulted:
*/
static int futex_unlock_pi(u32 __user *uaddr, unsigned int flags)
{
- struct futex_hash_bucket *hb;
- struct futex_q *this, *next;
- struct plist_head *head;
+ u32 uninitialized_var(curval), uval, vpid = task_pid_vnr(current);
union futex_key key = FUTEX_KEY_INIT;
- u32 uval, vpid = task_pid_vnr(current);
+ struct futex_hash_bucket *hb;
+ struct futex_q *match;
int ret;
retry:
@@ -2265,61 +2248,50 @@ retry:
return -EPERM;
ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, VERIFY_WRITE);
- if (unlikely(ret != 0))
- goto out;
+ if (ret)
+ return ret;
hb = hash_futex(&key);
spin_lock(&hb->lock);
/*
- * To avoid races, try to do the TID -> 0 atomic transition
- * again. If it succeeds then we can return without waking
- * anyone else up. We only try this if neither the waiters nor
- * the owner died bit are set.
+ * Check waiters first. We do not trust user space values at
+ * all and we at least want to know if user space fiddled
+ * with the futex value instead of blindly unlocking.
*/
- if (!(uval & ~FUTEX_TID_MASK) &&
- cmpxchg_futex_value_locked(&uval, uaddr, vpid, 0))
- goto pi_faulted;
- /*
- * Rare case: we managed to release the lock atomically,
- * no need to wake anyone else up:
- */
- if (unlikely(uval == vpid))
- goto out_unlock;
-
- /*
- * Ok, other tasks may need to be woken up - check waiters
- * and do the wakeup if necessary:
- */
- head = &hb->chain;
-
- plist_for_each_entry_safe(this, next, head, list) {
- if (!match_futex (&this->key, &key))
- continue;
- ret = wake_futex_pi(uaddr, uval, this);
+ match = futex_top_waiter(hb, &key);
+ if (match) {
+ ret = wake_futex_pi(uaddr, uval, match);
/*
- * The atomic access to the futex value
- * generated a pagefault, so retry the
- * user-access and the wakeup:
+ * The atomic access to the futex value generated a
+ * pagefault, so retry the user-access and the wakeup:
*/
if (ret == -EFAULT)
goto pi_faulted;
goto out_unlock;
}
+
/*
- * No waiters - kernel unlocks the futex:
+ * We have no kernel internal state, i.e. no waiters in the
+ * kernel. Waiters which are about to queue themselves are stuck
+ * on hb->lock. So we can safely ignore them. We do neither
+ * preserve the WAITERS bit not the OWNER_DIED one. We are the
+ * owner.
*/
- ret = unlock_futex_pi(uaddr, uval);
- if (ret == -EFAULT)
+ if (cmpxchg_futex_value_locked(&curval, uaddr, uval, 0))
goto pi_faulted;
+ /*
+ * If uval has changed, let user space handle it.
+ */
+ ret = (curval == uval) ? 0 : -EAGAIN;
+
out_unlock:
spin_unlock(&hb->lock);
put_futex_key(&key);
^ permalink raw reply related [flat|nested] 33+ messages in thread
* [PATCH RT 13/31] futex: Use futex_top_waiter() in lookup_pi_state()
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
` (11 preceding siblings ...)
2015-03-13 15:15 ` Steven Rostedt
@ 2015-03-13 15:15 ` Steven Rostedt
2015-03-13 15:15 ` [PATCH RT 14/31] futex: Split out the waiter check from lookup_pi_state() Steven Rostedt
` (16 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker, Darren Hart, Peter Zijlstra,
Davidlohr Bueso, Kees Cook, wad
[-- Attachment #1: 0013-futex-Use-futex_top_waiter-in-lookup_pi_state.patch --]
[-- Type: text/plain, Size: 4764 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: Thomas Gleixner <tglx@linutronix.de>
upstream commit: bd1dbcc67cd2c1181e2c01daac51eabf1b964dd8
No point in open coding the same function again.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Darren Hart <darren@dvhart.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Davidlohr Bueso <davidlohr@hp.com>
Cc: Kees Cook <kees@outflux.net>
Cc: wad@chromium.org
Link: http://lkml.kernel.org/r/20140611204237.092947239@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
kernel/futex.c | 127 +++++++++++++++++++++++++++------------------------------
1 file changed, 61 insertions(+), 66 deletions(-)
diff --git a/kernel/futex.c b/kernel/futex.c
index d9189e82f3a2..816b7f53d3f0 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -647,90 +647,85 @@ static int
lookup_pi_state(u32 uval, struct futex_hash_bucket *hb,
union futex_key *key, struct futex_pi_state **ps)
{
+ struct futex_q *match = futex_top_waiter(hb, key);
struct futex_pi_state *pi_state = NULL;
- struct futex_q *this, *next;
- struct plist_head *head;
struct task_struct *p;
pid_t pid = uval & FUTEX_TID_MASK;
- head = &hb->chain;
-
- plist_for_each_entry_safe(this, next, head, list) {
- if (match_futex(&this->key, key)) {
- /*
- * Sanity check the waiter before increasing
- * the refcount and attaching to it.
- */
- pi_state = this->pi_state;
- /*
- * Userspace might have messed up non-PI and
- * PI futexes [3]
- */
- if (unlikely(!pi_state))
- return -EINVAL;
+ if (match) {
+ /*
+ * Sanity check the waiter before increasing the
+ * refcount and attaching to it.
+ */
+ pi_state = match->pi_state;
+ /*
+ * Userspace might have messed up non-PI and PI
+ * futexes [3]
+ */
+ if (unlikely(!pi_state))
+ return -EINVAL;
- WARN_ON(!atomic_read(&pi_state->refcount));
+ WARN_ON(!atomic_read(&pi_state->refcount));
+ /*
+ * Handle the owner died case:
+ */
+ if (uval & FUTEX_OWNER_DIED) {
/*
- * Handle the owner died case:
+ * exit_pi_state_list sets owner to NULL and
+ * wakes the topmost waiter. The task which
+ * acquires the pi_state->rt_mutex will fixup
+ * owner.
*/
- if (uval & FUTEX_OWNER_DIED) {
+ if (!pi_state->owner) {
/*
- * exit_pi_state_list sets owner to NULL and
- * wakes the topmost waiter. The task which
- * acquires the pi_state->rt_mutex will fixup
- * owner.
+ * No pi state owner, but the user
+ * space TID is not 0. Inconsistent
+ * state. [5]
*/
- if (!pi_state->owner) {
- /*
- * No pi state owner, but the user
- * space TID is not 0. Inconsistent
- * state. [5]
- */
- if (pid)
- return -EINVAL;
- /*
- * Take a ref on the state and
- * return. [4]
- */
- goto out_state;
- }
-
- /*
- * If TID is 0, then either the dying owner
- * has not yet executed exit_pi_state_list()
- * or some waiter acquired the rtmutex in the
- * pi state, but did not yet fixup the TID in
- * user space.
- *
- * Take a ref on the state and return. [6]
- */
- if (!pid)
- goto out_state;
- } else {
+ if (pid)
+ return -EINVAL;
/*
- * If the owner died bit is not set,
- * then the pi_state must have an
- * owner. [7]
+ * Take a ref on the state and
+ * return. [4]
*/
- if (!pi_state->owner)
- return -EINVAL;
+ goto out_state;
}
/*
- * Bail out if user space manipulated the
- * futex value. If pi state exists then the
- * owner TID must be the same as the user
- * space TID. [9/10]
+ * If TID is 0, then either the dying owner
+ * has not yet executed exit_pi_state_list()
+ * or some waiter acquired the rtmutex in the
+ * pi state, but did not yet fixup the TID in
+ * user space.
+ *
+ * Take a ref on the state and return. [6]
+ */
+ if (!pid)
+ goto out_state;
+ } else {
+ /*
+ * If the owner died bit is not set,
+ * then the pi_state must have an
+ * owner. [7]
*/
- if (pid != task_pid_vnr(pi_state->owner))
+ if (!pi_state->owner)
return -EINVAL;
-
- out_state:
- atomic_inc(&pi_state->refcount);
- *ps = pi_state;
- return 0;
}
+
+ /*
+ * Bail out if user space manipulated the
+ * futex value. If pi state exists then the
+ * owner TID must be the same as the user
+ * space TID. [9/10]
+ */
+ if (pid != task_pid_vnr(pi_state->owner))
+ return -EINVAL;
+
+ out_state:
+ atomic_inc(&pi_state->refcount);
+ *ps = pi_state;
+ return 0;
}
/*
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 14/31] futex: Split out the waiter check from lookup_pi_state()
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
` (12 preceding siblings ...)
2015-03-13 15:15 ` [PATCH RT 13/31] futex: Use futex_top_waiter() in lookup_pi_state() Steven Rostedt
@ 2015-03-13 15:15 ` Steven Rostedt
2015-03-13 15:15 ` [PATCH RT 15/31] futex: Split out the first waiter attachment " Steven Rostedt
` (15 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker, Darren Hart, Peter Zijlstra,
Davidlohr Bueso, Kees Cook, wad
[-- Attachment #1: 0014-futex-Split-out-the-waiter-check-from-lookup_pi_stat.patch --]
[-- Type: text/plain, Size: 5471 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: Thomas Gleixner <tglx@linutronix.de>
upstream commit: e60cbc5ceaa518d630ab8f35a7d05cee1c752648
We want to be a bit more clever in futex_lock_pi_atomic() and separate
the possible states. Split out the waiter verification into a separate
function. No functional change.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Darren Hart <darren@dvhart.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Davidlohr Bueso <davidlohr@hp.com>
Cc: Kees Cook <kees@outflux.net>
Cc: wad@chromium.org
Link: http://lkml.kernel.org/r/20140611204237.180458410@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
kernel/futex.c | 138 +++++++++++++++++++++++++++++----------------------------
1 file changed, 71 insertions(+), 67 deletions(-)
diff --git a/kernel/futex.c b/kernel/futex.c
index 816b7f53d3f0..8faa72097ebc 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -643,92 +643,96 @@ void exit_pi_state_list(struct task_struct *curr)
* [10] There is no transient state which leaves owner and user space
* TID out of sync.
*/
-static int
-lookup_pi_state(u32 uval, struct futex_hash_bucket *hb,
- union futex_key *key, struct futex_pi_state **ps)
+
+/*
+ * Validate that the existing waiter has a pi_state and sanity check
+ * the pi_state against the user space value. If correct, attach to
+ * it.
+ */
+static int attach_to_pi_state(u32 uval, struct futex_pi_state *pi_state,
+ struct futex_pi_state **ps)
{
- struct futex_q *match = futex_top_waiter(hb, key);
- struct futex_pi_state *pi_state = NULL;
- struct task_struct *p;
pid_t pid = uval & FUTEX_TID_MASK;
- if (match) {
- /*
- * Sanity check the waiter before increasing the
- * refcount and attaching to it.
- */
- pi_state = match->pi_state;
- /*
- * Userspace might have messed up non-PI and PI
- * futexes [3]
- */
- if (unlikely(!pi_state))
- return -EINVAL;
+ /*
+ * Userspace might have messed up non-PI and PI futexes [3]
+ */
+ if (unlikely(!pi_state))
+ return -EINVAL;
- WARN_ON(!atomic_read(&pi_state->refcount));
+ WARN_ON(!atomic_read(&pi_state->refcount));
+ /*
+ * Handle the owner died case:
+ */
+ if (uval & FUTEX_OWNER_DIED) {
/*
- * Handle the owner died case:
+ * exit_pi_state_list sets owner to NULL and wakes the
+ * topmost waiter. The task which acquires the
+ * pi_state->rt_mutex will fixup owner.
*/
- if (uval & FUTEX_OWNER_DIED) {
- /*
- * exit_pi_state_list sets owner to NULL and
- * wakes the topmost waiter. The task which
- * acquires the pi_state->rt_mutex will fixup
- * owner.
- */
- if (!pi_state->owner) {
- /*
- * No pi state owner, but the user
- * space TID is not 0. Inconsistent
- * state. [5]
- */
- if (pid)
- return -EINVAL;
- /*
- * Take a ref on the state and
- * return. [4]
- */
- goto out_state;
- }
-
+ if (!pi_state->owner) {
/*
- * If TID is 0, then either the dying owner
- * has not yet executed exit_pi_state_list()
- * or some waiter acquired the rtmutex in the
- * pi state, but did not yet fixup the TID in
- * user space.
- *
- * Take a ref on the state and return. [6]
+ * No pi state owner, but the user space TID
+ * is not 0. Inconsistent state. [5]
*/
- if (!pid)
- goto out_state;
- } else {
+ if (pid)
+ return -EINVAL;
/*
- * If the owner died bit is not set,
- * then the pi_state must have an
- * owner. [7]
+ * Take a ref on the state and return success. [4]
*/
- if (!pi_state->owner)
- return -EINVAL;
+ goto out_state;
}
/*
- * Bail out if user space manipulated the
- * futex value. If pi state exists then the
- * owner TID must be the same as the user
- * space TID. [9/10]
+ * If TID is 0, then either the dying owner has not
+ * yet executed exit_pi_state_list() or some waiter
+ * acquired the rtmutex in the pi state, but did not
+ * yet fixup the TID in user space.
+ *
+ * Take a ref on the state and return success. [6]
+ */
+ if (!pid)
+ goto out_state;
+ } else {
+ /*
+ * If the owner died bit is not set, then the pi_state
+ * must have an owner. [7]
*/
- if (pid != task_pid_vnr(pi_state->owner))
+ if (!pi_state->owner)
return -EINVAL;
-
- out_state:
- atomic_inc(&pi_state->refcount);
- *ps = pi_state;
- return 0;
}
/*
+ * Bail out if user space manipulated the futex value. If pi
+ * state exists then the owner TID must be the same as the
+ * user space TID. [9/10]
+ */
+ if (pid != task_pid_vnr(pi_state->owner))
+ return -EINVAL;
+out_state:
+ atomic_inc(&pi_state->refcount);
+ *ps = pi_state;
+ return 0;
+}
+
+static int
+lookup_pi_state(u32 uval, struct futex_hash_bucket *hb,
+ union futex_key *key, struct futex_pi_state **ps)
+{
+ struct futex_q *match = futex_top_waiter(hb, key);
+ struct futex_pi_state *pi_state = NULL;
+ struct task_struct *p;
+ pid_t pid = uval & FUTEX_TID_MASK;
+
+ /*
+ * If there is a waiter on that futex, validate it and
+ * attach to the pi_state when the validation succeeds.
+ */
+ if (match)
+ return attach_to_pi_state(uval, match->pi_state, ps);
+
+ /*
* We are the first waiter - try to look up the real owner and attach
* the new pi_state to it, but bail out when TID = 0 [1]
*/
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 15/31] futex: Split out the first waiter attachment from lookup_pi_state()
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
` (13 preceding siblings ...)
2015-03-13 15:15 ` [PATCH RT 14/31] futex: Split out the waiter check from lookup_pi_state() Steven Rostedt
@ 2015-03-13 15:15 ` Steven Rostedt
2015-03-13 15:15 ` [PATCH RT 16/31] futex: Simplify futex_lock_pi_atomic() and make it more robust Steven Rostedt
` (14 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker, Darren Hart, Peter Zijlstra,
Davidlohr Bueso, Kees Cook, wad
[-- Attachment #1: 0015-futex-Split-out-the-first-waiter-attachment-from-loo.patch --]
[-- Type: text/plain, Size: 3035 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: Thomas Gleixner <tglx@linutronix.de>
upstream commit: 04e1b2e52b17195c9a1daa5935c55a4c8716095c
We want to be a bit more clever in futex_lock_pi_atomic() and separate
the possible states. Split out the code which attaches the first
waiter to the owner into a separate function. No functional change.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Darren Hart <darren@dvhart.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Davidlohr Bueso <davidlohr@hp.com>
Cc: Kees Cook <kees@outflux.net>
Cc: wad@chromium.org
Link: http://lkml.kernel.org/r/20140611204237.271300614@linutronix.de
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
kernel/futex.c | 42 ++++++++++++++++++++++++++++--------------
1 file changed, 28 insertions(+), 14 deletions(-)
diff --git a/kernel/futex.c b/kernel/futex.c
index 8faa72097ebc..0b83f9ff2615 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -716,21 +716,16 @@ out_state:
return 0;
}
-static int
-lookup_pi_state(u32 uval, struct futex_hash_bucket *hb,
- union futex_key *key, struct futex_pi_state **ps)
+/*
+ * Lookup the task for the TID provided from user space and attach to
+ * it after doing proper sanity checks.
+ */
+static int attach_to_pi_owner(u32 uval, union futex_key *key,
+ struct futex_pi_state **ps)
{
- struct futex_q *match = futex_top_waiter(hb, key);
- struct futex_pi_state *pi_state = NULL;
- struct task_struct *p;
pid_t pid = uval & FUTEX_TID_MASK;
-
- /*
- * If there is a waiter on that futex, validate it and
- * attach to the pi_state when the validation succeeds.
- */
- if (match)
- return attach_to_pi_state(uval, match->pi_state, ps);
+ struct futex_pi_state *pi_state;
+ struct task_struct *p;
/*
* We are the first waiter - try to look up the real owner and attach
@@ -773,7 +768,7 @@ lookup_pi_state(u32 uval, struct futex_hash_bucket *hb,
pi_state = alloc_pi_state();
/*
- * Initialize the pi_mutex in locked state and make 'p'
+ * Initialize the pi_mutex in locked state and make @p
* the owner of it:
*/
rt_mutex_init_proxy_locked(&pi_state->pi_mutex, p);
@@ -793,6 +788,25 @@ lookup_pi_state(u32 uval, struct futex_hash_bucket *hb,
return 0;
}
+static int lookup_pi_state(u32 uval, struct futex_hash_bucket *hb,
+ union futex_key *key, struct futex_pi_state **ps)
+{
+ struct futex_q *match = futex_top_waiter(hb, key);
+
+ /*
+ * If there is a waiter on that futex, validate it and
+ * attach to the pi_state when the validation succeeds.
+ */
+ if (match)
+ return attach_to_pi_state(uval, match->pi_state, ps);
+
+ /*
+ * We are the first waiter - try to look up the owner based on
+ * @uval and attach to it.
+ */
+ return attach_to_pi_owner(uval, key, ps);
+}
+
/**
* futex_lock_pi_atomic() - Atomic work required to acquire a pi aware futex
* @uaddr: the pi futex user address
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 16/31] futex: Simplify futex_lock_pi_atomic() and make it more robust
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
` (14 preceding siblings ...)
2015-03-13 15:15 ` [PATCH RT 15/31] futex: Split out the first waiter attachment " Steven Rostedt
@ 2015-03-13 15:15 ` Steven Rostedt
2015-03-13 15:15 ` [PATCH RT 17/31] rt-mutex: avoid a NULL pointer dereference on deadlock Steven Rostedt
` (13 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker, Peter Zijlstra, Davidlohr Bueso,
Kees Cook, wad, Darren Hart
[-- Attachment #1: 0016-futex-Simplify-futex_lock_pi_atomic-and-make-it-more.patch --]
[-- Type: text/plain, Size: 7046 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: Thomas Gleixner <tglx@linutronix.de>
upstream commit: af54d6a1c3ad474bbc9893c9905022646be6092c
futex_lock_pi_atomic() is a maze of retry hoops and loops.
Reduce it to simple and understandable states:
First step is to lookup existing waiters (state) in the kernel.
If there is an existing waiter, validate it and attach to it.
If there is no existing waiter, check the user space value
If the TID encoded in the user space value is 0, take over the futex
preserving the owner died bit.
If the TID encoded in the user space value is != 0, lookup the owner
task, validate it and attach to it.
Reduces text size by 128 bytes on x8664.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Davidlohr Bueso <davidlohr@hp.com>
Cc: Kees Cook <kees@outflux.net>
Cc: wad@chromium.org
Cc: Darren Hart <darren@dvhart.com>
Link: http://lkml.kernel.org/r/alpine.DEB.2.10.1406131137020.5170@nanos
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
kernel/futex.c | 148 ++++++++++++++++++++++++---------------------------------
1 file changed, 61 insertions(+), 87 deletions(-)
diff --git a/kernel/futex.c b/kernel/futex.c
index 0b83f9ff2615..e820a56daa6b 100644
--- a/kernel/futex.c
+++ b/kernel/futex.c
@@ -807,6 +807,17 @@ static int lookup_pi_state(u32 uval, struct futex_hash_bucket *hb,
return attach_to_pi_owner(uval, key, ps);
}
+static int lock_pi_update_atomic(u32 __user *uaddr, u32 uval, u32 newval)
+{
+ u32 uninitialized_var(curval);
+
+ if (unlikely(cmpxchg_futex_value_locked(&curval, uaddr, uval, newval)))
+ return -EFAULT;
+
+ /*If user space value changed, let the caller retry */
+ return curval != uval ? -EAGAIN : 0;
+}
+
/**
* futex_lock_pi_atomic() - Atomic work required to acquire a pi aware futex
* @uaddr: the pi futex user address
@@ -830,113 +841,69 @@ static int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb,
struct futex_pi_state **ps,
struct task_struct *task, int set_waiters)
{
- int lock_taken, ret, force_take = 0;
- u32 uval, newval, curval, vpid = task_pid_vnr(task);
-
-retry:
- ret = lock_taken = 0;
+ u32 uval, newval, vpid = task_pid_vnr(task);
+ struct futex_q *match;
+ int ret;
/*
- * To avoid races, we attempt to take the lock here again
- * (by doing a 0 -> TID atomic cmpxchg), while holding all
- * the locks. It will most likely not succeed.
+ * Read the user space value first so we can validate a few
+ * things before proceeding further.
*/
- newval = vpid;
- if (set_waiters)
- newval |= FUTEX_WAITERS;
-
- if (unlikely(cmpxchg_futex_value_locked(&curval, uaddr, 0, newval)))
+ if (get_futex_value_locked(&uval, uaddr))
return -EFAULT;
/*
* Detect deadlocks.
*/
- if ((unlikely((curval & FUTEX_TID_MASK) == vpid)))
+ if ((unlikely((uval & FUTEX_TID_MASK) == vpid)))
return -EDEADLK;
/*
- * Surprise - we got the lock, but we do not trust user space at all.
+ * Lookup existing state first. If it exists, try to attach to
+ * its pi_state.
*/
- if (unlikely(!curval)) {
- /*
- * We verify whether there is kernel state for this
- * futex. If not, we can safely assume, that the 0 ->
- * TID transition is correct. If state exists, we do
- * not bother to fixup the user space state as it was
- * corrupted already.
- */
- return futex_top_waiter(hb, key) ? -EINVAL : 1;
- }
-
- uval = curval;
-
- /*
- * Set the FUTEX_WAITERS flag, so the owner will know it has someone
- * to wake at the next unlock.
- */
- newval = curval | FUTEX_WAITERS;
+ match = futex_top_waiter(hb, key);
+ if (match)
+ return attach_to_pi_state(uval, match->pi_state, ps);
/*
- * Should we force take the futex? See below.
+ * No waiter and user TID is 0. We are here because the
+ * waiters or the owner died bit is set or called from
+ * requeue_cmp_pi or for whatever reason something took the
+ * syscall.
*/
- if (unlikely(force_take)) {
+ if (!(uval & FUTEX_TID_MASK)) {
/*
- * Keep the OWNER_DIED and the WAITERS bit and set the
- * new TID value.
+ * We take over the futex. No other waiters and the user space
+ * TID is 0. We preserve the owner died bit.
*/
- newval = (curval & ~FUTEX_TID_MASK) | vpid;
- force_take = 0;
- lock_taken = 1;
- }
+ newval = uval & FUTEX_OWNER_DIED;
+ newval |= vpid;
- if (unlikely(cmpxchg_futex_value_locked(&curval, uaddr, uval, newval)))
- return -EFAULT;
- if (unlikely(curval != uval))
- goto retry;
+ /* The futex requeue_pi code can enforce the waiters bit */
+ if (set_waiters)
+ newval |= FUTEX_WAITERS;
+
+ ret = lock_pi_update_atomic(uaddr, uval, newval);
+ /* If the take over worked, return 1 */
+ return ret < 0 ? ret : 1;
+ }
/*
- * We took the lock due to forced take over.
+ * First waiter. Set the waiters bit before attaching ourself to
+ * the owner. If owner tries to unlock, it will be forced into
+ * the kernel and blocked on hb->lock.
*/
- if (unlikely(lock_taken))
- return 1;
-
+ newval = uval | FUTEX_WAITERS;
+ ret = lock_pi_update_atomic(uaddr, uval, newval);
+ if (ret)
+ return ret;
/*
- * We dont have the lock. Look up the PI state (or create it if
- * we are the first waiter):
+ * If the update of the user space value succeeded, we try to
+ * attach to the owner. If that fails, no harm done, we only
+ * set the FUTEX_WAITERS bit in the user space variable.
*/
- ret = lookup_pi_state(uval, hb, key, ps);
-
- if (unlikely(ret)) {
- switch (ret) {
- case -ESRCH:
- /*
- * We failed to find an owner for this
- * futex. So we have no pi_state to block
- * on. This can happen in two cases:
- *
- * 1) The owner died
- * 2) A stale FUTEX_WAITERS bit
- *
- * Re-read the futex value.
- */
- if (get_futex_value_locked(&curval, uaddr))
- return -EFAULT;
-
- /*
- * If the owner died or we have a stale
- * WAITERS bit the owner TID in the user space
- * futex is 0.
- */
- if (!(curval & FUTEX_TID_MASK)) {
- force_take = 1;
- goto retry;
- }
- default:
- break;
- }
- }
-
- return ret;
+ return attach_to_pi_owner(uval, key, ps);
}
/**
@@ -1506,7 +1473,12 @@ retry_private:
goto retry;
goto out;
case -EAGAIN:
- /* The owner was exiting, try again. */
+ /*
+ * Two reasons for this:
+ * - Owner is exiting and we just wait for the
+ * exit to complete.
+ * - The user space value changed.
+ */
double_unlock_hb(hb1, hb2);
put_futex_key(&key2);
put_futex_key(&key1);
@@ -2160,8 +2132,10 @@ retry_private:
goto uaddr_faulted;
case -EAGAIN:
/*
- * Task is exiting and we just wait for the
- * exit to complete.
+ * Two reasons for this:
+ * - Task is exiting and we just wait for the
+ * exit to complete.
+ * - The user space value changed.
*/
queue_unlock(&q, hb);
put_futex_key(&q.key);
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 17/31] rt-mutex: avoid a NULL pointer dereference on deadlock
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
` (15 preceding siblings ...)
2015-03-13 15:15 ` [PATCH RT 16/31] futex: Simplify futex_lock_pi_atomic() and make it more robust Steven Rostedt
@ 2015-03-13 15:15 ` Steven Rostedt
2015-03-13 15:15 ` [PATCH RT 18/31] x86: UV: raw_spinlock conversion Steven Rostedt
` (12 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker, stable-rt
[-- Attachment #1: 0017-rt-mutex-avoid-a-NULL-pointer-dereference-on-deadloc.patch --]
[-- Type: text/plain, Size: 1315 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
With task_blocks_on_rt_mutex() returning early -EDEADLK we never add the
waiter to the waitqueue. Later, we try to remove it via remove_waiter()
and go boom in rt_mutex_top_waiter() because rb_entry() gives a NULL
pointer.
Tested on v3.18-RT where rtmutex is used for regular mutex and I tried
to get one twice in a row.
Not sure when this started but I guess 397335f00 ("rtmutex: Fix deadlock
detector for real") or commit 3d5c9340 ("rtmutex: Handle deadlock
detection smarter").
Cc: stable-rt@vger.kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
kernel/rtmutex.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/kernel/rtmutex.c b/kernel/rtmutex.c
index b7b1de2acf0b..5388ba98acaf 100644
--- a/kernel/rtmutex.c
+++ b/kernel/rtmutex.c
@@ -1490,7 +1490,8 @@ rt_mutex_slowlock(struct rt_mutex *lock, int state,
set_current_state(TASK_RUNNING);
if (unlikely(ret)) {
- remove_waiter(lock, &waiter);
+ if (rt_mutex_has_waiters(lock))
+ remove_waiter(lock, &waiter);
rt_mutex_handle_deadlock(ret, chwalk, &waiter);
}
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 18/31] x86: UV: raw_spinlock conversion
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
` (16 preceding siblings ...)
2015-03-13 15:15 ` [PATCH RT 17/31] rt-mutex: avoid a NULL pointer dereference on deadlock Steven Rostedt
@ 2015-03-13 15:15 ` Steven Rostedt
2015-03-13 15:15 ` [PATCH RT 20/31] arm/futex: disable preemption during futex_atomic_cmpxchg_inatomic() Steven Rostedt
` (11 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker, stable-rt, Mike Galbraith
[-- Attachment #1: 0018-x86-UV-raw_spinlock-conversion.patch --]
[-- Type: text/plain, Size: 8298 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: Mike Galbraith <umgwanakikbuti@gmail.com>
Shrug. Lots of hobbyists have a beast in their basement, right?
Cc: stable-rt@vger.kernel.org
Signed-off-by: Mike Galbraith <mgalbraith@suse.de>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
arch/x86/include/asm/uv/uv_bau.h | 14 +++++++-------
arch/x86/include/asm/uv/uv_hub.h | 2 +-
arch/x86/kernel/apic/x2apic_uv_x.c | 2 +-
arch/x86/platform/uv/tlb_uv.c | 26 +++++++++++++-------------
arch/x86/platform/uv/uv_time.c | 21 +++++++++++++--------
5 files changed, 35 insertions(+), 30 deletions(-)
diff --git a/arch/x86/include/asm/uv/uv_bau.h b/arch/x86/include/asm/uv/uv_bau.h
index a06983cdc125..afaaf2900796 100644
--- a/arch/x86/include/asm/uv/uv_bau.h
+++ b/arch/x86/include/asm/uv/uv_bau.h
@@ -611,9 +611,9 @@ struct bau_control {
cycles_t send_message;
cycles_t period_end;
cycles_t period_time;
- spinlock_t uvhub_lock;
- spinlock_t queue_lock;
- spinlock_t disable_lock;
+ raw_spinlock_t uvhub_lock;
+ raw_spinlock_t queue_lock;
+ raw_spinlock_t disable_lock;
/* tunables */
int max_concurr;
int max_concurr_const;
@@ -770,15 +770,15 @@ static inline int atom_asr(short i, struct atomic_short *v)
* to be lowered below the current 'v'. atomic_add_unless can only stop
* on equal.
*/
-static inline int atomic_inc_unless_ge(spinlock_t *lock, atomic_t *v, int u)
+static inline int atomic_inc_unless_ge(raw_spinlock_t *lock, atomic_t *v, int u)
{
- spin_lock(lock);
+ raw_spin_lock(lock);
if (atomic_read(v) >= u) {
- spin_unlock(lock);
+ raw_spin_unlock(lock);
return 0;
}
atomic_inc(v);
- spin_unlock(lock);
+ raw_spin_unlock(lock);
return 1;
}
diff --git a/arch/x86/include/asm/uv/uv_hub.h b/arch/x86/include/asm/uv/uv_hub.h
index 2c32df95bb78..2c676e8d39ac 100644
--- a/arch/x86/include/asm/uv/uv_hub.h
+++ b/arch/x86/include/asm/uv/uv_hub.h
@@ -502,7 +502,7 @@ struct uv_blade_info {
unsigned short nr_online_cpus;
unsigned short pnode;
short memory_nid;
- spinlock_t nmi_lock;
+ raw_spinlock_t nmi_lock;
unsigned long nmi_count;
};
extern struct uv_blade_info *uv_blade_info;
diff --git a/arch/x86/kernel/apic/x2apic_uv_x.c b/arch/x86/kernel/apic/x2apic_uv_x.c
index b32dbb411a9a..7a6ab4f38278 100644
--- a/arch/x86/kernel/apic/x2apic_uv_x.c
+++ b/arch/x86/kernel/apic/x2apic_uv_x.c
@@ -928,7 +928,7 @@ void __init uv_system_init(void)
uv_blade_info[blade].pnode = pnode;
uv_blade_info[blade].nr_possible_cpus = 0;
uv_blade_info[blade].nr_online_cpus = 0;
- spin_lock_init(&uv_blade_info[blade].nmi_lock);
+ raw_spin_lock_init(&uv_blade_info[blade].nmi_lock);
min_pnode = min(pnode, min_pnode);
max_pnode = max(pnode, max_pnode);
blade++;
diff --git a/arch/x86/platform/uv/tlb_uv.c b/arch/x86/platform/uv/tlb_uv.c
index 0f92173a12b6..7ddc988cfa93 100644
--- a/arch/x86/platform/uv/tlb_uv.c
+++ b/arch/x86/platform/uv/tlb_uv.c
@@ -695,9 +695,9 @@ static void destination_plugged(struct bau_desc *bau_desc,
quiesce_local_uvhub(hmaster);
- spin_lock(&hmaster->queue_lock);
+ raw_spin_lock(&hmaster->queue_lock);
reset_with_ipi(&bau_desc->distribution, bcp);
- spin_unlock(&hmaster->queue_lock);
+ raw_spin_unlock(&hmaster->queue_lock);
end_uvhub_quiesce(hmaster);
@@ -717,9 +717,9 @@ static void destination_timeout(struct bau_desc *bau_desc,
quiesce_local_uvhub(hmaster);
- spin_lock(&hmaster->queue_lock);
+ raw_spin_lock(&hmaster->queue_lock);
reset_with_ipi(&bau_desc->distribution, bcp);
- spin_unlock(&hmaster->queue_lock);
+ raw_spin_unlock(&hmaster->queue_lock);
end_uvhub_quiesce(hmaster);
@@ -740,7 +740,7 @@ static void disable_for_period(struct bau_control *bcp, struct ptc_stats *stat)
cycles_t tm1;
hmaster = bcp->uvhub_master;
- spin_lock(&hmaster->disable_lock);
+ raw_spin_lock(&hmaster->disable_lock);
if (!bcp->baudisabled) {
stat->s_bau_disabled++;
tm1 = get_cycles();
@@ -753,7 +753,7 @@ static void disable_for_period(struct bau_control *bcp, struct ptc_stats *stat)
}
}
}
- spin_unlock(&hmaster->disable_lock);
+ raw_spin_unlock(&hmaster->disable_lock);
}
static void count_max_concurr(int stat, struct bau_control *bcp,
@@ -816,7 +816,7 @@ static void record_send_stats(cycles_t time1, cycles_t time2,
*/
static void uv1_throttle(struct bau_control *hmaster, struct ptc_stats *stat)
{
- spinlock_t *lock = &hmaster->uvhub_lock;
+ raw_spinlock_t *lock = &hmaster->uvhub_lock;
atomic_t *v;
v = &hmaster->active_descriptor_count;
@@ -948,7 +948,7 @@ static int check_enable(struct bau_control *bcp, struct ptc_stats *stat)
struct bau_control *hmaster;
hmaster = bcp->uvhub_master;
- spin_lock(&hmaster->disable_lock);
+ raw_spin_lock(&hmaster->disable_lock);
if (bcp->baudisabled && (get_cycles() >= bcp->set_bau_on_time)) {
stat->s_bau_reenabled++;
for_each_present_cpu(tcpu) {
@@ -960,10 +960,10 @@ static int check_enable(struct bau_control *bcp, struct ptc_stats *stat)
tbcp->period_giveups = 0;
}
}
- spin_unlock(&hmaster->disable_lock);
+ raw_spin_unlock(&hmaster->disable_lock);
return 0;
}
- spin_unlock(&hmaster->disable_lock);
+ raw_spin_unlock(&hmaster->disable_lock);
return -1;
}
@@ -1880,9 +1880,9 @@ static void __init init_per_cpu_tunables(void)
bcp->cong_reps = congested_reps;
bcp->disabled_period = sec_2_cycles(disabled_period);
bcp->giveup_limit = giveup_limit;
- spin_lock_init(&bcp->queue_lock);
- spin_lock_init(&bcp->uvhub_lock);
- spin_lock_init(&bcp->disable_lock);
+ raw_spin_lock_init(&bcp->queue_lock);
+ raw_spin_lock_init(&bcp->uvhub_lock);
+ raw_spin_lock_init(&bcp->disable_lock);
}
}
diff --git a/arch/x86/platform/uv/uv_time.c b/arch/x86/platform/uv/uv_time.c
index 5c86786bbfd2..c039afa26aa2 100644
--- a/arch/x86/platform/uv/uv_time.c
+++ b/arch/x86/platform/uv/uv_time.c
@@ -58,7 +58,7 @@ static DEFINE_PER_CPU(struct clock_event_device, cpu_ced);
/* There is one of these allocated per node */
struct uv_rtc_timer_head {
- spinlock_t lock;
+ raw_spinlock_t lock;
/* next cpu waiting for timer, local node relative: */
int next_cpu;
/* number of cpus on this node: */
@@ -178,7 +178,7 @@ static __init int uv_rtc_allocate_timers(void)
uv_rtc_deallocate_timers();
return -ENOMEM;
}
- spin_lock_init(&head->lock);
+ raw_spin_lock_init(&head->lock);
head->ncpus = uv_blade_nr_possible_cpus(bid);
head->next_cpu = -1;
blade_info[bid] = head;
@@ -232,7 +232,7 @@ static int uv_rtc_set_timer(int cpu, u64 expires)
unsigned long flags;
int next_cpu;
- spin_lock_irqsave(&head->lock, flags);
+ raw_spin_lock_irqsave(&head->lock, flags);
next_cpu = head->next_cpu;
*t = expires;
@@ -244,12 +244,12 @@ static int uv_rtc_set_timer(int cpu, u64 expires)
if (uv_setup_intr(cpu, expires)) {
*t = ULLONG_MAX;
uv_rtc_find_next_timer(head, pnode);
- spin_unlock_irqrestore(&head->lock, flags);
+ raw_spin_unlock_irqrestore(&head->lock, flags);
return -ETIME;
}
}
- spin_unlock_irqrestore(&head->lock, flags);
+ raw_spin_unlock_irqrestore(&head->lock, flags);
return 0;
}
@@ -268,7 +268,7 @@ static int uv_rtc_unset_timer(int cpu, int force)
unsigned long flags;
int rc = 0;
- spin_lock_irqsave(&head->lock, flags);
+ raw_spin_lock_irqsave(&head->lock, flags);
if ((head->next_cpu == bcpu && uv_read_rtc(NULL) >= *t) || force)
rc = 1;
@@ -280,7 +280,7 @@ static int uv_rtc_unset_timer(int cpu, int force)
uv_rtc_find_next_timer(head, pnode);
}
- spin_unlock_irqrestore(&head->lock, flags);
+ raw_spin_unlock_irqrestore(&head->lock, flags);
return rc;
}
@@ -300,13 +300,18 @@ static int uv_rtc_unset_timer(int cpu, int force)
static cycle_t uv_read_rtc(struct clocksource *cs)
{
unsigned long offset;
+ cycle_t cycles;
+ preempt_disable();
if (uv_get_min_hub_revision_id() == 1)
offset = 0;
else
offset = (uv_blade_processor_id() * L1_CACHE_BYTES) % PAGE_SIZE;
- return (cycle_t)uv_read_local_mmr(UVH_RTC | offset);
+ cycles = (cycle_t)uv_read_local_mmr(UVH_RTC | offset);
+ preempt_enable();
+
+ return cycles;
}
/*
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 20/31] arm/futex: disable preemption during futex_atomic_cmpxchg_inatomic()
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
` (17 preceding siblings ...)
2015-03-13 15:15 ` [PATCH RT 18/31] x86: UV: raw_spinlock conversion Steven Rostedt
@ 2015-03-13 15:15 ` Steven Rostedt
2015-03-13 15:15 ` [PATCH RT 21/31] ARM: cmpxchg: define __HAVE_ARCH_CMPXCHG for armv6 and later Steven Rostedt
` (10 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker, stable-rt
[-- Attachment #1: 0020-arm-futex-disable-preemption-during-futex_atomic_cmp.patch --]
[-- Type: text/plain, Size: 1879 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
The ARM UP implementation of futex_atomic_cmpxchg_inatomic() assumes that
pagefault_disable() inherits a preempt disabled section. This assumtion
is true for mainline but -RT reverts this and allows preemption in
pagefault disabled regions.
The code sequence of futex_atomic_cmpxchg_inatomic():
| x = *futex;
| if (x == oldval)
| *futex = newval;
The problem occurs if the code is preempted after reading the futex value or
after comparing it with x. While preempted, the futex owner has to be
scheduled which then releases the lock (in userland because it has no waiter
yet). Once the code is back on the CPU, it overwrites the futex value
with with the old PID and the waiter bit set.
The workaround is to explicit disable code preemption to avoid the
described race window.
Debugged-by: Thomas Gleixner <tglx@linutronix.de>
Cc: stable-rt@vger.kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
arch/arm/include/asm/futex.h | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/arch/arm/include/asm/futex.h b/arch/arm/include/asm/futex.h
index 2aff798fbef4..54d26a98ff84 100644
--- a/arch/arm/include/asm/futex.h
+++ b/arch/arm/include/asm/futex.h
@@ -90,6 +90,8 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32)))
return -EFAULT;
+ preempt_disable_rt();
+
__asm__ __volatile__("@futex_atomic_cmpxchg_inatomic\n"
"1: " TUSER(ldr) " %1, [%4]\n"
" teq %1, %2\n"
@@ -101,6 +103,8 @@ futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *uaddr,
: "cc", "memory");
*uval = val;
+
+ preempt_enable_rt();
return ret;
}
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 21/31] ARM: cmpxchg: define __HAVE_ARCH_CMPXCHG for armv6 and later
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
` (18 preceding siblings ...)
2015-03-13 15:15 ` [PATCH RT 20/31] arm/futex: disable preemption during futex_atomic_cmpxchg_inatomic() Steven Rostedt
@ 2015-03-13 15:15 ` Steven Rostedt
2015-03-13 15:15 ` [PATCH RT 22/31] sas-ata/isci: dontt disable interrupts in qc_issue handler Steven Rostedt
` (9 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker, stable-rt
[-- Attachment #1: 0021-ARM-cmpxchg-define-__HAVE_ARCH_CMPXCHG-for-armv6-and.patch --]
[-- Type: text/plain, Size: 1566 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
Both pi_stress and sigwaittest in rt-test show performance gain with
__HAVE_ARCH_CMPXCHG. Testing result on coretile_express_a9x4:
pi_stress -p 99 --duration=300 (on linux-3.4-rc5; bigger is better)
vanilla: Total inversion performed: 5493381
patched: Total inversion performed: 5621746
sigwaittest -p 99 -l 100000 (on linux-3.4-rc5-rt6; less is better)
3.4-rc5-rt6: Min 24, Cur 27, Avg 30, Max 98
patched: Min 19, Cur 21, Avg 23, Max 96
Signed-off-by: Yong Zhang <yong.zhang0 at gmail.com>
Cc: Russell King <rmk+kernel at arm.linux.org.uk>
Cc: Nicolas Pitre <nico at linaro.org>
Cc: Will Deacon <will.deacon at arm.com>
Cc: Catalin Marinas <catalin.marinas at arm.com>
Cc: Thomas Gleixner <tglx at linutronix.de>
Cc: linux-arm-kernel at lists.infradead.org
Cc: stable-rt@vger.kernel.org
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
arch/arm/include/asm/cmpxchg.h | 2 ++
1 file changed, 2 insertions(+)
diff --git a/arch/arm/include/asm/cmpxchg.h b/arch/arm/include/asm/cmpxchg.h
index 4f009c10540d..a86b680dd7cf 100644
--- a/arch/arm/include/asm/cmpxchg.h
+++ b/arch/arm/include/asm/cmpxchg.h
@@ -127,6 +127,8 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size
#else /* min ARCH >= ARMv6 */
+#define __HAVE_ARCH_CMPXCHG 1
+
extern void __bad_cmpxchg(volatile void *ptr, int size);
/*
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 22/31] sas-ata/isci: dontt disable interrupts in qc_issue handler
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
` (19 preceding siblings ...)
2015-03-13 15:15 ` [PATCH RT 21/31] ARM: cmpxchg: define __HAVE_ARCH_CMPXCHG for armv6 and later Steven Rostedt
@ 2015-03-13 15:15 ` Steven Rostedt
2015-03-13 15:15 ` [PATCH RT 23/31] scheduling while atomic in cgroup code Steven Rostedt
` (8 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker, stable-rt
[-- Attachment #1: 0022-sas-ata-isci-dont-t-disable-interrupts-in-qc_issue-h.patch --]
[-- Type: text/plain, Size: 3473 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: Paul Gortmaker <paul.gortmaker@windriver.com>
On 3.14-rt we see the following trace on Canoe Pass for
SCSI_ISCI "Intel(R) C600 Series Chipset SAS Controller"
when the sas qc_issue handler is run:
BUG: sleeping function called from invalid context at kernel/locking/rtmutex.c:905
in_atomic(): 0, irqs_disabled(): 1, pid: 432, name: udevd
CPU: 11 PID: 432 Comm: udevd Not tainted 3.14.28-rt22 #2
Hardware name: Intel Corporation S2600CP/S2600CP, BIOS SE5C600.86B.02.01.0002.082220131453 08/22/2013
ffff880fab500000 ffff880fa9f239c0 ffffffff81a2d273 0000000000000000
ffff880fa9f239d8 ffffffff8107f023 ffff880faac23dc0 ffff880fa9f239f0
ffffffff81a33cc0 ffff880faaeb1400 ffff880fa9f23a40 ffffffff815de891
Call Trace:
[<ffffffff81a2d273>] dump_stack+0x4e/0x7a
[<ffffffff8107f023>] __might_sleep+0xe3/0x160
[<ffffffff81a33cc0>] rt_spin_lock+0x20/0x50
[<ffffffff815de891>] isci_task_execute_task+0x171/0x2f0 <-----
[<ffffffff815cfecb>] sas_ata_qc_issue+0x25b/0x2a0
[<ffffffff81606363>] ata_qc_issue+0x1f3/0x370
[<ffffffff8160c600>] ? ata_scsi_invalid_field+0x40/0x40
[<ffffffff8160c8f5>] ata_scsi_translate+0xa5/0x1b0
[<ffffffff8160efc6>] ata_sas_queuecmd+0x86/0x280
[<ffffffff815ce446>] sas_queuecommand+0x196/0x230
[<ffffffff81081fad>] ? get_parent_ip+0xd/0x50
[<ffffffff815b05a4>] scsi_dispatch_cmd+0xb4/0x210
[<ffffffff815b7744>] scsi_request_fn+0x314/0x530
and gdb shows:
(gdb) list * isci_task_execute_task+0x171
0xffffffff815ddfb1 is in isci_task_execute_task (drivers/scsi/isci/task.c:138).
133 dev_dbg(&ihost->pdev->dev, "%s: num=%d\n", __func__, num);
134
135 for_each_sas_task(num, task) {
136 enum sci_status status = SCI_FAILURE;
137
138 spin_lock_irqsave(&ihost->scic_lock, flags); <-----
139 idev = isci_lookup_device(task->dev);
140 io_ready = isci_device_io_ready(idev, task);
141 tag = isci_alloc_tag(ihost);
142 spin_unlock_irqrestore(&ihost->scic_lock, flags);
(gdb)
In addition to the scic_lock, the function also contains locking of
the task_state_lock -- which is clearly not a candidate for raw lock
conversion. As can be seen by the comment nearby, we really should
be running the qc_issue code with interrupts enabled anyway.
Cc: stable-rt@vger.kernel.org
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
drivers/scsi/libsas/sas_ata.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/drivers/scsi/libsas/sas_ata.c b/drivers/scsi/libsas/sas_ata.c
index d2895836f9fa..c85e07ab51e4 100644
--- a/drivers/scsi/libsas/sas_ata.c
+++ b/drivers/scsi/libsas/sas_ata.c
@@ -191,7 +191,7 @@ static unsigned int sas_ata_qc_issue(struct ata_queued_cmd *qc)
/* TODO: audit callers to ensure they are ready for qc_issue to
* unconditionally re-enable interrupts
*/
- local_irq_save(flags);
+ local_irq_save_nort(flags);
spin_unlock(ap->lock);
/* If the device fell off, no sense in issuing commands */
@@ -261,7 +261,7 @@ static unsigned int sas_ata_qc_issue(struct ata_queued_cmd *qc)
out:
spin_lock(ap->lock);
- local_irq_restore(flags);
+ local_irq_restore_nort(flags);
return ret;
}
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 23/31] scheduling while atomic in cgroup code
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
` (20 preceding siblings ...)
2015-03-13 15:15 ` [PATCH RT 22/31] sas-ata/isci: dontt disable interrupts in qc_issue handler Steven Rostedt
@ 2015-03-13 15:15 ` Steven Rostedt
2015-03-13 15:15 ` [PATCH RT 24/31] work-simple: Simple work queue implemenation Steven Rostedt
` (7 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker, stable-rt, Nikita Yushchenko,
Mike Galbraith
[-- Attachment #1: 0023-scheduling-while-atomic-in-cgroup-code.patch --]
[-- Type: text/plain, Size: 2201 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: Mike Galbraith <umgwanakikbuti@gmail.com>
mm, memcg: make refill_stock() use get_cpu_light()
Nikita reported the following memcg scheduling while atomic bug:
Call Trace:
[e22d5a90] [c0007ea8] show_stack+0x4c/0x168 (unreliable)
[e22d5ad0] [c0618c04] __schedule_bug+0x94/0xb0
[e22d5ae0] [c060b9ec] __schedule+0x530/0x550
[e22d5bf0] [c060bacc] schedule+0x30/0xbc
[e22d5c00] [c060ca24] rt_spin_lock_slowlock+0x180/0x27c
[e22d5c70] [c00b39dc] res_counter_uncharge_until+0x40/0xc4
[e22d5ca0] [c013ca88] drain_stock.isra.20+0x54/0x98
[e22d5cc0] [c01402ac] __mem_cgroup_try_charge+0x2e8/0xbac
[e22d5d70] [c01410d4] mem_cgroup_charge_common+0x3c/0x70
[e22d5d90] [c0117284] __do_fault+0x38c/0x510
[e22d5df0] [c011a5f4] handle_pte_fault+0x98/0x858
[e22d5e50] [c060ed08] do_page_fault+0x42c/0x6fc
[e22d5f40] [c000f5b4] handle_page_fault+0xc/0x80
What happens:
refill_stock()
get_cpu_var()
drain_stock()
res_counter_uncharge()
res_counter_uncharge_until()
spin_lock() <== boom
Fix it by replacing get/put_cpu_var() with get/put_cpu_light().
Cc: stable-rt@vger.kernel.org
Reported-by: Nikita Yushchenko <nyushchenko@dev.rtsoft.ru>
Signed-off-by: Mike Galbraith <umgwanakikbuti@gmail.com>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
mm/memcontrol.c | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 33d8392f33eb..89540bca5c5b 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -2463,14 +2463,17 @@ static void __init memcg_stock_init(void)
*/
static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
{
- struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);
+ struct memcg_stock_pcp *stock;
+ int cpu = get_cpu_light();
+
+ stock = &per_cpu(memcg_stock, cpu);
if (stock->cached != memcg) { /* reset if necessary */
drain_stock(stock);
stock->cached = memcg;
}
stock->nr_pages += nr_pages;
- put_cpu_var(memcg_stock);
+ put_cpu_light();
}
/*
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 24/31] work-simple: Simple work queue implemenation
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
` (21 preceding siblings ...)
2015-03-13 15:15 ` [PATCH RT 23/31] scheduling while atomic in cgroup code Steven Rostedt
@ 2015-03-13 15:15 ` Steven Rostedt
2015-03-13 15:15 ` [PATCH RT 25/31] sunrpc: make svc_xprt_do_enqueue() use get_cpu_light() Steven Rostedt
` (6 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker, Daniel Wagner
[-- Attachment #1: 0024-work-simple-Simple-work-queue-implemenation.patch --]
[-- Type: text/plain, Size: 5727 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: Daniel Wagner <daniel.wagner@bmw-carit.de>
Provides a framework for enqueuing callbacks from irq context
PREEMPT_RT_FULL safe. The callbacks are executed in kthread context.
Bases on wait-simple.
Signed-off-by: Daniel Wagner <daniel.wagner@bmw-carit.de>
Cc: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
include/linux/work-simple.h | 24 +++++++
kernel/sched/Makefile | 1 +
kernel/sched/work-simple.c | 172 ++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 197 insertions(+)
create mode 100644 include/linux/work-simple.h
create mode 100644 kernel/sched/work-simple.c
diff --git a/include/linux/work-simple.h b/include/linux/work-simple.h
new file mode 100644
index 000000000000..f175fa9a6016
--- /dev/null
+++ b/include/linux/work-simple.h
@@ -0,0 +1,24 @@
+#ifndef _LINUX_SWORK_H
+#define _LINUX_SWORK_H
+
+#include <linux/list.h>
+
+struct swork_event {
+ struct list_head item;
+ unsigned long flags;
+ void (*func)(struct swork_event *);
+};
+
+static inline void INIT_SWORK(struct swork_event *event,
+ void (*func)(struct swork_event *))
+{
+ event->flags = 0;
+ event->func = func;
+}
+
+bool swork_queue(struct swork_event *sev);
+
+int swork_get(void);
+void swork_put(void);
+
+#endif /* _LINUX_SWORK_H */
diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile
index deaf90e4a1de..45759d3d5451 100644
--- a/kernel/sched/Makefile
+++ b/kernel/sched/Makefile
@@ -12,6 +12,7 @@ CFLAGS_core.o := $(PROFILING) -fno-omit-frame-pointer
endif
obj-y += core.o clock.o cputime.o idle_task.o fair.o rt.o stop_task.o
+obj-y += work-simple.o
obj-$(CONFIG_SMP) += cpupri.o
obj-$(CONFIG_SCHED_AUTOGROUP) += auto_group.o
obj-$(CONFIG_SCHEDSTATS) += stats.o
diff --git a/kernel/sched/work-simple.c b/kernel/sched/work-simple.c
new file mode 100644
index 000000000000..c996f755dba6
--- /dev/null
+++ b/kernel/sched/work-simple.c
@@ -0,0 +1,172 @@
+/*
+ * Copyright (C) 2014 BMW Car IT GmbH, Daniel Wagner daniel.wagner@bmw-carit.de
+ *
+ * Provides a framework for enqueuing callbacks from irq context
+ * PREEMPT_RT_FULL safe. The callbacks are executed in kthread context.
+ */
+
+#include <linux/wait-simple.h>
+#include <linux/work-simple.h>
+#include <linux/kthread.h>
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+
+#define SWORK_EVENT_PENDING (1 << 0)
+
+static DEFINE_MUTEX(worker_mutex);
+static struct sworker *glob_worker;
+
+struct sworker {
+ struct list_head events;
+ struct swait_head wq;
+
+ raw_spinlock_t lock;
+
+ struct task_struct *task;
+ int refs;
+};
+
+static bool swork_readable(struct sworker *worker)
+{
+ bool r;
+
+ if (kthread_should_stop())
+ return true;
+
+ raw_spin_lock_irq(&worker->lock);
+ r = !list_empty(&worker->events);
+ raw_spin_unlock_irq(&worker->lock);
+
+ return r;
+}
+
+static int swork_kthread(void *arg)
+{
+ struct sworker *worker = arg;
+
+ for (;;) {
+ swait_event_interruptible(worker->wq,
+ swork_readable(worker));
+ if (kthread_should_stop())
+ break;
+
+ raw_spin_lock_irq(&worker->lock);
+ while (!list_empty(&worker->events)) {
+ struct swork_event *sev;
+
+ sev = list_first_entry(&worker->events,
+ struct swork_event, item);
+ list_del(&sev->item);
+ raw_spin_unlock_irq(&worker->lock);
+
+ WARN_ON_ONCE(!test_and_clear_bit(SWORK_EVENT_PENDING,
+ &sev->flags));
+ sev->func(sev);
+ raw_spin_lock_irq(&worker->lock);
+ }
+ raw_spin_unlock_irq(&worker->lock);
+ }
+ return 0;
+}
+
+static struct sworker *swork_create(void)
+{
+ struct sworker *worker;
+
+ worker = kzalloc(sizeof(*worker), GFP_KERNEL);
+ if (!worker)
+ return ERR_PTR(-ENOMEM);
+
+ INIT_LIST_HEAD(&worker->events);
+ raw_spin_lock_init(&worker->lock);
+ init_swait_head(&worker->wq);
+
+ worker->task = kthread_run(swork_kthread, worker, "kswork");
+ if (IS_ERR(worker->task)) {
+ kfree(worker);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ return worker;
+}
+
+static void swork_destroy(struct sworker *worker)
+{
+ kthread_stop(worker->task);
+
+ WARN_ON(!list_empty(&worker->events));
+ kfree(worker);
+}
+
+/**
+ * swork_queue - queue swork
+ *
+ * Returns %false if @work was already on a queue, %true otherwise.
+ *
+ * The work is queued and processed on a random CPU
+ */
+bool swork_queue(struct swork_event *sev)
+{
+ unsigned long flags;
+
+ if (test_and_set_bit(SWORK_EVENT_PENDING, &sev->flags))
+ return false;
+
+ raw_spin_lock_irqsave(&glob_worker->lock, flags);
+ list_add_tail(&sev->item, &glob_worker->events);
+ raw_spin_unlock_irqrestore(&glob_worker->lock, flags);
+
+ swait_wake(&glob_worker->wq);
+ return true;
+}
+EXPORT_SYMBOL_GPL(swork_queue);
+
+/**
+ * swork_get - get an instance of the sworker
+ *
+ * Returns an negative error code if the initialization if the worker did not
+ * work, %0 otherwise.
+ *
+ */
+int swork_get(void)
+{
+ struct sworker *worker;
+
+ mutex_lock(&worker_mutex);
+ if (!glob_worker) {
+ worker = swork_create();
+ if (IS_ERR(worker)) {
+ mutex_unlock(&worker_mutex);
+ return -ENOMEM;
+ }
+
+ glob_worker = worker;
+ }
+
+ glob_worker->refs++;
+ mutex_unlock(&worker_mutex);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(swork_get);
+
+/**
+ * swork_put - puts an instance of the sworker
+ *
+ * Will destroy the sworker thread. This function must not be called until all
+ * queued events have been completed.
+ */
+void swork_put(void)
+{
+ mutex_lock(&worker_mutex);
+
+ glob_worker->refs--;
+ if (glob_worker->refs > 0)
+ goto out;
+
+ swork_destroy(glob_worker);
+ glob_worker = NULL;
+out:
+ mutex_unlock(&worker_mutex);
+}
+EXPORT_SYMBOL_GPL(swork_put);
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 25/31] sunrpc: make svc_xprt_do_enqueue() use get_cpu_light()
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
` (22 preceding siblings ...)
2015-03-13 15:15 ` [PATCH RT 24/31] work-simple: Simple work queue implemenation Steven Rostedt
@ 2015-03-13 15:15 ` Steven Rostedt
2015-03-13 15:15 ` [PATCH RT 26/31] lockdep: selftest: fix warnings due to missing PREEMPT_RT conditionals Steven Rostedt
` (5 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker, stable-rt, Mike Galbraith
[-- Attachment #1: 0025-sunrpc-make-svc_xprt_do_enqueue-use-get_cpu_light.patch --]
[-- Type: text/plain, Size: 2120 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: Mike Galbraith <umgwanakikbuti@gmail.com>
|BUG: sleeping function called from invalid context at kernel/locking/rtmutex.c:915
|in_atomic(): 1, irqs_disabled(): 0, pid: 3194, name: rpc.nfsd
|Preemption disabled at:[<ffffffffa06bf0bb>] svc_xprt_received+0x4b/0xc0 [sunrpc]
|CPU: 6 PID: 3194 Comm: rpc.nfsd Not tainted 3.18.7-rt1 #9
|Hardware name: MEDION MS-7848/MS-7848, BIOS M7848W08.404 11/06/2014
| ffff880409630000 ffff8800d9a33c78 ffffffff815bdeb5 0000000000000002
| 0000000000000000 ffff8800d9a33c98 ffffffff81073c86 ffff880408dd6008
| ffff880408dd6000 ffff8800d9a33cb8 ffffffff815c3d84 ffff88040b3ac000
|Call Trace:
| [<ffffffff815bdeb5>] dump_stack+0x4f/0x9e
| [<ffffffff81073c86>] __might_sleep+0xe6/0x150
| [<ffffffff815c3d84>] rt_spin_lock+0x24/0x50
| [<ffffffffa06beec0>] svc_xprt_do_enqueue+0x80/0x230 [sunrpc]
| [<ffffffffa06bf0bb>] svc_xprt_received+0x4b/0xc0 [sunrpc]
| [<ffffffffa06c03ed>] svc_add_new_perm_xprt+0x6d/0x80 [sunrpc]
| [<ffffffffa06b2693>] svc_addsock+0x143/0x200 [sunrpc]
| [<ffffffffa072e69c>] write_ports+0x28c/0x340 [nfsd]
| [<ffffffffa072d2ac>] nfsctl_transaction_write+0x4c/0x80 [nfsd]
| [<ffffffff8117ee83>] vfs_write+0xb3/0x1d0
| [<ffffffff8117f889>] SyS_write+0x49/0xb0
| [<ffffffff815c4556>] system_call_fastpath+0x16/0x1b
Cc: stable-rt@vger.kernel.org
Signed-off-by: Mike Galbraith <umgwanakikbuti@gmail.com>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
net/sunrpc/svc_xprt.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c
index b9aad4723a9d..ab50dbd58c20 100644
--- a/net/sunrpc/svc_xprt.c
+++ b/net/sunrpc/svc_xprt.c
@@ -349,9 +349,9 @@ void svc_xprt_enqueue(struct svc_xprt *xprt)
if (!svc_xprt_has_something_to_do(xprt))
return;
- cpu = get_cpu();
+ cpu = get_cpu_light();
pool = svc_pool_for_cpu(xprt->xpt_server, cpu);
- put_cpu();
+ put_cpu_light();
spin_lock_bh(&pool->sp_lock);
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 26/31] lockdep: selftest: fix warnings due to missing PREEMPT_RT conditionals
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
` (23 preceding siblings ...)
2015-03-13 15:15 ` [PATCH RT 25/31] sunrpc: make svc_xprt_do_enqueue() use get_cpu_light() Steven Rostedt
@ 2015-03-13 15:15 ` Steven Rostedt
2015-03-13 15:15 ` [PATCH RT 27/31] timers: Track total number of timers in list Steven Rostedt
` (4 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker, stable-rt, Josh Cartwright,
Xander Huff, Gratian Crisan
[-- Attachment #1: 0026-lockdep-selftest-fix-warnings-due-to-missing-PREEMPT.patch --]
[-- Type: text/plain, Size: 4377 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: Josh Cartwright <josh.cartwright@ni.com>
"lockdep: Selftest: Only do hardirq context test for raw spinlock"
disabled the execution of certain tests with PREEMPT_RT_FULL, but did
not prevent the tests from still being defined. This leads to warnings
like:
./linux/lib/locking-selftest.c:574:1: warning: 'irqsafe1_hard_rlock_12' defined but not used [-Wunused-function]
./linux/lib/locking-selftest.c:574:1: warning: 'irqsafe1_hard_rlock_21' defined but not used [-Wunused-function]
./linux/lib/locking-selftest.c:577:1: warning: 'irqsafe1_hard_wlock_12' defined but not used [-Wunused-function]
./linux/lib/locking-selftest.c:577:1: warning: 'irqsafe1_hard_wlock_21' defined but not used [-Wunused-function]
./linux/lib/locking-selftest.c:580:1: warning: 'irqsafe1_soft_spin_12' defined but not used [-Wunused-function]
...
Fixed by wrapping the test definitions in #ifndef CONFIG_PREEMPT_RT_FULL
conditionals.
Cc: stable-rt@vger.kernel.org
Signed-off-by: Josh Cartwright <josh.cartwright@ni.com>
Signed-off-by: Xander Huff <xander.huff@ni.com>
Acked-by: Gratian Crisan <gratian.crisan@ni.com>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
lib/locking-selftest.c | 27 +++++++++++++++++++++++++++
1 file changed, 27 insertions(+)
diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c
index 23b856400e5c..3508e2c80465 100644
--- a/lib/locking-selftest.c
+++ b/lib/locking-selftest.c
@@ -569,6 +569,8 @@ GENERATE_TESTCASE(init_held_rsem)
#include "locking-selftest-spin-hardirq.h"
GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_spin)
+#ifndef CONFIG_PREEMPT_RT_FULL
+
#include "locking-selftest-rlock-hardirq.h"
GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_rlock)
@@ -584,9 +586,12 @@ GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_soft_rlock)
#include "locking-selftest-wlock-softirq.h"
GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_soft_wlock)
+#endif
+
#undef E1
#undef E2
+#ifndef CONFIG_PREEMPT_RT_FULL
/*
* Enabling hardirqs with a softirq-safe lock held:
*/
@@ -619,6 +624,8 @@ GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2A_rlock)
#undef E1
#undef E2
+#endif
+
/*
* Enabling irqs with an irq-safe lock held:
*/
@@ -642,6 +649,8 @@ GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2A_rlock)
#include "locking-selftest-spin-hardirq.h"
GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_spin)
+#ifndef CONFIG_PREEMPT_RT_FULL
+
#include "locking-selftest-rlock-hardirq.h"
GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_rlock)
@@ -657,6 +666,8 @@ GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_rlock)
#include "locking-selftest-wlock-softirq.h"
GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_wlock)
+#endif
+
#undef E1
#undef E2
@@ -688,6 +699,8 @@ GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_wlock)
#include "locking-selftest-spin-hardirq.h"
GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_spin)
+#ifndef CONFIG_PREEMPT_RT_FULL
+
#include "locking-selftest-rlock-hardirq.h"
GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_rlock)
@@ -703,6 +716,8 @@ GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_rlock)
#include "locking-selftest-wlock-softirq.h"
GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_wlock)
+#endif
+
#undef E1
#undef E2
#undef E3
@@ -736,6 +751,8 @@ GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_wlock)
#include "locking-selftest-spin-hardirq.h"
GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_spin)
+#ifndef CONFIG_PREEMPT_RT_FULL
+
#include "locking-selftest-rlock-hardirq.h"
GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_rlock)
@@ -751,10 +768,14 @@ GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_soft_rlock)
#include "locking-selftest-wlock-softirq.h"
GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_soft_wlock)
+#endif
+
#undef E1
#undef E2
#undef E3
+#ifndef CONFIG_PREEMPT_RT_FULL
+
/*
* read-lock / write-lock irq inversion.
*
@@ -817,6 +838,10 @@ GENERATE_PERMUTATIONS_3_EVENTS(irq_inversion_soft_wlock)
#undef E2
#undef E3
+#endif
+
+#ifndef CONFIG_PREEMPT_RT_FULL
+
/*
* read-lock / write-lock recursion that is actually safe.
*/
@@ -855,6 +880,8 @@ GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft)
#undef E2
#undef E3
+#endif
+
/*
* read-lock / write-lock recursion that is unsafe.
*/
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 27/31] timers: Track total number of timers in list
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
` (24 preceding siblings ...)
2015-03-13 15:15 ` [PATCH RT 26/31] lockdep: selftest: fix warnings due to missing PREEMPT_RT conditionals Steven Rostedt
@ 2015-03-13 15:15 ` Steven Rostedt
2015-03-13 15:15 ` [PATCH RT 28/31] timers: Reduce __run_timers() latency for empty list Steven Rostedt
` (3 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker, Paul E. McKenney, Josh Triplett,
Peter Zijlstra, Oleg Nesterov, Mike Galbraith
[-- Attachment #1: 0027-timers-Track-total-number-of-timers-in-list.patch --]
[-- Type: text/plain, Size: 2259 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
upstream commit: fff421580f512fc044cc7421fdff31a7a6997350
Currently, the tvec_base structure's ->active_timers field tracks only
the non-deferrable timers, which means that even if ->active_timers is
zero, there might well be deferrable timers in the list. This commit
therefore adds an ->all_timers field to track all the timers, whether
deferrable or not.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
Acked-by: Peter Zijlstra <peterz@infradead.org>
Reviewed-by: Oleg Nesterov <oleg@redhat.com>
Reviewed-by: Steven Rostedt <rostedt@goodmis.org>
Tested-by: Mike Galbraith <bitbucket@online.de>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
kernel/timer.c | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/kernel/timer.c b/kernel/timer.c
index 40cfa14131ca..23919c7c212f 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -84,6 +84,7 @@ struct tvec_base {
unsigned long timer_jiffies;
unsigned long next_timer;
unsigned long active_timers;
+ unsigned long all_timers;
struct tvec_root tv1;
struct tvec tv2;
struct tvec tv3;
@@ -395,6 +396,7 @@ static void internal_add_timer(struct tvec_base *base, struct timer_list *timer)
base->next_timer = timer->expires;
base->active_timers++;
}
+ base->all_timers++;
}
#ifdef CONFIG_TIMER_STATS
@@ -674,6 +676,7 @@ detach_expired_timer(struct timer_list *timer, struct tvec_base *base)
detach_timer(timer, true);
if (!tbase_get_deferrable(timer->base))
base->active_timers--;
+ base->all_timers--;
}
static int detach_if_pending(struct timer_list *timer, struct tvec_base *base,
@@ -688,6 +691,7 @@ static int detach_if_pending(struct timer_list *timer, struct tvec_base *base,
if (timer->expires == base->next_timer)
base->next_timer = base->timer_jiffies;
}
+ base->all_timers--;
return 1;
}
@@ -1678,6 +1682,7 @@ static int __cpuinit init_timers_cpu(int cpu)
base->timer_jiffies = jiffies;
base->next_timer = base->timer_jiffies;
base->active_timers = 0;
+ base->all_timers = 0;
return 0;
}
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 28/31] timers: Reduce __run_timers() latency for empty list
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
` (25 preceding siblings ...)
2015-03-13 15:15 ` [PATCH RT 27/31] timers: Track total number of timers in list Steven Rostedt
@ 2015-03-13 15:15 ` Steven Rostedt
2015-03-13 15:15 ` [PATCH RT 29/31] timers: Reduce future __run_timers() latency for newly emptied list Steven Rostedt
` (2 subsequent siblings)
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker, Paul E. McKenney, Josh Triplett,
Peter Zijlstra, Oleg Nesterov, Mike Galbraith
[-- Attachment #1: 0028-timers-Reduce-__run_timers-latency-for-empty-list.patch --]
[-- Type: text/plain, Size: 2130 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
upstream commit: d550e81dc0ddc04f1b417c179c214103a28e0ee8
The __run_timers() function currently steps through the list one jiffy at
a time in order to update the timer wheel. However, if the timer wheel
is empty, no adjustment is needed other than updating ->timer_jiffies.
In this case, which is likely to be common for NO_HZ_FULL kernels, the
kernel currently incurs a large latency for no good reason. This commit
therefore short-circuits this case.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
Acked-by: Peter Zijlstra <peterz@infradead.org>
Reviewed-by: Oleg Nesterov <oleg@redhat.com>
Reviewed-by: Steven Rostedt <rostedt@goodmis.org>
Tested-by: Mike Galbraith <bitbucket@online.de>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
kernel/timer.c | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
diff --git a/kernel/timer.c b/kernel/timer.c
index 23919c7c212f..c20bd12a4f06 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -341,6 +341,20 @@ void set_timer_slack(struct timer_list *timer, int slack_hz)
}
EXPORT_SYMBOL_GPL(set_timer_slack);
+/*
+ * If the list is empty, catch up ->timer_jiffies to the current time.
+ * The caller must hold the tvec_base lock. Returns true if the list
+ * was empty and therefore ->timer_jiffies was updated.
+ */
+static bool catchup_timer_jiffies(struct tvec_base *base)
+{
+ if (!base->all_timers) {
+ base->timer_jiffies = jiffies;
+ return true;
+ }
+ return false;
+}
+
static void
__internal_add_timer(struct tvec_base *base, struct timer_list *timer)
{
@@ -1203,6 +1217,10 @@ static inline void __run_timers(struct tvec_base *base)
struct timer_list *timer;
spin_lock_irq(&base->lock);
+ if (catchup_timer_jiffies(base)) {
+ spin_unlock_irq(&base->lock);
+ return;
+ }
while (time_after_eq(jiffies, base->timer_jiffies)) {
struct list_head work_list;
struct list_head *head = &work_list;
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 29/31] timers: Reduce future __run_timers() latency for newly emptied list
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
` (26 preceding siblings ...)
2015-03-13 15:15 ` [PATCH RT 28/31] timers: Reduce __run_timers() latency for empty list Steven Rostedt
@ 2015-03-13 15:15 ` Steven Rostedt
2015-03-13 15:15 ` [PATCH RT 30/31] timers: Reduce future __run_timers() latency for first add to empty list Steven Rostedt
2015-03-13 15:15 ` [PATCH RT 31/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker, Paul E. McKenney, Josh Triplett,
Peter Zijlstra, Oleg Nesterov, Mike Galbraith
[-- Attachment #1: 0029-timers-Reduce-future-__run_timers-latency-for-newly-.patch --]
[-- Type: text/plain, Size: 1897 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
upstream commit: 16d937f880312e3f47157d4d6d6ebf7e61523378
The __run_timers() function currently steps through the list one jiffy at
a time in order to update the timer wheel. However, if the timer wheel
is empty, no adjustment is needed other than updating ->timer_jiffies.
Therefore, if we just emptied the timer wheel, for example, by deleting
the last timer, we should mark the timer wheel as being up to date.
This marking will reduce (and perhaps eliminate) the jiffy-stepping that
a future __run_timers() call will need to do in response to some future
timer posting or migration. This commit therefore catches ->timer_jiffies
for this case.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
Acked-by: Peter Zijlstra <peterz@infradead.org>
Reviewed-by: Oleg Nesterov <oleg@redhat.com>
Reviewed-by: Steven Rostedt <rostedt@goodmis.org>
Tested-by: Mike Galbraith <bitbucket@online.de>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
kernel/timer.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/kernel/timer.c b/kernel/timer.c
index c20bd12a4f06..39f5caef6714 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -691,6 +691,7 @@ detach_expired_timer(struct timer_list *timer, struct tvec_base *base)
if (!tbase_get_deferrable(timer->base))
base->active_timers--;
base->all_timers--;
+ (void)catchup_timer_jiffies(base);
}
static int detach_if_pending(struct timer_list *timer, struct tvec_base *base,
@@ -706,6 +707,7 @@ static int detach_if_pending(struct timer_list *timer, struct tvec_base *base,
base->next_timer = base->timer_jiffies;
}
base->all_timers--;
+ (void)catchup_timer_jiffies(base);
return 1;
}
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 30/31] timers: Reduce future __run_timers() latency for first add to empty list
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
` (27 preceding siblings ...)
2015-03-13 15:15 ` [PATCH RT 29/31] timers: Reduce future __run_timers() latency for newly emptied list Steven Rostedt
@ 2015-03-13 15:15 ` Steven Rostedt
2015-03-13 15:15 ` [PATCH RT 31/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker, Paul E. McKenney, Josh Triplett,
Peter Zijlstra, Oleg Nesterov, Mike Galbraith
[-- Attachment #1: 0030-timers-Reduce-future-__run_timers-latency-for-first-.patch --]
[-- Type: text/plain, Size: 1654 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
upstream commit: 18d8cb64c9c074cbe2bd677ab10fff8283abdb62
The __run_timers() function currently steps through the list one jiffy at
a time in order to update the timer wheel. However, if the timer wheel
is empty, no adjustment is needed other than updating ->timer_jiffies.
Therefore, just before we add a timer to an empty timer wheel, we should
mark the timer wheel as being up to date. This marking will reduce (and
perhaps eliminate) the jiffy-stepping that a future __run_timers() call
will need to do in response to some future timer posting or migration.
This commit therefore updates ->timer_jiffies for this case.
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Reviewed-by: Josh Triplett <josh@joshtriplett.org>
Acked-by: Peter Zijlstra <peterz@infradead.org>
Reviewed-by: Oleg Nesterov <oleg@redhat.com>
Reviewed-by: Steven Rostedt <rostedt@goodmis.org>
Tested-by: Mike Galbraith <bitbucket@online.de>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
---
kernel/timer.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/kernel/timer.c b/kernel/timer.c
index 39f5caef6714..06050920aee5 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -401,6 +401,7 @@ __internal_add_timer(struct tvec_base *base, struct timer_list *timer)
static void internal_add_timer(struct tvec_base *base, struct timer_list *timer)
{
+ (void)catchup_timer_jiffies(base);
__internal_add_timer(base, timer);
/*
* Update base->active_timers and base->next_timer
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread* [PATCH RT 31/31] Linux 3.10.70-rt75-rc1
2015-03-13 15:14 [PATCH RT 00/31] Linux 3.10.70-rt75-rc1 Steven Rostedt
` (28 preceding siblings ...)
2015-03-13 15:15 ` [PATCH RT 30/31] timers: Reduce future __run_timers() latency for first add to empty list Steven Rostedt
@ 2015-03-13 15:15 ` Steven Rostedt
29 siblings, 0 replies; 33+ messages in thread
From: Steven Rostedt @ 2015-03-13 15:15 UTC (permalink / raw)
To: linux-kernel, linux-rt-users
Cc: Thomas Gleixner, Carsten Emde, Sebastian Andrzej Siewior,
John Kacur, Paul Gortmaker
[-- Attachment #1: 0031-Linux-3.10.70-rt75-rc1.patch --]
[-- Type: text/plain, Size: 414 bytes --]
3.10.70-rt75-rc1 stable review patch.
If anyone has any objections, please let me know.
------------------
From: "Steven Rostedt (Red Hat)" <rostedt@goodmis.org>
---
localversion-rt | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/localversion-rt b/localversion-rt
index 7d028f4a9e56..5449cd22a524 100644
--- a/localversion-rt
+++ b/localversion-rt
@@ -1 +1 @@
--rt74
+-rt75-rc1
--
2.1.4
^ permalink raw reply related [flat|nested] 33+ messages in thread