Be more careful about semaphore contention memory ordering.

Don't touch the wakee stack after marking it runnable.

diff --git a/lib/rwsem-spinlock.c b/lib/rwsem-spinlock.c
index 84c2c4fc58dc8b52ebd1babbfdc87e53d28d259f..d6c0e14c272961c64f9f6b8455be50e15869cf1b 100644 (file)
--- a/lib/rwsem-spinlock.c
+++ b/lib/rwsem-spinlock.c
@@ -45,12 +45,13 @@ void fastcall init_rwsem(struct rw_semaphore *sem)
  *   - the 'active count' _reached_ zero
  *   - the 'waiting count' is non-zero
  * - the spinlock must be held by the caller
- * - woken process blocks are discarded from the list after having flags zeroised
+ * - woken process blocks are discarded from the list after having task zeroed
  * - writers are only woken if wakewrite is non-zero
  */
 static inline struct rw_semaphore *__rwsem_do_wake(struct rw_semaphore *sem, int wakewrite)
 {
        struct rwsem_waiter *waiter;
+       struct task_struct *tsk;
        int woken;
 
        rwsemtrace(sem,"Entering __rwsem_do_wake");
@@ -70,8 +71,10 @@ static inline struct rw_semaphore *__rwsem_do_wake(struct rw_semaphore *sem, int
        if (waiter->flags & RWSEM_WAITING_FOR_WRITE) {
                sem->activity = -1;
                list_del(&waiter->list);
-               waiter->flags = 0;
-               wake_up_process(waiter->task);
+               mb();
+               tsk = waiter->task;
+               waiter->task = NULL;
+               wake_up_process(tsk);
                goto out;
        }
 
@@ -82,8 +85,10 @@ static inline struct rw_semaphore *__rwsem_do_wake(struct rw_semaphore *sem, int
                struct list_head *next = waiter->list.next;
 
                list_del(&waiter->list);
-               waiter->flags = 0;
-               wake_up_process(waiter->task);
+               mb();
+               tsk = waiter->task;
+               waiter->task = NULL;
+               wake_up_process(tsk);
                woken++;
                if (list_empty(&sem->wait_list))
                        break;
@@ -103,14 +108,17 @@ static inline struct rw_semaphore *__rwsem_do_wake(struct rw_semaphore *sem, int
 static inline struct rw_semaphore *__rwsem_wake_one_writer(struct rw_semaphore *sem)
 {
        struct rwsem_waiter *waiter;
+       struct task_struct *tsk;
 
        sem->activity = -1;
 
        waiter = list_entry(sem->wait_list.next,struct rwsem_waiter,list);
        list_del(&waiter->list);
 
-       waiter->flags = 0;
-       wake_up_process(waiter->task);
+       mb();
+       tsk = waiter->task;
+       waiter->task = NULL;
+       wake_up_process(tsk);
        return sem;
 }
 
@@ -147,7 +155,7 @@ void fastcall __down_read(struct rw_semaphore *sem)
 
        /* wait to be given the lock */
        for (;;) {
-               if (!waiter.flags)
+               if (!waiter.task)
                        break;
                schedule();
                set_task_state(tsk, TASK_UNINTERRUPTIBLE);
@@ -215,7 +223,7 @@ void fastcall __down_write(struct rw_semaphore *sem)
 
        /* wait to be given the lock */
        for (;;) {
-               if (!waiter.flags)
+               if (!waiter.task)
                        break;
                schedule();
                set_task_state(tsk, TASK_UNINTERRUPTIBLE);

diff --git a/lib/rwsem.c b/lib/rwsem.c
index 3d8c4401c49c4324e9656039842313a744178558..7df79fdbe9ba7524172b79a684823ece89b6aaaa 100644 (file)
--- a/lib/rwsem.c
+++ b/lib/rwsem.c
@@ -34,12 +34,13 @@ void rwsemtrace(struct rw_semaphore *sem, const char *str)
  *   - the 'waiting part' of the count (&0xffff0000) is negative (and will still be so)
  *   - there must be someone on the queue
  * - the spinlock must be held by the caller
- * - woken process blocks are discarded from the list after having flags zeroised
+ * - woken process blocks are discarded from the list after having task zeroed
  * - writers are only woken if wakewrite is non-zero
  */
 static inline struct rw_semaphore *__rwsem_do_wake(struct rw_semaphore *sem, int wakewrite)
 {
        struct rwsem_waiter *waiter;
+       struct task_struct *tsk;
        struct list_head *next;
        signed long oldcount, woken, loop;
 
@@ -64,8 +65,10 @@ static inline struct rw_semaphore *__rwsem_do_wake(struct rw_semaphore *sem, int
                goto readers_only;
 
        list_del(&waiter->list);
-       waiter->flags = 0;
-       wake_up_process(waiter->task);
+       mb();
+       tsk = waiter->task;
+       waiter->task = NULL;
+       wake_up_process(tsk);
        goto out;
 
        /* don't want to wake any writers */
@@ -99,8 +102,10 @@ static inline struct rw_semaphore *__rwsem_do_wake(struct rw_semaphore *sem, int
        for (; loop>0; loop--) {
                waiter = list_entry(next,struct rwsem_waiter,list);
                next = waiter->list.next;
-               waiter->flags = 0;
-               wake_up_process(waiter->task);
+               mb();
+               tsk = waiter->task;
+               waiter->task = NULL;
+               wake_up_process(tsk);
        }
 
        sem->wait_list.next = next;
@@ -148,7 +153,7 @@ static inline struct rw_semaphore *rwsem_down_failed_common(struct rw_semaphore
 
        /* wait to be given the lock */
        for (;;) {
-               if (!waiter->flags)
+               if (!waiter->task)
                        break;
                schedule();
                set_task_state(tsk, TASK_UNINTERRUPTIBLE);