[PATCH] hashed b_wait

Implements hashed waitqueues for buffer_heads.  Drops twelve bytes from
struct buffer_head.

diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index 5e97c4e07899004996d6d08132e9be79f3489060..7a2f950cf1fc0cdfff706fc23c567df0cd251c5f 100644 (file)
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -166,7 +166,6 @@ static int grow_buffers(struct stripe_head *sh, int num, int b_size, int priorit
                if (!bh)
                        return 1;
                memset(bh, 0, sizeof (struct buffer_head));
-               init_waitqueue_head(&bh->b_wait);
                if ((page = alloc_page(priority)))
                        bh->b_data = page_address(page);
                else {

diff --git a/fs/buffer.c b/fs/buffer.c
index 52abaa214a5e380aa03b8e9b9588bedb3a741177..273ea1b3e54b44d93aff822398cfd63158949c12 100644 (file)
--- a/fs/buffer.c
+++ b/fs/buffer.c
@@ -19,6 +19,7 @@
  */
 
 #include <linux/config.h>
+#include <linux/kernel.h>
 #include <linux/fs.h>
 #include <linux/mm.h>
 #include <linux/slab.h>
@@ -31,6 +32,7 @@
 #include <linux/module.h>
 #include <linux/writeback.h>
 #include <linux/mempool.h>
+#include <linux/hash.h>
 #include <asm/bitops.h>
 
 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_inode_buffers)
@@ -38,6 +40,14 @@
 /* This is used by some architectures to estimate available memory. */
 atomic_t buffermem_pages = ATOMIC_INIT(0);
 
+/*
+ * Hashed waitqueue_head's for wait_on_buffer()
+ */
+#define BH_WAIT_TABLE_ORDER    7
+static struct bh_wait_queue_head {
+       wait_queue_head_t wqh;
+} ____cacheline_aligned_in_smp bh_wait_queue_heads[1<<BH_WAIT_TABLE_ORDER];
+
 /*
  * Several of these buffer list functions are exported to filesystems,
  * so we do funny things with the spinlocking to support those
@@ -76,6 +86,35 @@ init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
        bh->b_private = private;
 }
 
+/*
+ * Return the address of the waitqueue_head to be used for this
+ * buffer_head
+ */
+static wait_queue_head_t *bh_waitq_head(struct buffer_head *bh)
+{
+       return &bh_wait_queue_heads[hash_ptr(bh, BH_WAIT_TABLE_ORDER)].wqh;
+}
+
+/*
+ * Wait on a buffer until someone does a wakeup on it.  Needs
+ * lots of external locking.  ext3 uses this.  Fix it.
+ */
+void sleep_on_buffer(struct buffer_head *bh)
+{
+       wait_queue_head_t *wq = bh_waitq_head(bh);
+       sleep_on(wq);
+}
+EXPORT_SYMBOL(sleep_on_buffer);
+
+void wake_up_buffer(struct buffer_head *bh)
+{
+       wait_queue_head_t *wq = bh_waitq_head(bh);
+
+       if (waitqueue_active(wq))
+               wake_up_all(wq);
+}
+EXPORT_SYMBOL(wake_up_buffer);
+
 void unlock_buffer(struct buffer_head *bh)
 {
        /*
@@ -89,8 +128,32 @@ void unlock_buffer(struct buffer_head *bh)
 
        clear_buffer_locked(bh);
        smp_mb__after_clear_bit();
-       if (waitqueue_active(&bh->b_wait))
-               wake_up(&bh->b_wait);
+       wake_up_buffer(bh);
+}
+
+/*
+ * Block until a buffer comes unlocked.  This doesn't stop it
+ * from becoming locked again - you have to lock it yourself
+ * if you want to preserve its state.
+ */
+void __wait_on_buffer(struct buffer_head * bh)
+{
+       wait_queue_head_t *wq = bh_waitq_head(bh);
+       struct task_struct *tsk = current;
+       DECLARE_WAITQUEUE(wait, tsk);
+
+       get_bh(bh);
+       add_wait_queue(wq, &wait);
+       do {
+               run_task_queue(&tq_disk);
+               set_task_state(tsk, TASK_UNINTERRUPTIBLE);
+               if (!buffer_locked(bh))
+                       break;
+               schedule();
+       } while (buffer_locked(bh));
+       tsk->state = TASK_RUNNING;
+       remove_wait_queue(wq, &wait);
+       put_bh(bh);
 }
 
 static inline void
@@ -121,30 +184,6 @@ __clear_page_buffers(struct page *page)
        page_cache_release(page);
 }
 
-/*
- * Block until a buffer comes unlocked.  This doesn't stop it
- * from becoming locked again - you have to lock it yourself
- * if you want to preserve its state.
- */
-void __wait_on_buffer(struct buffer_head * bh)
-{
-       struct task_struct *tsk = current;
-       DECLARE_WAITQUEUE(wait, tsk);
-
-       get_bh(bh);
-       add_wait_queue(&bh->b_wait, &wait);
-       do {
-               run_task_queue(&tq_disk);
-               set_task_state(tsk, TASK_UNINTERRUPTIBLE);
-               if (!buffer_locked(bh))
-                       break;
-               schedule();
-       } while (buffer_locked(bh));
-       tsk->state = TASK_RUNNING;
-       remove_wait_queue(&bh->b_wait, &wait);
-       put_bh(bh);
-}
-
 /*
  * Default synchronous end-of-IO handler..  Just mark it up-to-date and
  * unlock the buffer. This is what ll_rw_block uses too.
@@ -2265,7 +2304,6 @@ static void init_buffer_head(void *data, kmem_cache_t *cachep, unsigned long fla
                memset(bh, 0, sizeof(*bh));
                bh->b_blocknr = -1;
                INIT_LIST_HEAD(&bh->b_inode_buffers);
-               init_waitqueue_head(&bh->b_wait);
        }
 }
 
@@ -2284,9 +2322,13 @@ static void bh_mempool_free(void *element, void *pool_data)
 
 void __init buffer_init(void)
 {
+       int i;
+
        bh_cachep = kmem_cache_create("buffer_head",
                        sizeof(struct buffer_head), 0,
                        SLAB_HWCACHE_ALIGN, init_buffer_head, NULL);
        bh_mempool = mempool_create(MAX_UNUSED_BUFFERS, bh_mempool_alloc,
                                bh_mempool_free, NULL);
+       for (i = 0; i < ARRAY_SIZE(bh_wait_queue_heads); i++)
+               init_waitqueue_head(&bh_wait_queue_heads[i].wqh);
 }

diff --git a/fs/jbd/commit.c b/fs/jbd/commit.c
index 1e8307671dc9fc4de47399ad253450c18e3a8423..3fb33ea0f3d5f86ada5211a499b41c8bd58de29d 100644 (file)
--- a/fs/jbd/commit.c
+++ b/fs/jbd/commit.c
@@ -530,7 +530,7 @@ start_journal_io:
                journal_file_buffer(jh, commit_transaction, BJ_Forget);
                /* Wake up any transactions which were waiting for this
                   IO to complete */
-               wake_up(&bh->b_wait);
+               wake_up_buffer(bh);
                JBUFFER_TRACE(jh, "brelse shadowed buffer");
                __brelse(bh);
        }

diff --git a/fs/jbd/transaction.c b/fs/jbd/transaction.c
index 64b5b0434bfa39c9d7f4f5230a0bf4a8b654f554..d0a1518729e5f344e571a389c7be93fbe4e9cb7a 100644 (file)
--- a/fs/jbd/transaction.c
+++ b/fs/jbd/transaction.c
@@ -654,7 +654,7 @@ repeat:
                        spin_unlock(&journal_datalist_lock);
                        unlock_journal(journal);
                        /* commit wakes up all shadow buffers after IO */
-                       sleep_on(&jh2bh(jh)->b_wait);
+                       sleep_on_buffer(jh2bh(jh));
                        lock_journal(journal);
                        goto repeat;
                }

diff --git a/include/linux/buffer_head.h b/include/linux/buffer_head.h
index daf9b1c6d7d19409063622d4d262d0140724c10f..20a586b730cac9f06bc314b708b06918d9ebe891 100644 (file)
--- a/include/linux/buffer_head.h
+++ b/include/linux/buffer_head.h
@@ -56,9 +56,6 @@ struct buffer_head {
        char * b_data;                  /* pointer to data block */
        bh_end_io_t *b_end_io;          /* I/O completion */
        void *b_private;                /* reserved for b_end_io */
-
-       wait_queue_head_t b_wait;
-
        struct list_head     b_inode_buffers; /* list of inode dirty buffers */
 };
 
@@ -166,6 +163,8 @@ void invalidate_bdev(struct block_device *, int);
 void __invalidate_buffers(kdev_t dev, int);
 int sync_buffers(struct block_device *, int);
 void __wait_on_buffer(struct buffer_head *);
+void sleep_on_buffer(struct buffer_head *bh);
+void wake_up_buffer(struct buffer_head *bh);
 int fsync_dev(kdev_t);
 int fsync_bdev(struct block_device *);
 int fsync_super(struct super_block *);