[PATCH] ia64: make hugetlbfs page size a boot-time option

This patch makes it possible to select the hugetlbfs page size at boot-time,
rather than at compile-time.  We believe we have resolved all the remaining
issues.  All critical speed path has been taken care of, i.e., vhpt handler
and context switch.  There should be no performance penalty with this dynamic
hugetlb page size feature.

We would like to thank Jack Steiner for his initiative on this feature
and his initial cool working patch.

diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig
index 97833363f7a2ff13078098e7775c2d25352523a3..53bf17211c795da69d8329521b67b2477b753191 100644 (file)
--- a/arch/ia64/Kconfig
+++ b/arch/ia64/Kconfig
@@ -288,39 +288,6 @@ config FORCE_MAX_ZONEORDER
        int
        default "18"
 
-choice
-       prompt "Huge TLB page size"
-       depends on HUGETLB_PAGE
-       default HUGETLB_PAGE_SIZE_16MB
-
-config HUGETLB_PAGE_SIZE_4GB
-       depends on MCKINLEY
-       bool "4GB"
-
-config HUGETLB_PAGE_SIZE_1GB
-       depends on MCKINLEY
-       bool "1GB"
-
-config HUGETLB_PAGE_SIZE_256MB
-       bool "256MB"
-
-config HUGETLB_PAGE_SIZE_64MB
-       bool "64MB"
-
-config HUGETLB_PAGE_SIZE_16MB
-       bool "16MB"
-
-config HUGETLB_PAGE_SIZE_4MB
-       bool "4MB"
-
-config HUGETLB_PAGE_SIZE_1MB
-       bool "1MB"
-
-config HUGETLB_PAGE_SIZE_256KB
-       bool "256KB"
-
-endchoice
-
 config IA64_PAL_IDLE
        bool "Use PAL_HALT_LIGHT in idle loop"
        help

diff --git a/arch/ia64/kernel/ivt.S b/arch/ia64/kernel/ivt.S
index 8aed612cb284fe6fd14b705e3fb933ae754a6244..c65b6bfb29beb67304e42df39e2fd97357f5c660 100644 (file)
--- a/arch/ia64/kernel/ivt.S
+++ b/arch/ia64/kernel/ivt.S
@@ -118,10 +118,11 @@ ENTRY(vhpt_miss)
 #ifdef CONFIG_HUGETLB_PAGE
        extr.u r26=r25,2,6
        ;;
-       cmp.eq p8,p0=HPAGE_SHIFT,r26
+       cmp.ne p8,p0=r18,r26
+       sub r27=r26,r18
        ;;
 (p8)   dep r25=r18,r25,2,6
-(p8)   shr r22=r22,HPAGE_SHIFT-PAGE_SHIFT
+(p8)   shr r22=r22,r27
 #endif
        ;;
        cmp.eq p6,p7=5,r17                      // is IFA pointing into to region 5?

diff --git a/arch/ia64/mm/hugetlbpage.c b/arch/ia64/mm/hugetlbpage.c
index f343f7067f160844c300e31021e4fde2f4d62a5b..84db44246b448961a190d24247a6ccc1a2c20da4 100644 (file)
--- a/arch/ia64/mm/hugetlbpage.c
+++ b/arch/ia64/mm/hugetlbpage.c
@@ -1,7 +1,11 @@
 /*
  * IA-64 Huge TLB Page Support for Kernel.
  *
- * Copyright (C) 2002, Rohit Seth <rohit.seth@intel.com>
+ * Copyright (C) 2002-2004 Rohit Seth <rohit.seth@intel.com>
+ * Copyright (C) 2003-2004 Ken Chen <kenneth.w.chen@intel.com>
+ *
+ * Sep, 2003: add numa support
+ * Feb, 2004: dynamic hugetlb page size via boot parameter
  */
 
 #include <linux/config.h>
@@ -18,11 +22,10 @@
 #include <asm/tlb.h>
 #include <asm/tlbflush.h>
 
-#define TASK_HPAGE_BASE (REGION_HPAGE << REGION_SHIFT)
-
 static long    htlbpagemem;
 int            htlbpage_max;
 static long    htlbzone_pages;
+unsigned int   hpage_shift=HPAGE_SHIFT_DEFAULT;
 
 static struct list_head hugepage_freelists[MAX_NUMNODES];
 static spinlock_t htlbpage_lock = SPIN_LOCK_UNLOCKED;
@@ -407,7 +410,7 @@ unsigned long hugetlb_get_unmapped_area(struct file *file, unsigned long addr, u
                return -EINVAL;
        /* This code assumes that REGION_HPAGE != 0. */
        if ((REGION_NUMBER(addr) != REGION_HPAGE) || (addr & (HPAGE_SIZE - 1)))
-               addr = TASK_HPAGE_BASE;
+               addr = HPAGE_REGION_BASE;
        else
                addr = ALIGN(addr, HPAGE_SIZE);
        for (vmm = find_vma(current->mm, addr); ; vmm = vmm->vm_next) {
@@ -520,6 +523,35 @@ static int __init hugetlb_setup(char *s)
 }
 __setup("hugepages=", hugetlb_setup);
 
+static int __init hugetlb_setup_sz(char *str)
+{
+       u64 tr_pages;
+       unsigned long long size;
+
+       if (ia64_pal_vm_page_size(&tr_pages, NULL) != 0)
+               /*
+                * shouldn't happen, but just in case.
+                */
+               tr_pages = 0x15557000UL;
+
+       size = memparse(str, &str);
+       if (*str || (size & (size-1)) || !(tr_pages & size) ||
+               size <= PAGE_SIZE ||
+               size >= (1UL << PAGE_SHIFT << MAX_ORDER)) {
+               printk(KERN_WARNING "Invalid huge page size specified\n");
+               return 1;
+       }
+
+       hpage_shift = __ffs(size);
+       /*
+        * boot cpu already executed ia64_mmu_init, and has HPAGE_SHIFT_DEFAULT
+        * override here with new page shift.
+        */
+       ia64_set_rr(HPAGE_REGION_BASE, hpage_shift << 2);
+       return 1;
+}
+__setup("hugepagesz=", hugetlb_setup_sz);
+
 static int __init hugetlb_init(void)
 {
        int i;
@@ -540,7 +572,7 @@ static int __init hugetlb_init(void)
        printk("Total HugeTLB memory allocated, %ld\n", htlbpagemem);
        return 0;
 }
-module_init(hugetlb_init);
+__initcall(hugetlb_init);
 
 int hugetlb_report_meminfo(char *buf)
 {

diff --git a/arch/ia64/mm/init.c b/arch/ia64/mm/init.c
index 86ded838902f723e8d9b016ff6aacabea1daeaa7..e4c573d743bac56eb809896261abb359442deb29 100644 (file)
--- a/arch/ia64/mm/init.c
+++ b/arch/ia64/mm/init.c
@@ -342,6 +342,10 @@ ia64_mmu_init (void *my_cpu_data)
 
        ia64_tlb_init();
 
+#ifdef CONFIG_HUGETLB_PAGE
+       ia64_set_rr(HPAGE_REGION_BASE, HPAGE_SHIFT << 2);
+#endif
+
 #ifdef CONFIG_IA64_MCA
        cpu = smp_processor_id();
 

diff --git a/include/asm-ia64/mmu_context.h b/include/asm-ia64/mmu_context.h
index 41d65da53e859c57904b3339a0730a60eb9a59dd..51558c0f024368be9c623f4d281a9f5e90bbd1ad 100644 (file)
--- a/include/asm-ia64/mmu_context.h
+++ b/include/asm-ia64/mmu_context.h
@@ -140,8 +140,9 @@ reload_context (mm_context_t context)
 {
        unsigned long rid;
        unsigned long rid_incr = 0;
-       unsigned long rr0, rr1, rr2, rr3, rr4;
+       unsigned long rr0, rr1, rr2, rr3, rr4, old_rr4;
 
+       old_rr4 = ia64_get_rr(0x8000000000000000);
        rid = context << 3;     /* make space for encoding the region number */
        rid_incr = 1 << 8;
 
@@ -152,7 +153,7 @@ reload_context (mm_context_t context)
        rr3 = rr0 + 3*rid_incr;
        rr4 = rr0 + 4*rid_incr;
 #ifdef  CONFIG_HUGETLB_PAGE
-       rr4 = (rr4 & (~(0xfcUL))) | (HPAGE_SHIFT << 2);
+       rr4 = (rr4 & (~(0xfcUL))) | (old_rr4 & 0xfc);
 #endif
 
        ia64_set_rr(0x0000000000000000, rr0);

diff --git a/include/asm-ia64/page.h b/include/asm-ia64/page.h
index 3bf82809aa4fdcffe7e0a4db7fe361526624177b..58cc7448242e48ad0575e06bc85c7fe232ccc7f0 100644 (file)
--- a/include/asm-ia64/page.h
+++ b/include/asm-ia64/page.h
@@ -37,31 +37,14 @@
 #define RGN_MAP_LIMIT  ((1UL << (4*PAGE_SHIFT - 12)) - PAGE_SIZE)      /* per region addr limit */
 
 #ifdef CONFIG_HUGETLB_PAGE
+# define REGION_HPAGE          (4UL)   /* note: this is hardcoded in reload_context()!*/
+# define REGION_SHIFT          61
+# define HPAGE_REGION_BASE     (REGION_HPAGE << REGION_SHIFT)
+# define HPAGE_SHIFT           hpage_shift
+# define HPAGE_SHIFT_DEFAULT   28      /* check ia64 SDM for architecture supported size */
+# define HPAGE_SIZE            (__IA64_UL_CONST(1) << HPAGE_SHIFT)
+# define HPAGE_MASK            (~(HPAGE_SIZE - 1))
 
-# if defined(CONFIG_HUGETLB_PAGE_SIZE_4GB)
-#  define HPAGE_SHIFT  32
-# elif defined(CONFIG_HUGETLB_PAGE_SIZE_1GB)
-#  define HPAGE_SHIFT  30
-# elif defined(CONFIG_HUGETLB_PAGE_SIZE_256MB)
-#  define HPAGE_SHIFT  28
-# elif defined(CONFIG_HUGETLB_PAGE_SIZE_64MB)
-#  define HPAGE_SHIFT  26
-# elif defined(CONFIG_HUGETLB_PAGE_SIZE_16MB)
-#  define HPAGE_SHIFT  24
-# elif defined(CONFIG_HUGETLB_PAGE_SIZE_4MB)
-#  define HPAGE_SHIFT  22
-# elif defined(CONFIG_HUGETLB_PAGE_SIZE_1MB)
-#  define HPAGE_SHIFT  20
-# elif defined(CONFIG_HUGETLB_PAGE_SIZE_256KB)
-#  define HPAGE_SHIFT  18
-# else
-#  error Unsupported IA-64 HugeTLB Page Size!
-# endif
-
-# define REGION_HPAGE  (4UL)   /* note: this is hardcoded in mmu_context.h:reload_context()!*/
-# define REGION_SHIFT  61
-# define HPAGE_SIZE    (__IA64_UL_CONST(1) << HPAGE_SHIFT)
-# define HPAGE_MASK    (~(HPAGE_SIZE - 1))
 # define HAVE_ARCH_HUGETLB_UNMAPPED_AREA
 # define ARCH_HAS_HUGEPAGE_ONLY_RANGE
 #endif /* CONFIG_HUGETLB_PAGE */
@@ -140,6 +123,7 @@ typedef union ia64_va {
 # define is_hugepage_only_range(addr, len)             \
         (REGION_NUMBER(addr) == REGION_HPAGE &&        \
          REGION_NUMBER((addr)+(len)) == REGION_HPAGE)
+extern unsigned int hpage_shift;
 #endif
 
 static __inline__ int