From c8712aebdb2455a095d596c1a5a6af358b3efde3 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Mon, 17 Jun 2002 19:43:55 -0700 Subject: [PATCH] [PATCH] change_page_attr and AGP update Add change_page_attr to change page attributes for the kernel linear map. Fix AGP driver to use change_page_attr for the AGP buffer. Clean up AGP driver a bit (only tested on i386/VIA+AMD) Change ioremap_nocache to use change_page_attr to avoid mappings with conflicting caching attributes. --- arch/i386/mm/Makefile | 3 +- arch/i386/mm/ioremap.c | 69 ++++++++++- arch/i386/mm/pageattr.c | 197 ++++++++++++++++++++++++++++++ drivers/char/agp/agp.h | 4 +- drivers/char/agp/agpgart_be.c | 155 +++++++++-------------- include/asm-alpha/agp.h | 11 ++ include/asm-i386/agp.h | 23 ++++ include/asm-i386/cacheflush.h | 3 + include/asm-i386/io.h | 26 +--- include/asm-i386/page.h | 3 + include/asm-i386/pgtable-2level.h | 1 + include/asm-i386/pgtable-3level.h | 2 + include/asm-i386/pgtable.h | 3 + include/asm-ia64/agp.h | 11 ++ include/asm-sparc64/agp.h | 11 ++ include/asm-x86_64/agp.h | 23 ++++ include/asm-x86_64/cacheflush.h | 3 + include/linux/vmalloc.h | 3 + mm/vmalloc.c | 28 +++-- 19 files changed, 443 insertions(+), 136 deletions(-) create mode 100644 arch/i386/mm/pageattr.c create mode 100644 include/asm-alpha/agp.h create mode 100644 include/asm-i386/agp.h create mode 100644 include/asm-ia64/agp.h create mode 100644 include/asm-sparc64/agp.h create mode 100644 include/asm-x86_64/agp.h diff --git a/arch/i386/mm/Makefile b/arch/i386/mm/Makefile index 73e25bd3022a..67df8b6f6594 100644 --- a/arch/i386/mm/Makefile +++ b/arch/i386/mm/Makefile @@ -9,6 +9,7 @@ O_TARGET := mm.o -obj-y := init.o fault.o ioremap.o extable.o +obj-y := init.o fault.o ioremap.o extable.o pageattr.o +export-objs := pageattr.o include $(TOPDIR)/Rules.make diff --git a/arch/i386/mm/ioremap.c b/arch/i386/mm/ioremap.c index f81fae4ff7a9..4ba5641b271f 100644 --- a/arch/i386/mm/ioremap.c +++ b/arch/i386/mm/ioremap.c @@ -10,12 +10,13 @@ #include #include +#include #include #include #include #include #include - +#include static inline void remap_area_pte(pte_t * pte, unsigned long address, unsigned long size, unsigned long phys_addr, unsigned long flags) @@ -155,6 +156,7 @@ void * __ioremap(unsigned long phys_addr, unsigned long size, unsigned long flag area = get_vm_area(size, VM_IOREMAP); if (!area) return NULL; + area->phys_addr = phys_addr; addr = area->addr; if (remap_area_pages(VMALLOC_VMADDR(addr), phys_addr, size, flags)) { vfree(addr); @@ -163,10 +165,71 @@ void * __ioremap(unsigned long phys_addr, unsigned long size, unsigned long flag return (void *) (offset + (char *)addr); } + +/** + * ioremap_nocache - map bus memory into CPU space + * @offset: bus address of the memory + * @size: size of the resource to map + * + * ioremap_nocache performs a platform specific sequence of operations to + * make bus memory CPU accessible via the readb/readw/readl/writeb/ + * writew/writel functions and the other mmio helpers. The returned + * address is not guaranteed to be usable directly as a virtual + * address. + * + * This version of ioremap ensures that the memory is marked uncachable + * on the CPU as well as honouring existing caching rules from things like + * the PCI bus. Note that there are other caches and buffers on many + * busses. In particular driver authors should read up on PCI writes + * + * It's useful if some control registers are in such an area and + * write combining or read caching is not desirable: + * + * Must be freed with iounmap. + */ + +void *ioremap_nocache (unsigned long phys_addr, unsigned long size) +{ + void *p = __ioremap(phys_addr, size, _PAGE_PCD); + if (!p) + return p; + + if (phys_addr + size < virt_to_phys(high_memory)) { + struct page *ppage = virt_to_page(__va(phys_addr)); + unsigned long npages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; + + BUG_ON(phys_addr+size > (unsigned long)high_memory); + BUG_ON(phys_addr + size < phys_addr); + + if (change_page_attr(ppage, npages, PAGE_KERNEL_NOCACHE) < 0) { + iounmap(p); + p = NULL; + } + } + + return p; +} + void iounmap(void *addr) { - if (addr > high_memory) - return vfree((void *) (PAGE_MASK & (unsigned long) addr)); + struct vm_struct *p; + if (addr < high_memory) + return; + p = remove_kernel_area(addr); + if (!p) { + printk("__iounmap: bad address %p\n", addr); + return; + } + + BUG_ON(p->phys_addr == 0); /* not allocated with ioremap */ + + vmfree_area_pages(VMALLOC_VMADDR(p->addr), p->size); + if (p->flags && p->phys_addr < virt_to_phys(high_memory)) { + change_page_attr(virt_to_page(__va(p->phys_addr)), + p->size >> PAGE_SHIFT, + PAGE_KERNEL); + } + kfree(p); } void __init *bt_ioremap(unsigned long phys_addr, unsigned long size) diff --git a/arch/i386/mm/pageattr.c b/arch/i386/mm/pageattr.c new file mode 100644 index 000000000000..c5e2374b6bc7 --- /dev/null +++ b/arch/i386/mm/pageattr.c @@ -0,0 +1,197 @@ +/* + * Copyright 2002 Andi Kleen, SuSE Labs. + * Thanks to Ben LaHaise for precious feedback. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +static inline pte_t *lookup_address(unsigned long address) +{ + pgd_t *pgd = pgd_offset_k(address); + pmd_t *pmd = pmd_offset(pgd, address); + if (pmd_large(*pmd)) + return (pte_t *)pmd; + return pte_offset_kernel(pmd, address); +} + +static struct page *split_large_page(unsigned long address, pgprot_t prot) +{ + int i; + unsigned long addr; + struct page *base = alloc_pages(GFP_KERNEL, 0); + pte_t *pbase; + if (!base) + return NULL; + address = __pa(address); + addr = address & LARGE_PAGE_MASK; + pbase = (pte_t *)page_address(base); + for (i = 0; i < PTRS_PER_PTE; i++, addr += PAGE_SIZE) { + pbase[i] = pfn_pte(addr >> PAGE_SHIFT, + addr == address ? prot : PAGE_KERNEL); + } + return base; +} + +static void flush_kernel_map(void *dummy) +{ + /* Could use CLFLUSH here if the CPU supports it (Hammer,P4) */ + if (boot_cpu_data.x86_model >= 4) + asm volatile("wbinvd":::"memory"); + /* Flush all to work around Errata in early athlons regarding + * large page flushing. + */ + __flush_tlb_all(); +} + +static void set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte) +{ + set_pte_atomic(kpte, pte); /* change init_mm */ +#ifndef CONFIG_X86_PAE + { + struct list_head *l; + spin_lock(&mmlist_lock); + list_for_each(l, &init_mm.mmlist) { + struct mm_struct *mm = list_entry(l, struct mm_struct, mmlist); + pmd_t *pmd = pmd_offset(pgd_offset(mm, address), address); + set_pte_atomic((pte_t *)pmd, pte); + } + spin_unlock(&mmlist_lock); + } +#endif +} + +/* + * No more special protections in this 2/4MB area - revert to a + * large page again. + */ +static inline void revert_page(struct page *kpte_page, unsigned long address) +{ + pte_t *linear = (pte_t *) + pmd_offset(pgd_offset(&init_mm, address), address); + set_pmd_pte(linear, address, + pfn_pte((__pa(address) & LARGE_PAGE_MASK) >> PAGE_SHIFT, + PAGE_KERNEL_LARGE)); +} + +static int +__change_page_attr(struct page *page, pgprot_t prot, struct page **oldpage) +{ + pte_t *kpte; + unsigned long address; + struct page *kpte_page; + +#ifdef CONFIG_HIGHMEM + if (page >= highmem_start_page) + BUG(); +#endif + address = (unsigned long)page_address(page); + + kpte = lookup_address(address); + kpte_page = virt_to_page(((unsigned long)kpte) & PAGE_MASK); + if (pgprot_val(prot) != pgprot_val(PAGE_KERNEL)) { + if ((pte_val(*kpte) & _PAGE_PSE) == 0) { + pte_t old = *kpte; + pte_t standard = mk_pte(page, PAGE_KERNEL); + + set_pte_atomic(kpte, mk_pte(page, prot)); + if (pte_same(old,standard)) + atomic_inc(&kpte_page->count); + } else { + struct page *split = split_large_page(address, prot); + if (!split) + return -ENOMEM; + set_pmd_pte(kpte,address,mk_pte(split, PAGE_KERNEL)); + } + } else if ((pte_val(*kpte) & _PAGE_PSE) == 0) { + set_pte_atomic(kpte, mk_pte(page, PAGE_KERNEL)); + atomic_dec(&kpte_page->count); + } + + if (cpu_has_pse && (atomic_read(&kpte_page->count) == 1)) { + *oldpage = kpte_page; + revert_page(kpte_page, address); + } + return 0; +} + +static inline void flush_map(void) +{ +#ifdef CONFIG_SMP + smp_call_function(flush_kernel_map, NULL, 1, 1); +#endif + flush_kernel_map(NULL); +} + +struct deferred_page { + struct deferred_page *next; + struct page *fpage; +}; +static struct deferred_page *df_list; /* protected by init_mm.mmap_sem */ + +/* + * Change the page attributes of an page in the linear mapping. + * + * This should be used when a page is mapped with a different caching policy + * than write-back somewhere - some CPUs do not like it when mappings with + * different caching policies exist. This changes the page attributes of the + * in kernel linear mapping too. + * + * The caller needs to ensure that there are no conflicting mappings elsewhere. + * This function only deals with the kernel linear map. + * + * Caller must call global_flush_tlb() after this. + */ +int change_page_attr(struct page *page, int numpages, pgprot_t prot) +{ + int err = 0; + struct page *fpage; + int i; + + down_write(&init_mm.mmap_sem); + for (i = 0; i < numpages; i++, page++) { + fpage = NULL; + err = __change_page_attr(page, prot, &fpage); + if (err) + break; + if (fpage) { + struct deferred_page *df; + df = kmalloc(sizeof(struct deferred_page), GFP_KERNEL); + if (!df) { + flush_map(); + __free_page(fpage); + } else { + df->next = df_list; + df->fpage = fpage; + df_list = df; + } + } + } + up_write(&init_mm.mmap_sem); + return err; +} + +void global_flush_tlb(void) +{ + struct deferred_page *df, *next_df; + + down_read(&init_mm.mmap_sem); + df = xchg(&df_list, NULL); + up_read(&init_mm.mmap_sem); + flush_map(); + for (; df; df = next_df) { + next_df = df->next; + if (df->fpage) + __free_page(df->fpage); + kfree(df); + } +} + +EXPORT_SYMBOL(change_page_attr); +EXPORT_SYMBOL(global_flush_tlb); diff --git a/drivers/char/agp/agp.h b/drivers/char/agp/agp.h index be8178161e80..94e405104df4 100644 --- a/drivers/char/agp/agp.h +++ b/drivers/char/agp/agp.h @@ -118,8 +118,8 @@ struct agp_bridge_data { int (*remove_memory) (agp_memory *, off_t, int); agp_memory *(*alloc_by_type) (size_t, int); void (*free_by_type) (agp_memory *); - unsigned long (*agp_alloc_page) (void); - void (*agp_destroy_page) (unsigned long); + void *(*agp_alloc_page) (void); + void (*agp_destroy_page) (void *); int (*suspend)(void); void (*resume)(void); diff --git a/drivers/char/agp/agpgart_be.c b/drivers/char/agp/agpgart_be.c index 10cc178c4d89..8ba761695215 100644 --- a/drivers/char/agp/agpgart_be.c +++ b/drivers/char/agp/agpgart_be.c @@ -22,6 +22,8 @@ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. * + * TODO: + * - Allocate more than order 0 pages to avoid too much linear map splitting. */ #include #include @@ -43,6 +45,7 @@ #include #include #include +#include #include #include "agp.h" @@ -59,56 +62,28 @@ EXPORT_SYMBOL(agp_enable); EXPORT_SYMBOL(agp_backend_acquire); EXPORT_SYMBOL(agp_backend_release); -static void flush_cache(void); - static struct agp_bridge_data agp_bridge; static int agp_try_unsupported __initdata = 0; - -static inline void flush_cache(void) -{ -#if defined(__i386__) || defined(__x86_64__) - asm volatile ("wbinvd":::"memory"); -#elif defined(__alpha__) || defined(__ia64__) || defined(__sparc__) - /* ??? I wonder if we'll really need to flush caches, or if the - core logic can manage to keep the system coherent. The ARM - speaks only of using `cflush' to get things in memory in - preparation for power failure. - - If we do need to call `cflush', we'll need a target page, - as we can only flush one page at a time. - - Ditto for IA-64. --davidm 00/08/07 */ - mb(); -#else -#error "Please define flush_cache." -#endif -} - #ifdef CONFIG_SMP -static atomic_t cpus_waiting; - static void ipi_handler(void *null) { - flush_cache(); - atomic_dec(&cpus_waiting); - while (atomic_read(&cpus_waiting) > 0) - barrier(); + flush_agp_cache(); } static void smp_flush_cache(void) { - atomic_set(&cpus_waiting, smp_num_cpus - 1); - if (smp_call_function(ipi_handler, NULL, 1, 0) != 0) + if (smp_call_function(ipi_handler, NULL, 1, 1) != 0) panic(PFX "timed out waiting for the other CPUs!\n"); - flush_cache(); - while (atomic_read(&cpus_waiting) > 0) - barrier(); + flush_agp_cache(); } #define global_cache_flush smp_flush_cache #else /* CONFIG_SMP */ -#define global_cache_flush flush_cache -#endif /* CONFIG_SMP */ +static void global_cache_flush(void) +{ + flush_agp_cache(); +} +#endif /* !CONFIG_SMP */ int agp_backend_acquire(void) { @@ -208,8 +183,7 @@ void agp_free_memory(agp_memory * curr) if (curr->page_count != 0) { for (i = 0; i < curr->page_count; i++) { curr->memory[i] &= ~(0x00000fff); - agp_bridge.agp_destroy_page((unsigned long) - phys_to_virt(curr->memory[i])); + agp_bridge.agp_destroy_page(phys_to_virt(curr->memory[i])); } } agp_free_key(curr->key); @@ -252,21 +226,22 @@ agp_memory *agp_allocate_memory(size_t page_count, u32 type) MOD_DEC_USE_COUNT; return NULL; } + for (i = 0; i < page_count; i++) { - new->memory[i] = agp_bridge.agp_alloc_page(); + void *addr = agp_bridge.agp_alloc_page(); - if (new->memory[i] == 0) { + if (addr == NULL) { /* Free this structure */ agp_free_memory(new); return NULL; } new->memory[i] = - agp_bridge.mask_memory( - virt_to_phys((void *) new->memory[i]), - type); + agp_bridge.mask_memory(virt_to_phys(addr), type); new->page_count++; } + flush_agp_mappings(); + return new; } @@ -561,6 +536,7 @@ static int agp_generic_create_gatt_table(void) agp_bridge.current_size; break; } + temp = agp_bridge.current_size; } else { agp_bridge.aperture_size_idx = i; } @@ -761,7 +737,7 @@ static void agp_generic_free_by_type(agp_memory * curr) * against a maximum value. */ -static unsigned long agp_generic_alloc_page(void) +static void *agp_generic_alloc_page(void) { struct page * page; @@ -769,24 +745,26 @@ static unsigned long agp_generic_alloc_page(void) if (page == NULL) return 0; + map_page_into_agp(page); + get_page(page); SetPageLocked(page); atomic_inc(&agp_bridge.current_memory_agp); - return (unsigned long)page_address(page); + return page_address(page); } -static void agp_generic_destroy_page(unsigned long addr) +static void agp_generic_destroy_page(void *addr) { - void *pt = (void *) addr; struct page *page; - if (pt == NULL) + if (addr == NULL) return; - page = virt_to_page(pt); + page = virt_to_page(addr); + unmap_page_from_agp(page); put_page(page); unlock_page(page); - free_page((unsigned long) pt); + free_page((unsigned long)addr); atomic_dec(&agp_bridge.current_memory_agp); } @@ -993,6 +971,7 @@ static agp_memory *intel_i810_alloc_by_type(size_t pg_count, int type) return new; } if(type == AGP_PHYS_MEMORY) { + void *addr; /* The I810 requires a physical address to program * it's mouse pointer into hardware. However the * Xserver still writes to it through the agp @@ -1007,17 +986,14 @@ static agp_memory *intel_i810_alloc_by_type(size_t pg_count, int type) return NULL; } MOD_INC_USE_COUNT; - new->memory[0] = agp_bridge.agp_alloc_page(); + addr = agp_bridge.agp_alloc_page(); - if (new->memory[0] == 0) { + if (addr == NULL) { /* Free this structure */ agp_free_memory(new); return NULL; } - new->memory[0] = - agp_bridge.mask_memory( - virt_to_phys((void *) new->memory[0]), - type); + new->memory[0] = agp_bridge.mask_memory(virt_to_phys(addr), type); new->page_count = 1; new->num_scratch_pages = 1; new->type = AGP_PHYS_MEMORY; @@ -1032,7 +1008,7 @@ static void intel_i810_free_by_type(agp_memory * curr) { agp_free_key(curr->key); if(curr->type == AGP_PHYS_MEMORY) { - agp_bridge.agp_destroy_page((unsigned long) + agp_bridge.agp_destroy_page( phys_to_virt(curr->memory[0])); vfree(curr->memory); } @@ -1291,7 +1267,7 @@ static agp_memory *intel_i830_alloc_by_type(size_t pg_count,int type) if (type == AGP_DCACHE_MEMORY) return(NULL); if (type == AGP_PHYS_MEMORY) { - unsigned long physical; + void *addr; /* The i830 requires a physical address to program * it's mouse pointer into hardware. However the @@ -1306,19 +1282,18 @@ static agp_memory *intel_i830_alloc_by_type(size_t pg_count,int type) if (nw == NULL) return(NULL); MOD_INC_USE_COUNT; - nw->memory[0] = agp_bridge.agp_alloc_page(); - physical = nw->memory[0]; - if (nw->memory[0] == 0) { + addr = agp_bridge.agp_alloc_page(); + if (addr == NULL) { /* free this structure */ agp_free_memory(nw); return(NULL); } - nw->memory[0] = agp_bridge.mask_memory(virt_to_phys((void *) nw->memory[0]),type); + nw->memory[0] = agp_bridge.mask_memory(virt_to_phys(addr),type); nw->page_count = 1; nw->num_scratch_pages = 1; nw->type = AGP_PHYS_MEMORY; - nw->physical = virt_to_phys((void *) physical); + nw->physical = virt_to_phys(addr); return(nw); } @@ -1849,16 +1824,17 @@ static int intel_i460_remove_memory(agp_memory * mem, off_t pg_start, int type) * Let's just hope nobody counts on the allocated AGP memory being there * before bind time (I don't think current drivers do)... */ -static unsigned long intel_i460_alloc_page(void) +static void * intel_i460_alloc_page(void) { if (intel_i460_cpk) return agp_generic_alloc_page(); /* Returning NULL would cause problems */ - return ~0UL; + /* AK: really dubious code. */ + return (void *)~0UL; } -static void intel_i460_destroy_page(unsigned long page) +static void intel_i460_destroy_page(void *page) { if (intel_i460_cpk) agp_generic_destroy_page(page); @@ -3298,38 +3274,29 @@ static void ali_cache_flush(void) } } -static unsigned long ali_alloc_page(void) +static void *ali_alloc_page(void) { - struct page *page; - u32 temp; + void *adr = agp_generic_alloc_page(); + unsigned temp; - page = alloc_page(GFP_KERNEL); - if (page == NULL) + if (adr == 0) return 0; - get_page(page); - SetPageLocked(page); - atomic_inc(&agp_bridge.current_memory_agp); - - global_cache_flush(); - if (agp_bridge.type == ALI_M1541) { pci_read_config_dword(agp_bridge.dev, ALI_CACHE_FLUSH_CTRL, &temp); pci_write_config_dword(agp_bridge.dev, ALI_CACHE_FLUSH_CTRL, (((temp & ALI_CACHE_FLUSH_ADDR_MASK) | - virt_to_phys(page_address(page))) | + virt_to_phys(adr)) | ALI_CACHE_FLUSH_EN )); } - return (unsigned long)page_address(page); + return adr; } -static void ali_destroy_page(unsigned long addr) +static void ali_destroy_page(void * addr) { u32 temp; - void *pt = (void *) addr; - struct page *page; - if (pt == NULL) + if (addr == NULL) return; global_cache_flush(); @@ -3338,15 +3305,11 @@ static void ali_destroy_page(unsigned long addr) pci_read_config_dword(agp_bridge.dev, ALI_CACHE_FLUSH_CTRL, &temp); pci_write_config_dword(agp_bridge.dev, ALI_CACHE_FLUSH_CTRL, (((temp & ALI_CACHE_FLUSH_ADDR_MASK) | - virt_to_phys((void *)pt)) | + virt_to_phys(addr)) | ALI_CACHE_FLUSH_EN)); } - page = virt_to_page(pt); - put_page(page); - unlock_page(page); - free_page((unsigned long) pt); - atomic_dec(&agp_bridge.current_memory_agp); + agp_generic_destroy_page(addr); } /* Setup function */ @@ -5011,15 +4974,15 @@ static int __init agp_backend_initialize(void) } if (agp_bridge.needs_scratch_page == TRUE) { - agp_bridge.scratch_page = agp_bridge.agp_alloc_page(); + void *addr; + addr = agp_bridge.agp_alloc_page(); - if (agp_bridge.scratch_page == 0) { + if (addr == NULL) { printk(KERN_ERR PFX "unable to get memory for " "scratch page.\n"); return -ENOMEM; } - agp_bridge.scratch_page = - virt_to_phys((void *) agp_bridge.scratch_page); + agp_bridge.scratch_page = virt_to_phys(addr); agp_bridge.scratch_page = agp_bridge.mask_memory(agp_bridge.scratch_page, 0); } @@ -5064,8 +5027,7 @@ static int __init agp_backend_initialize(void) err_out: if (agp_bridge.needs_scratch_page == TRUE) { agp_bridge.scratch_page &= ~(0x00000fff); - agp_bridge.agp_destroy_page((unsigned long) - phys_to_virt(agp_bridge.scratch_page)); + agp_bridge.agp_destroy_page(phys_to_virt(agp_bridge.scratch_page)); } if (got_gatt) agp_bridge.free_gatt_table(); @@ -5084,8 +5046,7 @@ static void agp_backend_cleanup(void) if (agp_bridge.needs_scratch_page == TRUE) { agp_bridge.scratch_page &= ~(0x00000fff); - agp_bridge.agp_destroy_page((unsigned long) - phys_to_virt(agp_bridge.scratch_page)); + agp_bridge.agp_destroy_page(phys_to_virt(agp_bridge.scratch_page)); } } diff --git a/include/asm-alpha/agp.h b/include/asm-alpha/agp.h new file mode 100644 index 000000000000..ba05bdf9a211 --- /dev/null +++ b/include/asm-alpha/agp.h @@ -0,0 +1,11 @@ +#ifndef AGP_H +#define AGP_H 1 + +/* dummy for now */ + +#define map_page_into_agp(page) +#define unmap_page_from_agp(page) +#define flush_agp_mappings() +#define flush_agp_cache() mb() + +#endif diff --git a/include/asm-i386/agp.h b/include/asm-i386/agp.h new file mode 100644 index 000000000000..9ae97c09fb49 --- /dev/null +++ b/include/asm-i386/agp.h @@ -0,0 +1,23 @@ +#ifndef AGP_H +#define AGP_H 1 + +#include + +/* + * Functions to keep the agpgart mappings coherent with the MMU. + * The GART gives the CPU a physical alias of pages in memory. The alias region is + * mapped uncacheable. Make sure there are no conflicting mappings + * with different cachability attributes for the same page. This avoids + * data corruption on some CPUs. + */ + +#define map_page_into_agp(page) change_page_attr(page, 1, PAGE_KERNEL_NOCACHE) +#define unmap_page_from_agp(page) change_page_attr(page, 1, PAGE_KERNEL) +#define flush_agp_mappings() global_flush_tlb() + +/* Could use CLFLUSH here if the cpu supports it. But then it would + need to be called for each cacheline of the whole page so it may not be + worth it. Would need a page for it. */ +#define flush_agp_cache() asm volatile("wbinvd":::"memory") + +#endif diff --git a/include/asm-i386/cacheflush.h b/include/asm-i386/cacheflush.h index 58d027dfc5ff..319e65a7047f 100644 --- a/include/asm-i386/cacheflush.h +++ b/include/asm-i386/cacheflush.h @@ -15,4 +15,7 @@ #define flush_icache_page(vma,pg) do { } while (0) #define flush_icache_user_range(vma,pg,adr,len) do { } while (0) +void global_flush_tlb(void); +int change_page_attr(struct page *page, int numpages, pgprot_t prot); + #endif /* _I386_CACHEFLUSH_H */ diff --git a/include/asm-i386/io.h b/include/asm-i386/io.h index 44996d06ecc3..9922dd823c9c 100644 --- a/include/asm-i386/io.h +++ b/include/asm-i386/io.h @@ -121,31 +121,7 @@ static inline void * ioremap (unsigned long offset, unsigned long size) return __ioremap(offset, size, 0); } -/** - * ioremap_nocache - map bus memory into CPU space - * @offset: bus address of the memory - * @size: size of the resource to map - * - * ioremap_nocache performs a platform specific sequence of operations to - * make bus memory CPU accessible via the readb/readw/readl/writeb/ - * writew/writel functions and the other mmio helpers. The returned - * address is not guaranteed to be usable directly as a virtual - * address. - * - * This version of ioremap ensures that the memory is marked uncachable - * on the CPU as well as honouring existing caching rules from things like - * the PCI bus. Note that there are other caches and buffers on many - * busses. In paticular driver authors should read up on PCI writes - * - * It's useful if some control registers are in such an area and - * write combining or read caching is not desirable: - */ - -static inline void * ioremap_nocache (unsigned long offset, unsigned long size) -{ - return __ioremap(offset, size, _PAGE_PCD); -} - +extern void * ioremap_nocache (unsigned long offset, unsigned long size); extern void iounmap(void *addr); /* diff --git a/include/asm-i386/page.h b/include/asm-i386/page.h index 4737ef69ae18..d8e1f404c08b 100644 --- a/include/asm-i386/page.h +++ b/include/asm-i386/page.h @@ -6,6 +6,9 @@ #define PAGE_SIZE (1UL << PAGE_SHIFT) #define PAGE_MASK (~(PAGE_SIZE-1)) +#define LARGE_PAGE_MASK (~(LARGE_PAGE_SIZE-1)) +#define LARGE_PAGE_SIZE (1UL << PMD_SHIFT) + #ifdef __KERNEL__ #ifndef __ASSEMBLY__ diff --git a/include/asm-i386/pgtable-2level.h b/include/asm-i386/pgtable-2level.h index e22db0cc6824..9f8bdc13adac 100644 --- a/include/asm-i386/pgtable-2level.h +++ b/include/asm-i386/pgtable-2level.h @@ -40,6 +40,7 @@ static inline int pgd_present(pgd_t pgd) { return 1; } * hook is made available. */ #define set_pte(pteptr, pteval) (*(pteptr) = pteval) +#define set_pte_atomic(pteptr, pteval) set_pte(pteptr,pteval) /* * (pmds are folded into pgds so this doesnt get actually called, * but the define is needed for a generic inline function.) diff --git a/include/asm-i386/pgtable-3level.h b/include/asm-i386/pgtable-3level.h index bb2eaea63fde..beb0c1bc3d30 100644 --- a/include/asm-i386/pgtable-3level.h +++ b/include/asm-i386/pgtable-3level.h @@ -49,6 +49,8 @@ static inline void set_pte(pte_t *ptep, pte_t pte) smp_wmb(); ptep->pte_low = pte.pte_low; } +#define set_pte_atomic(pteptr,pteval) \ + set_64bit((unsigned long long *)(pteptr),pte_val(pteval)) #define set_pmd(pmdptr,pmdval) \ set_64bit((unsigned long long *)(pmdptr),pmd_val(pmdval)) #define set_pgd(pgdptr,pgdval) \ diff --git a/include/asm-i386/pgtable.h b/include/asm-i386/pgtable.h index f48db2beeeba..71b75fa234af 100644 --- a/include/asm-i386/pgtable.h +++ b/include/asm-i386/pgtable.h @@ -237,6 +237,9 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) #define pmd_page(pmd) \ (mem_map + (pmd_val(pmd) >> PAGE_SHIFT)) +#define pmd_large(pmd) \ + ((pmd_val(pmd) & (_PAGE_PSE|_PAGE_PRESENT)) == (_PAGE_PSE|_PAGE_PRESENT)) + /* to find an entry in a page-table-directory. */ #define pgd_index(address) ((address >> PGDIR_SHIFT) & (PTRS_PER_PGD-1)) diff --git a/include/asm-ia64/agp.h b/include/asm-ia64/agp.h new file mode 100644 index 000000000000..ba05bdf9a211 --- /dev/null +++ b/include/asm-ia64/agp.h @@ -0,0 +1,11 @@ +#ifndef AGP_H +#define AGP_H 1 + +/* dummy for now */ + +#define map_page_into_agp(page) +#define unmap_page_from_agp(page) +#define flush_agp_mappings() +#define flush_agp_cache() mb() + +#endif diff --git a/include/asm-sparc64/agp.h b/include/asm-sparc64/agp.h new file mode 100644 index 000000000000..ba05bdf9a211 --- /dev/null +++ b/include/asm-sparc64/agp.h @@ -0,0 +1,11 @@ +#ifndef AGP_H +#define AGP_H 1 + +/* dummy for now */ + +#define map_page_into_agp(page) +#define unmap_page_from_agp(page) +#define flush_agp_mappings() +#define flush_agp_cache() mb() + +#endif diff --git a/include/asm-x86_64/agp.h b/include/asm-x86_64/agp.h new file mode 100644 index 000000000000..8c2fabe80419 --- /dev/null +++ b/include/asm-x86_64/agp.h @@ -0,0 +1,23 @@ +#ifndef AGP_H +#define AGP_H 1 + +#include + +/* + * Functions to keep the agpgart mappings coherent. + * The GART gives the CPU a physical alias of memory. The alias is + * mapped uncacheable. Make sure there are no conflicting mappings + * with different cachability attributes for the same page. + */ + +#define map_page_into_agp(page) \ + change_page_attr(page, __pgprot(__PAGE_KERNEL | _PAGE_PCD)) +#define unmap_page_from_agp(page) change_page_attr(page, PAGE_KERNEL) +#define flush_agp_mappings() global_flush_tlb() + +/* Could use CLFLUSH here if the cpu supports it. But then it would + need to be called for each cacheline of the whole page so it may not be + worth it. Would need a page for it. */ +#define flush_agp_cache() asm volatile("wbinvd":::"memory") + +#endif diff --git a/include/asm-x86_64/cacheflush.h b/include/asm-x86_64/cacheflush.h index 58d027dfc5ff..319e65a7047f 100644 --- a/include/asm-x86_64/cacheflush.h +++ b/include/asm-x86_64/cacheflush.h @@ -15,4 +15,7 @@ #define flush_icache_page(vma,pg) do { } while (0) #define flush_icache_user_range(vma,pg,adr,len) do { } while (0) +void global_flush_tlb(void); +int change_page_attr(struct page *page, int numpages, pgprot_t prot); + #endif /* _I386_CACHEFLUSH_H */ diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index 4051c031a976..9cc67b500368 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -13,6 +13,7 @@ struct vm_struct { unsigned long flags; void * addr; unsigned long size; + unsigned long phys_addr; struct vm_struct * next; }; @@ -23,6 +24,8 @@ extern long vread(char *buf, char *addr, unsigned long count); extern void vmfree_area_pages(unsigned long address, unsigned long size); extern int vmalloc_area_pages(unsigned long address, unsigned long size, int gfp_mask, pgprot_t prot); +extern struct vm_struct *remove_kernel_area(void *addr); + /* * Various ways to allocate pages. */ diff --git a/mm/vmalloc.c b/mm/vmalloc.c index f95ebed746b0..50cc6d13f0ff 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -195,6 +195,7 @@ struct vm_struct * get_vm_area(unsigned long size, unsigned long flags) if (addr > VMALLOC_END-size) goto out; } + area->phys_addr = 0; area->flags = flags; area->addr = (void *)addr; area->size = size; @@ -209,9 +210,25 @@ out: return NULL; } -void vfree(void * addr) +struct vm_struct *remove_kernel_area(void *addr) { struct vm_struct **p, *tmp; + write_lock(&vmlist_lock); + for (p = &vmlist ; (tmp = *p) ; p = &tmp->next) { + if (tmp->addr == addr) { + *p = tmp->next; + write_unlock(&vmlist_lock); + return tmp; + } + + } + write_unlock(&vmlist_lock); + return NULL; +} + +void vfree(void * addr) +{ + struct vm_struct *tmp; if (!addr) return; @@ -219,17 +236,12 @@ void vfree(void * addr) printk(KERN_ERR "Trying to vfree() bad address (%p)\n", addr); return; } - write_lock(&vmlist_lock); - for (p = &vmlist ; (tmp = *p) ; p = &tmp->next) { - if (tmp->addr == addr) { - *p = tmp->next; + tmp = remove_kernel_area(addr); + if (tmp) { vmfree_area_pages(VMALLOC_VMADDR(tmp->addr), tmp->size); - write_unlock(&vmlist_lock); kfree(tmp); return; } - } - write_unlock(&vmlist_lock); printk(KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n", addr); } -- 2.39.5