X-Git-Url: http://git.onelab.eu/?a=blobdiff_plain;f=arch%2Farm%2Fmm%2Fmm-armv.c;h=95273de4f772515780e2eac97ed2ae892ac52a0d;hb=43bc926fffd92024b46cafaf7350d669ba9ca884;hp=2c2b93d77d433248dba64b6a1dd525eae51277fc;hpb=cee37fe97739d85991964371c1f3a745c00dd236;p=linux-2.6.git diff --git a/arch/arm/mm/mm-armv.c b/arch/arm/mm/mm-armv.c index 2c2b93d77..95273de4f 100644 --- a/arch/arm/mm/mm-armv.c +++ b/arch/arm/mm/mm-armv.c @@ -1,7 +1,7 @@ /* * linux/arch/arm/mm/mm-armv.c * - * Copyright (C) 1998-2002 Russell King + * Copyright (C) 1998-2005 Russell King * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -19,7 +19,6 @@ #include #include -#include #include #include @@ -169,14 +168,16 @@ pgd_t *get_pgd_slow(struct mm_struct *mm) memzero(new_pgd, FIRST_KERNEL_PGD_NR * sizeof(pgd_t)); + /* + * Copy over the kernel and IO PGD entries + */ init_pgd = pgd_offset_k(0); + memcpy(new_pgd + FIRST_KERNEL_PGD_NR, init_pgd + FIRST_KERNEL_PGD_NR, + (PTRS_PER_PGD - FIRST_KERNEL_PGD_NR) * sizeof(pgd_t)); - if (!vectors_high()) { - /* - * This lock is here just to satisfy pmd_alloc and pte_lock - */ - spin_lock(&mm->page_table_lock); + clean_dcache_area(new_pgd, PTRS_PER_PGD * sizeof(pgd_t)); + if (!vectors_high()) { /* * On ARM, first page must always be allocated since it * contains the machine vectors. @@ -194,31 +195,14 @@ pgd_t *get_pgd_slow(struct mm_struct *mm) set_pte(new_pte, *init_pte); pte_unmap_nested(init_pte); pte_unmap(new_pte); - - spin_unlock(&mm->page_table_lock); } - /* - * Copy over the kernel and IO PGD entries - */ - memcpy(new_pgd + FIRST_KERNEL_PGD_NR, init_pgd + FIRST_KERNEL_PGD_NR, - (PTRS_PER_PGD - FIRST_KERNEL_PGD_NR) * sizeof(pgd_t)); - - clean_dcache_area(new_pgd, PTRS_PER_PGD * sizeof(pgd_t)); - return new_pgd; no_pte: - spin_unlock(&mm->page_table_lock); pmd_free(new_pmd); - free_pages((unsigned long)new_pgd, 2); - return NULL; - no_pmd: - spin_unlock(&mm->page_table_lock); free_pages((unsigned long)new_pgd, 2); - return NULL; - no_pgd: return NULL; } @@ -244,6 +228,7 @@ void free_pgd_slow(pgd_t *pgd) pte = pmd_page(*pmd); pmd_clear(pmd); dec_page_state(nr_page_table_pages); + pte_lock_deinit(pte); pte_free(pte); pmd_free(pmd); free: @@ -276,11 +261,9 @@ alloc_init_supersection(unsigned long virt, unsigned long phys, int prot) int i; for (i = 0; i < 16; i += 1) { - alloc_init_section(virt, phys & SUPERSECTION_MASK, - prot | PMD_SECT_SUPER); + alloc_init_section(virt, phys, prot | PMD_SECT_SUPER); virt += (PGDIR_SIZE / 2); - phys += (PGDIR_SIZE / 2); } } @@ -298,30 +281,16 @@ alloc_init_page(unsigned long virt, unsigned long phys, unsigned int prot_l1, pg pte_t *ptep; if (pmd_none(*pmdp)) { - unsigned long pmdval; ptep = alloc_bootmem_low_pages(2 * PTRS_PER_PTE * sizeof(pte_t)); - pmdval = __pa(ptep) | prot_l1; - pmdp[0] = __pmd(pmdval); - pmdp[1] = __pmd(pmdval + 256 * sizeof(pte_t)); - flush_pmd_entry(pmdp); + __pmd_populate(pmdp, __pa(ptep) | prot_l1); } ptep = pte_offset_kernel(pmdp, virt); set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, prot)); } -/* - * Clear any PGD mapping. On a two-level page table system, - * the clearance is done by the middle-level functions (pmd) - * rather than the top-level (pgd) functions. - */ -static inline void clear_mapping(unsigned long virt) -{ - pmd_clear(pmd_off_k(virt)); -} - struct mem_types { unsigned int prot_pte; unsigned int prot_l1; @@ -374,16 +343,23 @@ static struct mem_types mem_types[] __initdata = { PMD_SECT_AP_WRITE | PMD_SECT_BUFFERABLE | PMD_SECT_TEX(1), .domain = DOMAIN_IO, + }, + [MT_NONSHARED_DEVICE] = { + .prot_l1 = PMD_TYPE_TABLE, + .prot_sect = PMD_TYPE_SECT | PMD_SECT_NONSHARED_DEV | + PMD_SECT_AP_WRITE, + .domain = DOMAIN_IO, } }; /* * Adjust the PMD section entries according to the CPU in use. */ -static void __init build_mem_type_table(void) +void __init build_mem_type_table(void) { struct cachepolicy *cp; unsigned int cr = get_cr(); + unsigned int user_pgprot, kern_pgprot; int cpu_arch = cpu_architecture(); int i; @@ -400,7 +376,7 @@ static void __init build_mem_type_table(void) ecc_mask = 0; } - if (cpu_arch <= CPU_ARCH_ARMv5) { + if (cpu_arch <= CPU_ARCH_ARMv5TEJ && !cpu_is_xscale()) { for (i = 0; i < ARRAY_SIZE(mem_types); i++) { if (mem_types[i].prot_l1) mem_types[i].prot_l1 |= PMD_BIT4; @@ -409,6 +385,20 @@ static void __init build_mem_type_table(void) } } + cp = &cache_policies[cachepolicy]; + kern_pgprot = user_pgprot = cp->pte; + + /* + * Enable CPU-specific coherency if supported. + * (Only available on XSC3 at the moment.) + */ + if (arch_is_coherent()) { + if (cpu_is_xsc3()) { + mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S; + mem_types[MT_MEMORY].prot_pte |= L_PTE_COHERENT; + } + } + /* * ARMv6 and above have extended page tables. */ @@ -419,6 +409,7 @@ static void __init build_mem_type_table(void) */ mem_types[MT_MEMORY].prot_sect &= ~PMD_BIT4; mem_types[MT_ROM].prot_sect &= ~PMD_BIT4; + /* * Mark cache clean areas and XIP ROM read only * from SVC mode and no access from userspace. @@ -426,34 +417,59 @@ static void __init build_mem_type_table(void) mem_types[MT_ROM].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; mem_types[MT_MINICLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; + + /* + * Mark the device area as "shared device" + */ + mem_types[MT_DEVICE].prot_pte |= L_PTE_BUFFERABLE; + mem_types[MT_DEVICE].prot_sect |= PMD_SECT_BUFFERED; + + /* + * User pages need to be mapped with the ASID + * (iow, non-global) + */ + user_pgprot |= L_PTE_ASID; + +#ifdef CONFIG_SMP + /* + * Mark memory with the "shared" attribute for SMP systems + */ + user_pgprot |= L_PTE_SHARED; + kern_pgprot |= L_PTE_SHARED; + mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S; +#endif } - cp = &cache_policies[cachepolicy]; + for (i = 0; i < 16; i++) { + unsigned long v = pgprot_val(protection_map[i]); + v = (v & ~(L_PTE_BUFFERABLE|L_PTE_CACHEABLE)) | user_pgprot; + protection_map[i] = __pgprot(v); + } + + mem_types[MT_LOW_VECTORS].prot_pte |= kern_pgprot; + mem_types[MT_HIGH_VECTORS].prot_pte |= kern_pgprot; if (cpu_arch >= CPU_ARCH_ARMv5) { - mem_types[MT_LOW_VECTORS].prot_pte |= cp->pte & PTE_CACHEABLE; - mem_types[MT_HIGH_VECTORS].prot_pte |= cp->pte & PTE_CACHEABLE; +#ifndef CONFIG_SMP + /* + * Only use write-through for non-SMP systems + */ + mem_types[MT_LOW_VECTORS].prot_pte &= ~L_PTE_BUFFERABLE; + mem_types[MT_HIGH_VECTORS].prot_pte &= ~L_PTE_BUFFERABLE; +#endif } else { - mem_types[MT_LOW_VECTORS].prot_pte |= cp->pte; - mem_types[MT_HIGH_VECTORS].prot_pte |= cp->pte; mem_types[MT_MINICLEAN].prot_sect &= ~PMD_SECT_TEX(1); } + pgprot_kernel = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | + L_PTE_DIRTY | L_PTE_WRITE | + L_PTE_EXEC | kern_pgprot); + mem_types[MT_LOW_VECTORS].prot_l1 |= ecc_mask; mem_types[MT_HIGH_VECTORS].prot_l1 |= ecc_mask; mem_types[MT_MEMORY].prot_sect |= ecc_mask | cp->pmd; mem_types[MT_ROM].prot_sect |= cp->pmd; - for (i = 0; i < 16; i++) { - unsigned long v = pgprot_val(protection_map[i]); - v &= (~(PTE_BUFFERABLE|PTE_CACHEABLE)) | cp->pte; - protection_map[i] = __pgprot(v); - } - - pgprot_kernel = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | - L_PTE_DIRTY | L_PTE_WRITE | - L_PTE_EXEC | cp->pte); - switch (cp->pmd) { case PMD_SECT_WT: mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_WT; @@ -476,25 +492,25 @@ static void __init build_mem_type_table(void) * offsets, and we take full advantage of sections and * supersections. */ -static void __init create_mapping(struct map_desc *md) +void __init create_mapping(struct map_desc *md) { unsigned long virt, length; int prot_sect, prot_l1, domain; pgprot_t prot_pte; - long off; + unsigned long off = (u32)__pfn_to_phys(md->pfn); if (md->virtual != vectors_base() && md->virtual < TASK_SIZE) { printk(KERN_WARNING "BUG: not creating mapping for " - "0x%08lx at 0x%08lx in user region\n", - md->physical, md->virtual); + "0x%08llx at 0x%08lx in user region\n", + __pfn_to_phys((u64)md->pfn), md->virtual); return; } if ((md->type == MT_DEVICE || md->type == MT_ROM) && md->virtual >= PAGE_OFFSET && md->virtual < VMALLOC_END) { - printk(KERN_WARNING "BUG: mapping for 0x%08lx at 0x%08lx " + printk(KERN_WARNING "BUG: mapping for 0x%08llx at 0x%08lx " "overlaps vmalloc space\n", - md->physical, md->virtual); + __pfn_to_phys((u64)md->pfn), md->virtual); } domain = mem_types[md->type].domain; @@ -502,15 +518,40 @@ static void __init create_mapping(struct map_desc *md) prot_l1 = mem_types[md->type].prot_l1 | PMD_DOMAIN(domain); prot_sect = mem_types[md->type].prot_sect | PMD_DOMAIN(domain); + /* + * Catch 36-bit addresses + */ + if(md->pfn >= 0x100000) { + if(domain) { + printk(KERN_ERR "MM: invalid domain in supersection " + "mapping for 0x%08llx at 0x%08lx\n", + __pfn_to_phys((u64)md->pfn), md->virtual); + return; + } + if((md->virtual | md->length | __pfn_to_phys(md->pfn)) + & ~SUPERSECTION_MASK) { + printk(KERN_ERR "MM: cannot create mapping for " + "0x%08llx at 0x%08lx invalid alignment\n", + __pfn_to_phys((u64)md->pfn), md->virtual); + return; + } + + /* + * Shift bits [35:32] of address into bits [23:20] of PMD + * (See ARMv6 spec). + */ + off |= (((md->pfn >> (32 - PAGE_SHIFT)) & 0xF) << 20); + } + virt = md->virtual; - off = md->physical - virt; + off -= virt; length = md->length; if (mem_types[md->type].prot_l1 == 0 && (virt & 0xfffff || (virt + off) & 0xfffff || (virt + length) & 0xfffff)) { printk(KERN_WARNING "BUG: map for 0x%08lx at 0x%08lx can not " "be mapped using pages, ignoring.\n", - md->physical, md->virtual); + __pfn_to_phys(md->pfn), md->virtual); return; } @@ -527,14 +568,24 @@ static void __init create_mapping(struct map_desc *md) * supersections are only allocated for domain 0 regardless * of the actual domain assignments in use. */ - if (cpu_architecture() >= CPU_ARCH_ARMv6 && domain == 0) { - /* Align to supersection boundary */ - while ((virt & ~SUPERSECTION_MASK || (virt + off) & - ~SUPERSECTION_MASK) && length >= (PGDIR_SIZE / 2)) { - alloc_init_section(virt, virt + off, prot_sect); - - virt += (PGDIR_SIZE / 2); - length -= (PGDIR_SIZE / 2); + if ((cpu_architecture() >= CPU_ARCH_ARMv6 || cpu_is_xsc3()) + && domain == 0) { + /* + * Align to supersection boundary if !high pages. + * High pages have already been checked for proper + * alignment above and they will fail the SUPSERSECTION_MASK + * check because of the way the address is encoded into + * offset. + */ + if (md->pfn <= 0x100000) { + while ((virt & ~SUPERSECTION_MASK || + (virt + off) & ~SUPERSECTION_MASK) && + length >= (PGDIR_SIZE / 2)) { + alloc_init_section(virt, virt + off, prot_sect); + + virt += (PGDIR_SIZE / 2); + length -= (PGDIR_SIZE / 2); + } } while (length >= SUPERSECTION_SIZE) { @@ -570,23 +621,23 @@ static void __init create_mapping(struct map_desc *md) */ void setup_mm_for_reboot(char mode) { - unsigned long pmdval; + unsigned long base_pmdval; pgd_t *pgd; - pmd_t *pmd; int i; - int cpu_arch = cpu_architecture(); if (current->mm && current->mm->pgd) pgd = current->mm->pgd; else pgd = init_mm.pgd; - for (i = 0; i < FIRST_USER_PGD_NR + USER_PTRS_PER_PGD; i++) { - pmdval = (i << PGDIR_SHIFT) | - PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | - PMD_TYPE_SECT; - if (cpu_arch <= CPU_ARCH_ARMv5) - pmdval |= PMD_BIT4; + base_pmdval = PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | PMD_TYPE_SECT; + if (cpu_architecture() <= CPU_ARCH_ARMv5TEJ && !cpu_is_xscale()) + base_pmdval |= PMD_BIT4; + + for (i = 0; i < FIRST_USER_PGD_NR + USER_PTRS_PER_PGD; i++, pgd++) { + unsigned long pmdval = (i << PGDIR_SHIFT) | base_pmdval; + pmd_t *pmd; + pmd = pmd_off(pgd, i << PGDIR_SHIFT); pmd[0] = __pmd(pmdval); pmd[1] = __pmd(pmdval + (1 << (PGDIR_SHIFT - 1))); @@ -594,100 +645,6 @@ void setup_mm_for_reboot(char mode) } } -extern void _stext, _etext; - -/* - * Setup initial mappings. We use the page we allocated for zero page to hold - * the mappings, which will get overwritten by the vectors in traps_init(). - * The mappings must be in virtual address order. - */ -void __init memtable_init(struct meminfo *mi) -{ - struct map_desc *init_maps, *p, *q; - unsigned long address = 0; - int i; - - build_mem_type_table(); - - init_maps = p = alloc_bootmem_low_pages(PAGE_SIZE); - -#ifdef CONFIG_XIP_KERNEL - p->physical = CONFIG_XIP_PHYS_ADDR & PMD_MASK; - p->virtual = (unsigned long)&_stext & PMD_MASK; - p->length = ((unsigned long)&_etext - p->virtual + ~PMD_MASK) & PMD_MASK; - p->type = MT_ROM; - p ++; -#endif - - for (i = 0; i < mi->nr_banks; i++) { - if (mi->bank[i].size == 0) - continue; - - p->physical = mi->bank[i].start; - p->virtual = __phys_to_virt(p->physical); - p->length = mi->bank[i].size; - p->type = MT_MEMORY; - p ++; - } - -#ifdef FLUSH_BASE - p->physical = FLUSH_BASE_PHYS; - p->virtual = FLUSH_BASE; - p->length = PGDIR_SIZE; - p->type = MT_CACHECLEAN; - p ++; -#endif - -#ifdef FLUSH_BASE_MINICACHE - p->physical = FLUSH_BASE_PHYS + PGDIR_SIZE; - p->virtual = FLUSH_BASE_MINICACHE; - p->length = PGDIR_SIZE; - p->type = MT_MINICLEAN; - p ++; -#endif - - /* - * Go through the initial mappings, but clear out any - * pgdir entries that are not in the description. - */ - q = init_maps; - do { - if (address < q->virtual || q == p) { - clear_mapping(address); - address += PGDIR_SIZE; - } else { - create_mapping(q); - - address = q->virtual + q->length; - address = (address + PGDIR_SIZE - 1) & PGDIR_MASK; - - q ++; - } - } while (address != 0); - - /* - * Create a mapping for the machine vectors at the high-vectors - * location (0xffff0000). If we aren't using high-vectors, also - * create a mapping at the low-vectors virtual address. - */ - init_maps->physical = virt_to_phys(init_maps); - init_maps->virtual = 0xffff0000; - init_maps->length = PAGE_SIZE; - init_maps->type = MT_HIGH_VECTORS; - create_mapping(init_maps); - - if (!vectors_high()) { - init_maps->virtual = 0; - init_maps->type = MT_LOW_VECTORS; - create_mapping(init_maps); - } - - flush_cache_all(); - flush_tlb_all(); - - top_pmd = pmd_off_k(0xffff0000); -} - /* * Create the architecture specific mappings */ @@ -698,75 +655,3 @@ void __init iotable_init(struct map_desc *io_desc, int nr) for (i = 0; i < nr; i++) create_mapping(io_desc + i); } - -static inline void -free_memmap(int node, unsigned long start_pfn, unsigned long end_pfn) -{ - struct page *start_pg, *end_pg; - unsigned long pg, pgend; - - /* - * Convert start_pfn/end_pfn to a struct page pointer. - */ - start_pg = pfn_to_page(start_pfn); - end_pg = pfn_to_page(end_pfn); - - /* - * Convert to physical addresses, and - * round start upwards and end downwards. - */ - pg = PAGE_ALIGN(__pa(start_pg)); - pgend = __pa(end_pg) & PAGE_MASK; - - /* - * If there are free pages between these, - * free the section of the memmap array. - */ - if (pg < pgend) - free_bootmem_node(NODE_DATA(node), pg, pgend - pg); -} - -static inline void free_unused_memmap_node(int node, struct meminfo *mi) -{ - unsigned long bank_start, prev_bank_end = 0; - unsigned int i; - - /* - * [FIXME] This relies on each bank being in address order. This - * may not be the case, especially if the user has provided the - * information on the command line. - */ - for (i = 0; i < mi->nr_banks; i++) { - if (mi->bank[i].size == 0 || mi->bank[i].node != node) - continue; - - bank_start = mi->bank[i].start >> PAGE_SHIFT; - if (bank_start < prev_bank_end) { - printk(KERN_ERR "MEM: unordered memory banks. " - "Not freeing memmap.\n"); - break; - } - - /* - * If we had a previous bank, and there is a space - * between the current bank and the previous, free it. - */ - if (prev_bank_end && prev_bank_end != bank_start) - free_memmap(node, prev_bank_end, bank_start); - - prev_bank_end = PAGE_ALIGN(mi->bank[i].start + - mi->bank[i].size) >> PAGE_SHIFT; - } -} - -/* - * The mem_map array can get very big. Free - * the unused area of the memory map. - */ -void __init create_memmap_holes(struct meminfo *mi) -{ - int node; - - for_each_online_node(node) - free_unused_memmap_node(node, mi); -}