diff options
Diffstat (limited to 'kernel/arch/xtensa/mm')
| -rw-r--r-- | kernel/arch/xtensa/mm/Makefile | 7 | ||||
| -rw-r--r-- | kernel/arch/xtensa/mm/cache.c | 317 | ||||
| -rw-r--r-- | kernel/arch/xtensa/mm/fault.c | 260 | ||||
| -rw-r--r-- | kernel/arch/xtensa/mm/highmem.c | 95 | ||||
| -rw-r--r-- | kernel/arch/xtensa/mm/init.c | 430 | ||||
| -rw-r--r-- | kernel/arch/xtensa/mm/misc.S | 484 | ||||
| -rw-r--r-- | kernel/arch/xtensa/mm/mmu.c | 110 | ||||
| -rw-r--r-- | kernel/arch/xtensa/mm/tlb.c | 278 |
8 files changed, 1981 insertions, 0 deletions
diff --git a/kernel/arch/xtensa/mm/Makefile b/kernel/arch/xtensa/mm/Makefile new file mode 100644 index 000000000..e601e2fbe --- /dev/null +++ b/kernel/arch/xtensa/mm/Makefile @@ -0,0 +1,7 @@ +# +# Makefile for the Linux/Xtensa-specific parts of the memory manager. +# + +obj-y := init.o misc.o +obj-$(CONFIG_MMU) += cache.o fault.o mmu.o tlb.o +obj-$(CONFIG_HIGHMEM) += highmem.o diff --git a/kernel/arch/xtensa/mm/cache.c b/kernel/arch/xtensa/mm/cache.c new file mode 100644 index 000000000..d75aa1476 --- /dev/null +++ b/kernel/arch/xtensa/mm/cache.c @@ -0,0 +1,317 @@ +/* + * arch/xtensa/mm/cache.c + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2001-2006 Tensilica Inc. + * + * Chris Zankel <chris@zankel.net> + * Joe Taylor + * Marc Gauthier + * + */ + +#include <linux/init.h> +#include <linux/signal.h> +#include <linux/sched.h> +#include <linux/kernel.h> +#include <linux/errno.h> +#include <linux/string.h> +#include <linux/types.h> +#include <linux/ptrace.h> +#include <linux/bootmem.h> +#include <linux/swap.h> +#include <linux/pagemap.h> + +#include <asm/bootparam.h> +#include <asm/mmu_context.h> +#include <asm/tlb.h> +#include <asm/tlbflush.h> +#include <asm/page.h> +#include <asm/pgalloc.h> +#include <asm/pgtable.h> + +//#define printd(x...) printk(x) +#define printd(x...) do { } while(0) + +/* + * Note: + * The kernel provides one architecture bit PG_arch_1 in the page flags that + * can be used for cache coherency. + * + * I$-D$ coherency. + * + * The Xtensa architecture doesn't keep the instruction cache coherent with + * the data cache. We use the architecture bit to indicate if the caches + * are coherent. The kernel clears this bit whenever a page is added to the + * page cache. At that time, the caches might not be in sync. We, therefore, + * define this flag as 'clean' if set. + * + * D-cache aliasing. + * + * With cache aliasing, we have to always flush the cache when pages are + * unmapped (see tlb_start_vma(). So, we use this flag to indicate a dirty + * page. + * + * + * + */ + +#if (DCACHE_WAY_SIZE > PAGE_SIZE) +static inline void kmap_invalidate_coherent(struct page *page, + unsigned long vaddr) +{ + if (!DCACHE_ALIAS_EQ(page_to_phys(page), vaddr)) { + unsigned long kvaddr; + + if (!PageHighMem(page)) { + kvaddr = (unsigned long)page_to_virt(page); + + __invalidate_dcache_page(kvaddr); + } else { + kvaddr = TLBTEMP_BASE_1 + + (page_to_phys(page) & DCACHE_ALIAS_MASK); + + __invalidate_dcache_page_alias(kvaddr, + page_to_phys(page)); + } + } +} + +static inline void *coherent_kvaddr(struct page *page, unsigned long base, + unsigned long vaddr, unsigned long *paddr) +{ + if (PageHighMem(page) || !DCACHE_ALIAS_EQ(page_to_phys(page), vaddr)) { + *paddr = page_to_phys(page); + return (void *)(base + (vaddr & DCACHE_ALIAS_MASK)); + } else { + *paddr = 0; + return page_to_virt(page); + } +} + +void clear_user_highpage(struct page *page, unsigned long vaddr) +{ + unsigned long paddr; + void *kvaddr = coherent_kvaddr(page, TLBTEMP_BASE_1, vaddr, &paddr); + + pagefault_disable(); + kmap_invalidate_coherent(page, vaddr); + set_bit(PG_arch_1, &page->flags); + clear_page_alias(kvaddr, paddr); + pagefault_enable(); +} + +void copy_user_highpage(struct page *dst, struct page *src, + unsigned long vaddr, struct vm_area_struct *vma) +{ + unsigned long dst_paddr, src_paddr; + void *dst_vaddr = coherent_kvaddr(dst, TLBTEMP_BASE_1, vaddr, + &dst_paddr); + void *src_vaddr = coherent_kvaddr(src, TLBTEMP_BASE_2, vaddr, + &src_paddr); + + pagefault_disable(); + kmap_invalidate_coherent(dst, vaddr); + set_bit(PG_arch_1, &dst->flags); + copy_page_alias(dst_vaddr, src_vaddr, dst_paddr, src_paddr); + pagefault_enable(); +} + +#endif /* DCACHE_WAY_SIZE > PAGE_SIZE */ + +#if (DCACHE_WAY_SIZE > PAGE_SIZE) && XCHAL_DCACHE_IS_WRITEBACK + +/* + * Any time the kernel writes to a user page cache page, or it is about to + * read from a page cache page this routine is called. + * + */ + +void flush_dcache_page(struct page *page) +{ + struct address_space *mapping = page_mapping(page); + + /* + * If we have a mapping but the page is not mapped to user-space + * yet, we simply mark this page dirty and defer flushing the + * caches until update_mmu(). + */ + + if (mapping && !mapping_mapped(mapping)) { + if (!test_bit(PG_arch_1, &page->flags)) + set_bit(PG_arch_1, &page->flags); + return; + + } else { + + unsigned long phys = page_to_phys(page); + unsigned long temp = page->index << PAGE_SHIFT; + unsigned long alias = !(DCACHE_ALIAS_EQ(temp, phys)); + unsigned long virt; + + /* + * Flush the page in kernel space and user space. + * Note that we can omit that step if aliasing is not + * an issue, but we do have to synchronize I$ and D$ + * if we have a mapping. + */ + + if (!alias && !mapping) + return; + + virt = TLBTEMP_BASE_1 + (phys & DCACHE_ALIAS_MASK); + __flush_invalidate_dcache_page_alias(virt, phys); + + virt = TLBTEMP_BASE_1 + (temp & DCACHE_ALIAS_MASK); + + if (alias) + __flush_invalidate_dcache_page_alias(virt, phys); + + if (mapping) + __invalidate_icache_page_alias(virt, phys); + } + + /* There shouldn't be an entry in the cache for this page anymore. */ +} + + +/* + * For now, flush the whole cache. FIXME?? + */ + +void local_flush_cache_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end) +{ + __flush_invalidate_dcache_all(); + __invalidate_icache_all(); +} + +/* + * Remove any entry in the cache for this page. + * + * Note that this function is only called for user pages, so use the + * alias versions of the cache flush functions. + */ + +void local_flush_cache_page(struct vm_area_struct *vma, unsigned long address, + unsigned long pfn) +{ + /* Note that we have to use the 'alias' address to avoid multi-hit */ + + unsigned long phys = page_to_phys(pfn_to_page(pfn)); + unsigned long virt = TLBTEMP_BASE_1 + (address & DCACHE_ALIAS_MASK); + + __flush_invalidate_dcache_page_alias(virt, phys); + __invalidate_icache_page_alias(virt, phys); +} + +#endif + +void +update_mmu_cache(struct vm_area_struct * vma, unsigned long addr, pte_t *ptep) +{ + unsigned long pfn = pte_pfn(*ptep); + struct page *page; + + if (!pfn_valid(pfn)) + return; + + page = pfn_to_page(pfn); + + /* Invalidate old entry in TLBs */ + + flush_tlb_page(vma, addr); + +#if (DCACHE_WAY_SIZE > PAGE_SIZE) && XCHAL_DCACHE_IS_WRITEBACK + + if (!PageReserved(page) && test_bit(PG_arch_1, &page->flags)) { + unsigned long phys = page_to_phys(page); + unsigned long tmp; + + tmp = TLBTEMP_BASE_1 + (phys & DCACHE_ALIAS_MASK); + __flush_invalidate_dcache_page_alias(tmp, phys); + tmp = TLBTEMP_BASE_1 + (addr & DCACHE_ALIAS_MASK); + __flush_invalidate_dcache_page_alias(tmp, phys); + __invalidate_icache_page_alias(tmp, phys); + + clear_bit(PG_arch_1, &page->flags); + } +#else + if (!PageReserved(page) && !test_bit(PG_arch_1, &page->flags) + && (vma->vm_flags & VM_EXEC) != 0) { + unsigned long paddr = (unsigned long)kmap_atomic(page); + __flush_dcache_page(paddr); + __invalidate_icache_page(paddr); + set_bit(PG_arch_1, &page->flags); + kunmap_atomic((void *)paddr); + } +#endif +} + +/* + * access_process_vm() has called get_user_pages(), which has done a + * flush_dcache_page() on the page. + */ + +#if (DCACHE_WAY_SIZE > PAGE_SIZE) && XCHAL_DCACHE_IS_WRITEBACK + +void copy_to_user_page(struct vm_area_struct *vma, struct page *page, + unsigned long vaddr, void *dst, const void *src, + unsigned long len) +{ + unsigned long phys = page_to_phys(page); + unsigned long alias = !(DCACHE_ALIAS_EQ(vaddr, phys)); + + /* Flush and invalidate user page if aliased. */ + + if (alias) { + unsigned long t = TLBTEMP_BASE_1 + (vaddr & DCACHE_ALIAS_MASK); + __flush_invalidate_dcache_page_alias(t, phys); + } + + /* Copy data */ + + memcpy(dst, src, len); + + /* + * Flush and invalidate kernel page if aliased and synchronize + * data and instruction caches for executable pages. + */ + + if (alias) { + unsigned long t = TLBTEMP_BASE_1 + (vaddr & DCACHE_ALIAS_MASK); + + __flush_invalidate_dcache_range((unsigned long) dst, len); + if ((vma->vm_flags & VM_EXEC) != 0) + __invalidate_icache_page_alias(t, phys); + + } else if ((vma->vm_flags & VM_EXEC) != 0) { + __flush_dcache_range((unsigned long)dst,len); + __invalidate_icache_range((unsigned long) dst, len); + } +} + +extern void copy_from_user_page(struct vm_area_struct *vma, struct page *page, + unsigned long vaddr, void *dst, const void *src, + unsigned long len) +{ + unsigned long phys = page_to_phys(page); + unsigned long alias = !(DCACHE_ALIAS_EQ(vaddr, phys)); + + /* + * Flush user page if aliased. + * (Note: a simply flush would be sufficient) + */ + + if (alias) { + unsigned long t = TLBTEMP_BASE_1 + (vaddr & DCACHE_ALIAS_MASK); + __flush_invalidate_dcache_page_alias(t, phys); + } + + memcpy(dst, src, len); +} + +#endif diff --git a/kernel/arch/xtensa/mm/fault.c b/kernel/arch/xtensa/mm/fault.c new file mode 100644 index 000000000..83a44a33c --- /dev/null +++ b/kernel/arch/xtensa/mm/fault.c @@ -0,0 +1,260 @@ +// TODO VM_EXEC flag work-around, cache aliasing +/* + * arch/xtensa/mm/fault.c + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2001 - 2010 Tensilica Inc. + * + * Chris Zankel <chris@zankel.net> + * Joe Taylor <joe@tensilica.com, joetylr@yahoo.com> + */ + +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/hardirq.h> +#include <linux/uaccess.h> +#include <asm/mmu_context.h> +#include <asm/cacheflush.h> +#include <asm/hardirq.h> +#include <asm/pgalloc.h> + +DEFINE_PER_CPU(unsigned long, asid_cache) = ASID_USER_FIRST; +void bad_page_fault(struct pt_regs*, unsigned long, int); + +#undef DEBUG_PAGE_FAULT + +/* + * This routine handles page faults. It determines the address, + * and the problem, and then passes it off to one of the appropriate + * routines. + * + * Note: does not handle Miss and MultiHit. + */ + +void do_page_fault(struct pt_regs *regs) +{ + struct vm_area_struct * vma; + struct mm_struct *mm = current->mm; + unsigned int exccause = regs->exccause; + unsigned int address = regs->excvaddr; + siginfo_t info; + + int is_write, is_exec; + int fault; + unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + + info.si_code = SEGV_MAPERR; + + /* We fault-in kernel-space virtual memory on-demand. The + * 'reference' page table is init_mm.pgd. + */ + if (address >= TASK_SIZE && !user_mode(regs)) + goto vmalloc_fault; + + /* If we're in an interrupt or have no user + * context, we must not take the fault.. + */ + if (faulthandler_disabled() || !mm) { + bad_page_fault(regs, address, SIGSEGV); + return; + } + + is_write = (exccause == EXCCAUSE_STORE_CACHE_ATTRIBUTE) ? 1 : 0; + is_exec = (exccause == EXCCAUSE_ITLB_PRIVILEGE || + exccause == EXCCAUSE_ITLB_MISS || + exccause == EXCCAUSE_FETCH_CACHE_ATTRIBUTE) ? 1 : 0; + +#ifdef DEBUG_PAGE_FAULT + printk("[%s:%d:%08x:%d:%08x:%s%s]\n", current->comm, current->pid, + address, exccause, regs->pc, is_write? "w":"", is_exec? "x":""); +#endif + + if (user_mode(regs)) + flags |= FAULT_FLAG_USER; +retry: + down_read(&mm->mmap_sem); + vma = find_vma(mm, address); + + if (!vma) + goto bad_area; + if (vma->vm_start <= address) + goto good_area; + if (!(vma->vm_flags & VM_GROWSDOWN)) + goto bad_area; + if (expand_stack(vma, address)) + goto bad_area; + + /* Ok, we have a good vm_area for this memory access, so + * we can handle it.. + */ + +good_area: + info.si_code = SEGV_ACCERR; + + if (is_write) { + if (!(vma->vm_flags & VM_WRITE)) + goto bad_area; + flags |= FAULT_FLAG_WRITE; + } else if (is_exec) { + if (!(vma->vm_flags & VM_EXEC)) + goto bad_area; + } else /* Allow read even from write-only pages. */ + if (!(vma->vm_flags & (VM_READ | VM_WRITE))) + goto bad_area; + + /* If for any reason at all we couldn't handle the fault, + * make sure we exit gracefully rather than endlessly redo + * the fault. + */ + fault = handle_mm_fault(mm, vma, address, flags); + + if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) + return; + + if (unlikely(fault & VM_FAULT_ERROR)) { + if (fault & VM_FAULT_OOM) + goto out_of_memory; + else if (fault & VM_FAULT_SIGSEGV) + goto bad_area; + else if (fault & VM_FAULT_SIGBUS) + goto do_sigbus; + BUG(); + } + if (flags & FAULT_FLAG_ALLOW_RETRY) { + if (fault & VM_FAULT_MAJOR) + current->maj_flt++; + else + current->min_flt++; + if (fault & VM_FAULT_RETRY) { + flags &= ~FAULT_FLAG_ALLOW_RETRY; + flags |= FAULT_FLAG_TRIED; + + /* No need to up_read(&mm->mmap_sem) as we would + * have already released it in __lock_page_or_retry + * in mm/filemap.c. + */ + + goto retry; + } + } + + up_read(&mm->mmap_sem); + return; + + /* Something tried to access memory that isn't in our memory map.. + * Fix it, but check if it's kernel or user first.. + */ +bad_area: + up_read(&mm->mmap_sem); + if (user_mode(regs)) { + current->thread.bad_vaddr = address; + current->thread.error_code = is_write; + info.si_signo = SIGSEGV; + info.si_errno = 0; + /* info.si_code has been set above */ + info.si_addr = (void *) address; + force_sig_info(SIGSEGV, &info, current); + return; + } + bad_page_fault(regs, address, SIGSEGV); + return; + + + /* We ran out of memory, or some other thing happened to us that made + * us unable to handle the page fault gracefully. + */ +out_of_memory: + up_read(&mm->mmap_sem); + if (!user_mode(regs)) + bad_page_fault(regs, address, SIGKILL); + else + pagefault_out_of_memory(); + return; + +do_sigbus: + up_read(&mm->mmap_sem); + + /* Send a sigbus, regardless of whether we were in kernel + * or user mode. + */ + current->thread.bad_vaddr = address; + info.si_code = SIGBUS; + info.si_errno = 0; + info.si_code = BUS_ADRERR; + info.si_addr = (void *) address; + force_sig_info(SIGBUS, &info, current); + + /* Kernel mode? Handle exceptions or die */ + if (!user_mode(regs)) + bad_page_fault(regs, address, SIGBUS); + return; + +vmalloc_fault: + { + /* Synchronize this task's top level page-table + * with the 'reference' page table. + */ + struct mm_struct *act_mm = current->active_mm; + int index = pgd_index(address); + pgd_t *pgd, *pgd_k; + pmd_t *pmd, *pmd_k; + pte_t *pte_k; + + if (act_mm == NULL) + goto bad_page_fault; + + pgd = act_mm->pgd + index; + pgd_k = init_mm.pgd + index; + + if (!pgd_present(*pgd_k)) + goto bad_page_fault; + + pgd_val(*pgd) = pgd_val(*pgd_k); + + pmd = pmd_offset(pgd, address); + pmd_k = pmd_offset(pgd_k, address); + if (!pmd_present(*pmd) || !pmd_present(*pmd_k)) + goto bad_page_fault; + + pmd_val(*pmd) = pmd_val(*pmd_k); + pte_k = pte_offset_kernel(pmd_k, address); + + if (!pte_present(*pte_k)) + goto bad_page_fault; + return; + } +bad_page_fault: + bad_page_fault(regs, address, SIGKILL); + return; +} + + +void +bad_page_fault(struct pt_regs *regs, unsigned long address, int sig) +{ + extern void die(const char*, struct pt_regs*, long); + const struct exception_table_entry *entry; + + /* Are we prepared to handle this kernel fault? */ + if ((entry = search_exception_tables(regs->pc)) != NULL) { +#ifdef DEBUG_PAGE_FAULT + printk(KERN_DEBUG "%s: Exception at pc=%#010lx (%lx)\n", + current->comm, regs->pc, entry->fixup); +#endif + current->thread.bad_uaddr = address; + regs->pc = entry->fixup; + return; + } + + /* Oops. The kernel tried to access some bad page. We'll have to + * terminate things with extreme prejudice. + */ + printk(KERN_ALERT "Unable to handle kernel paging request at virtual " + "address %08lx\n pc = %08lx, ra = %08lx\n", + address, regs->pc, regs->areg[0]); + die("Oops", regs, sig); + do_exit(sig); +} diff --git a/kernel/arch/xtensa/mm/highmem.c b/kernel/arch/xtensa/mm/highmem.c new file mode 100644 index 000000000..184ceadcc --- /dev/null +++ b/kernel/arch/xtensa/mm/highmem.c @@ -0,0 +1,95 @@ +/* + * High memory support for Xtensa architecture + * + * This file is subject to the terms and conditions of the GNU General + * Public License. See the file "COPYING" in the main directory of + * this archive for more details. + * + * Copyright (C) 2014 Cadence Design Systems Inc. + */ + +#include <linux/export.h> +#include <linux/highmem.h> +#include <asm/tlbflush.h> + +static pte_t *kmap_pte; + +#if DCACHE_WAY_SIZE > PAGE_SIZE +unsigned int last_pkmap_nr_arr[DCACHE_N_COLORS]; +wait_queue_head_t pkmap_map_wait_arr[DCACHE_N_COLORS]; + +static void __init kmap_waitqueues_init(void) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(pkmap_map_wait_arr); ++i) + init_waitqueue_head(pkmap_map_wait_arr + i); +} +#else +static inline void kmap_waitqueues_init(void) +{ +} +#endif + +static inline enum fixed_addresses kmap_idx(int type, unsigned long color) +{ + return (type + KM_TYPE_NR * smp_processor_id()) * DCACHE_N_COLORS + + color; +} + +void *kmap_atomic(struct page *page) +{ + enum fixed_addresses idx; + unsigned long vaddr; + + preempt_disable(); + pagefault_disable(); + if (!PageHighMem(page)) + return page_address(page); + + idx = kmap_idx(kmap_atomic_idx_push(), + DCACHE_ALIAS(page_to_phys(page))); + vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); +#ifdef CONFIG_DEBUG_HIGHMEM + BUG_ON(!pte_none(*(kmap_pte + idx))); +#endif + set_pte(kmap_pte + idx, mk_pte(page, PAGE_KERNEL_EXEC)); + + return (void *)vaddr; +} +EXPORT_SYMBOL(kmap_atomic); + +void __kunmap_atomic(void *kvaddr) +{ + if (kvaddr >= (void *)FIXADDR_START && + kvaddr < (void *)FIXADDR_TOP) { + int idx = kmap_idx(kmap_atomic_idx(), + DCACHE_ALIAS((unsigned long)kvaddr)); + + /* + * Force other mappings to Oops if they'll try to access this + * pte without first remap it. Keeping stale mappings around + * is a bad idea also, in case the page changes cacheability + * attributes or becomes a protected page in a hypervisor. + */ + pte_clear(&init_mm, kvaddr, kmap_pte + idx); + local_flush_tlb_kernel_range((unsigned long)kvaddr, + (unsigned long)kvaddr + PAGE_SIZE); + + kmap_atomic_idx_pop(); + } + + pagefault_enable(); + preempt_enable(); +} +EXPORT_SYMBOL(__kunmap_atomic); + +void __init kmap_init(void) +{ + unsigned long kmap_vstart; + + /* cache the first kmap pte */ + kmap_vstart = __fix_to_virt(FIX_KMAP_BEGIN); + kmap_pte = kmap_get_fixmap_pte(kmap_vstart); + kmap_waitqueues_init(); +} diff --git a/kernel/arch/xtensa/mm/init.c b/kernel/arch/xtensa/mm/init.c new file mode 100644 index 000000000..9a9a5935b --- /dev/null +++ b/kernel/arch/xtensa/mm/init.c @@ -0,0 +1,430 @@ +/* + * arch/xtensa/mm/init.c + * + * Derived from MIPS, PPC. + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2001 - 2005 Tensilica Inc. + * Copyright (C) 2014 Cadence Design Systems Inc. + * + * Chris Zankel <chris@zankel.net> + * Joe Taylor <joe@tensilica.com, joetylr@yahoo.com> + * Marc Gauthier + * Kevin Chea + */ + +#include <linux/kernel.h> +#include <linux/errno.h> +#include <linux/bootmem.h> +#include <linux/gfp.h> +#include <linux/highmem.h> +#include <linux/swap.h> +#include <linux/mman.h> +#include <linux/nodemask.h> +#include <linux/mm.h> + +#include <asm/bootparam.h> +#include <asm/page.h> +#include <asm/sections.h> +#include <asm/sysmem.h> + +struct sysmem_info sysmem __initdata; + +static void __init sysmem_dump(void) +{ + unsigned i; + + pr_debug("Sysmem:\n"); + for (i = 0; i < sysmem.nr_banks; ++i) + pr_debug(" 0x%08lx - 0x%08lx (%ldK)\n", + sysmem.bank[i].start, sysmem.bank[i].end, + (sysmem.bank[i].end - sysmem.bank[i].start) >> 10); +} + +/* + * Find bank with maximal .start such that bank.start <= start + */ +static inline struct meminfo * __init find_bank(unsigned long start) +{ + unsigned i; + struct meminfo *it = NULL; + + for (i = 0; i < sysmem.nr_banks; ++i) + if (sysmem.bank[i].start <= start) + it = sysmem.bank + i; + else + break; + return it; +} + +/* + * Move all memory banks starting at 'from' to a new place at 'to', + * adjust nr_banks accordingly. + * Both 'from' and 'to' must be inside the sysmem.bank. + * + * Returns: 0 (success), -ENOMEM (not enough space in the sysmem.bank). + */ +static int __init move_banks(struct meminfo *to, struct meminfo *from) +{ + unsigned n = sysmem.nr_banks - (from - sysmem.bank); + + if (to > from && to - from + sysmem.nr_banks > SYSMEM_BANKS_MAX) + return -ENOMEM; + if (to != from) + memmove(to, from, n * sizeof(struct meminfo)); + sysmem.nr_banks += to - from; + return 0; +} + +/* + * Add new bank to sysmem. Resulting sysmem is the union of bytes of the + * original sysmem and the new bank. + * + * Returns: 0 (success), < 0 (error) + */ +int __init add_sysmem_bank(unsigned long start, unsigned long end) +{ + unsigned i; + struct meminfo *it = NULL; + unsigned long sz; + unsigned long bank_sz = 0; + + if (start == end || + (start < end) != (PAGE_ALIGN(start) < (end & PAGE_MASK))) { + pr_warn("Ignoring small memory bank 0x%08lx size: %ld bytes\n", + start, end - start); + return -EINVAL; + } + + start = PAGE_ALIGN(start); + end &= PAGE_MASK; + sz = end - start; + + it = find_bank(start); + + if (it) + bank_sz = it->end - it->start; + + if (it && bank_sz >= start - it->start) { + if (end - it->start > bank_sz) + it->end = end; + else + return 0; + } else { + if (!it) + it = sysmem.bank; + else + ++it; + + if (it - sysmem.bank < sysmem.nr_banks && + it->start - start <= sz) { + it->start = start; + if (it->end - it->start < sz) + it->end = end; + else + return 0; + } else { + if (move_banks(it + 1, it) < 0) { + pr_warn("Ignoring memory bank 0x%08lx size %ld bytes\n", + start, end - start); + return -EINVAL; + } + it->start = start; + it->end = end; + return 0; + } + } + sz = it->end - it->start; + for (i = it + 1 - sysmem.bank; i < sysmem.nr_banks; ++i) + if (sysmem.bank[i].start - it->start <= sz) { + if (sz < sysmem.bank[i].end - it->start) + it->end = sysmem.bank[i].end; + } else { + break; + } + + move_banks(it + 1, sysmem.bank + i); + return 0; +} + +/* + * mem_reserve(start, end, must_exist) + * + * Reserve some memory from the memory pool. + * If must_exist is set and a part of the region being reserved does not exist + * memory map is not altered. + * + * Parameters: + * start Start of region, + * end End of region, + * must_exist Must exist in memory pool. + * + * Returns: + * 0 (success) + * < 0 (error) + */ + +int __init mem_reserve(unsigned long start, unsigned long end, int must_exist) +{ + struct meminfo *it; + struct meminfo *rm = NULL; + unsigned long sz; + unsigned long bank_sz = 0; + + start = start & PAGE_MASK; + end = PAGE_ALIGN(end); + sz = end - start; + if (!sz) + return -EINVAL; + + it = find_bank(start); + + if (it) + bank_sz = it->end - it->start; + + if ((!it || end - it->start > bank_sz) && must_exist) { + pr_warn("mem_reserve: [0x%0lx, 0x%0lx) not in any region!\n", + start, end); + return -EINVAL; + } + + if (it && start - it->start <= bank_sz) { + if (start == it->start) { + if (end - it->start < bank_sz) { + it->start = end; + return 0; + } else { + rm = it; + } + } else { + it->end = start; + if (end - it->start < bank_sz) + return add_sysmem_bank(end, + it->start + bank_sz); + ++it; + } + } + + if (!it) + it = sysmem.bank; + + for (; it < sysmem.bank + sysmem.nr_banks; ++it) { + if (it->end - start <= sz) { + if (!rm) + rm = it; + } else { + if (it->start - start < sz) + it->start = end; + break; + } + } + + if (rm) + move_banks(rm, it); + + return 0; +} + + +/* + * Initialize the bootmem system and give it all low memory we have available. + */ + +void __init bootmem_init(void) +{ + unsigned long pfn; + unsigned long bootmap_start, bootmap_size; + int i; + + /* Reserve all memory below PLATFORM_DEFAULT_MEM_START, as memory + * accounting doesn't work for pages below that address. + * + * If PLATFORM_DEFAULT_MEM_START is zero reserve page at address 0: + * successfull allocations should never return NULL. + */ + if (PLATFORM_DEFAULT_MEM_START) + mem_reserve(0, PLATFORM_DEFAULT_MEM_START, 0); + else + mem_reserve(0, 1, 0); + + sysmem_dump(); + max_low_pfn = max_pfn = 0; + min_low_pfn = ~0; + + for (i=0; i < sysmem.nr_banks; i++) { + pfn = PAGE_ALIGN(sysmem.bank[i].start) >> PAGE_SHIFT; + if (pfn < min_low_pfn) + min_low_pfn = pfn; + pfn = PAGE_ALIGN(sysmem.bank[i].end - 1) >> PAGE_SHIFT; + if (pfn > max_pfn) + max_pfn = pfn; + } + + if (min_low_pfn > max_pfn) + panic("No memory found!\n"); + + max_low_pfn = max_pfn < MAX_MEM_PFN >> PAGE_SHIFT ? + max_pfn : MAX_MEM_PFN >> PAGE_SHIFT; + + /* Find an area to use for the bootmem bitmap. */ + + bootmap_size = bootmem_bootmap_pages(max_low_pfn - min_low_pfn); + bootmap_size <<= PAGE_SHIFT; + bootmap_start = ~0; + + for (i=0; i<sysmem.nr_banks; i++) + if (sysmem.bank[i].end - sysmem.bank[i].start >= bootmap_size) { + bootmap_start = sysmem.bank[i].start; + break; + } + + if (bootmap_start == ~0UL) + panic("Cannot find %ld bytes for bootmap\n", bootmap_size); + + /* Reserve the bootmem bitmap area */ + + mem_reserve(bootmap_start, bootmap_start + bootmap_size, 1); + bootmap_size = init_bootmem_node(NODE_DATA(0), + bootmap_start >> PAGE_SHIFT, + min_low_pfn, + max_low_pfn); + + /* Add all remaining memory pieces into the bootmem map */ + + for (i = 0; i < sysmem.nr_banks; i++) { + if (sysmem.bank[i].start >> PAGE_SHIFT < max_low_pfn) { + unsigned long end = min(max_low_pfn << PAGE_SHIFT, + sysmem.bank[i].end); + free_bootmem(sysmem.bank[i].start, + end - sysmem.bank[i].start); + } + } + +} + + +void __init zones_init(void) +{ + /* All pages are DMA-able, so we put them all in the DMA zone. */ + unsigned long zones_size[MAX_NR_ZONES] = { + [ZONE_DMA] = max_low_pfn - ARCH_PFN_OFFSET, +#ifdef CONFIG_HIGHMEM + [ZONE_HIGHMEM] = max_pfn - max_low_pfn, +#endif + }; + free_area_init_node(0, zones_size, ARCH_PFN_OFFSET, NULL); +} + +/* + * Initialize memory pages. + */ + +void __init mem_init(void) +{ +#ifdef CONFIG_HIGHMEM + unsigned long tmp; + + reset_all_zones_managed_pages(); + for (tmp = max_low_pfn; tmp < max_pfn; tmp++) + free_highmem_page(pfn_to_page(tmp)); +#endif + + max_mapnr = max_pfn - ARCH_PFN_OFFSET; + high_memory = (void *)__va(max_low_pfn << PAGE_SHIFT); + + free_all_bootmem(); + + mem_init_print_info(NULL); + pr_info("virtual kernel memory layout:\n" +#ifdef CONFIG_HIGHMEM + " pkmap : 0x%08lx - 0x%08lx (%5lu kB)\n" + " fixmap : 0x%08lx - 0x%08lx (%5lu kB)\n" +#endif +#ifdef CONFIG_MMU + " vmalloc : 0x%08x - 0x%08x (%5u MB)\n" +#endif + " lowmem : 0x%08lx - 0x%08lx (%5lu MB)\n", +#ifdef CONFIG_HIGHMEM + PKMAP_BASE, PKMAP_BASE + LAST_PKMAP * PAGE_SIZE, + (LAST_PKMAP*PAGE_SIZE) >> 10, + FIXADDR_START, FIXADDR_TOP, + (FIXADDR_TOP - FIXADDR_START) >> 10, +#endif +#ifdef CONFIG_MMU + VMALLOC_START, VMALLOC_END, + (VMALLOC_END - VMALLOC_START) >> 20, + PAGE_OFFSET, PAGE_OFFSET + + (max_low_pfn - min_low_pfn) * PAGE_SIZE, +#else + min_low_pfn * PAGE_SIZE, max_low_pfn * PAGE_SIZE, +#endif + ((max_low_pfn - min_low_pfn) * PAGE_SIZE) >> 20); +} + +#ifdef CONFIG_BLK_DEV_INITRD +extern int initrd_is_mapped; + +void free_initrd_mem(unsigned long start, unsigned long end) +{ + if (initrd_is_mapped) + free_reserved_area((void *)start, (void *)end, -1, "initrd"); +} +#endif + +void free_initmem(void) +{ + free_initmem_default(-1); +} + +static void __init parse_memmap_one(char *p) +{ + char *oldp; + unsigned long start_at, mem_size; + + if (!p) + return; + + oldp = p; + mem_size = memparse(p, &p); + if (p == oldp) + return; + + switch (*p) { + case '@': + start_at = memparse(p + 1, &p); + add_sysmem_bank(start_at, start_at + mem_size); + break; + + case '$': + start_at = memparse(p + 1, &p); + mem_reserve(start_at, start_at + mem_size, 0); + break; + + case 0: + mem_reserve(mem_size, 0, 0); + break; + + default: + pr_warn("Unrecognized memmap syntax: %s\n", p); + break; + } +} + +static int __init parse_memmap_opt(char *str) +{ + while (str) { + char *k = strchr(str, ','); + + if (k) + *k++ = 0; + + parse_memmap_one(str); + str = k; + } + + return 0; +} +early_param("memmap", parse_memmap_opt); diff --git a/kernel/arch/xtensa/mm/misc.S b/kernel/arch/xtensa/mm/misc.S new file mode 100644 index 000000000..11a01c3e9 --- /dev/null +++ b/kernel/arch/xtensa/mm/misc.S @@ -0,0 +1,484 @@ +/* + * arch/xtensa/mm/misc.S + * + * Miscellaneous assembly functions. + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2001 - 2007 Tensilica Inc. + * + * Chris Zankel <chris@zankel.net> + */ + + +#include <linux/linkage.h> +#include <asm/page.h> +#include <asm/pgtable.h> +#include <asm/asmmacro.h> +#include <asm/cacheasm.h> +#include <asm/tlbflush.h> + + +/* + * clear_page and clear_user_page are the same for non-cache-aliased configs. + * + * clear_page (unsigned long page) + * a2 + */ + +ENTRY(clear_page) + + entry a1, 16 + + movi a3, 0 + __loopi a2, a7, PAGE_SIZE, 32 + s32i a3, a2, 0 + s32i a3, a2, 4 + s32i a3, a2, 8 + s32i a3, a2, 12 + s32i a3, a2, 16 + s32i a3, a2, 20 + s32i a3, a2, 24 + s32i a3, a2, 28 + __endla a2, a7, 32 + + retw + +ENDPROC(clear_page) + +/* + * copy_page and copy_user_page are the same for non-cache-aliased configs. + * + * copy_page (void *to, void *from) + * a2 a3 + */ + +ENTRY(copy_page) + + entry a1, 16 + + __loopi a2, a4, PAGE_SIZE, 32 + + l32i a8, a3, 0 + l32i a9, a3, 4 + s32i a8, a2, 0 + s32i a9, a2, 4 + + l32i a8, a3, 8 + l32i a9, a3, 12 + s32i a8, a2, 8 + s32i a9, a2, 12 + + l32i a8, a3, 16 + l32i a9, a3, 20 + s32i a8, a2, 16 + s32i a9, a2, 20 + + l32i a8, a3, 24 + l32i a9, a3, 28 + s32i a8, a2, 24 + s32i a9, a2, 28 + + addi a2, a2, 32 + addi a3, a3, 32 + + __endl a2, a4 + + retw + +ENDPROC(copy_page) + +#ifdef CONFIG_MMU +/* + * If we have to deal with cache aliasing, we use temporary memory mappings + * to ensure that the source and destination pages have the same color as + * the virtual address. We use way 0 and 1 for temporary mappings in such cases. + * + * The temporary DTLB entries shouldn't be flushed by interrupts, but are + * flushed by preemptive task switches. Special code in the + * fast_second_level_miss handler re-established the temporary mapping. + * It requires that the PPNs for the destination and source addresses are + * in a6, and a7, respectively. + */ + +/* TLB miss exceptions are treated special in the following region */ + +ENTRY(__tlbtemp_mapping_start) + +#if (DCACHE_WAY_SIZE > PAGE_SIZE) + +/* + * clear_page_alias(void *addr, unsigned long paddr) + * a2 a3 + */ + +ENTRY(clear_page_alias) + + entry a1, 32 + + /* Skip setting up a temporary DTLB if not aliased low page. */ + + movi a5, PAGE_OFFSET + movi a6, 0 + beqz a3, 1f + + /* Setup a temporary DTLB for the addr. */ + + addi a6, a3, (PAGE_KERNEL | _PAGE_HW_WRITE) + mov a4, a2 + wdtlb a6, a2 + dsync + +1: movi a3, 0 + __loopi a2, a7, PAGE_SIZE, 32 + s32i a3, a2, 0 + s32i a3, a2, 4 + s32i a3, a2, 8 + s32i a3, a2, 12 + s32i a3, a2, 16 + s32i a3, a2, 20 + s32i a3, a2, 24 + s32i a3, a2, 28 + __endla a2, a7, 32 + + bnez a6, 1f + retw + + /* We need to invalidate the temporary idtlb entry, if any. */ + +1: idtlb a4 + dsync + + retw + +ENDPROC(clear_page_alias) + +/* + * copy_page_alias(void *to, void *from, + * a2 a3 + * unsigned long to_paddr, unsigned long from_paddr) + * a4 a5 + */ + +ENTRY(copy_page_alias) + + entry a1, 32 + + /* Skip setting up a temporary DTLB for destination if not aliased. */ + + movi a6, 0 + movi a7, 0 + beqz a4, 1f + + /* Setup a temporary DTLB for destination. */ + + addi a6, a4, (PAGE_KERNEL | _PAGE_HW_WRITE) + wdtlb a6, a2 + dsync + + /* Skip setting up a temporary DTLB for source if not aliased. */ + +1: beqz a5, 1f + + /* Setup a temporary DTLB for source. */ + + addi a7, a5, PAGE_KERNEL + addi a8, a3, 1 # way1 + + wdtlb a7, a8 + dsync + +1: __loopi a2, a4, PAGE_SIZE, 32 + + l32i a8, a3, 0 + l32i a9, a3, 4 + s32i a8, a2, 0 + s32i a9, a2, 4 + + l32i a8, a3, 8 + l32i a9, a3, 12 + s32i a8, a2, 8 + s32i a9, a2, 12 + + l32i a8, a3, 16 + l32i a9, a3, 20 + s32i a8, a2, 16 + s32i a9, a2, 20 + + l32i a8, a3, 24 + l32i a9, a3, 28 + s32i a8, a2, 24 + s32i a9, a2, 28 + + addi a2, a2, 32 + addi a3, a3, 32 + + __endl a2, a4 + + /* We need to invalidate any temporary mapping! */ + + bnez a6, 1f + bnez a7, 2f + retw + +1: addi a2, a2, -PAGE_SIZE + idtlb a2 + dsync + bnez a7, 2f + retw + +2: addi a3, a3, -PAGE_SIZE+1 + idtlb a3 + dsync + + retw + +ENDPROC(copy_page_alias) + +#endif + +#if (DCACHE_WAY_SIZE > PAGE_SIZE) + +/* + * void __flush_invalidate_dcache_page_alias (addr, phys) + * a2 a3 + */ + +ENTRY(__flush_invalidate_dcache_page_alias) + + entry sp, 16 + + movi a7, 0 # required for exception handler + addi a6, a3, (PAGE_KERNEL | _PAGE_HW_WRITE) + mov a4, a2 + wdtlb a6, a2 + dsync + + ___flush_invalidate_dcache_page a2 a3 + + idtlb a4 + dsync + + retw + +ENDPROC(__flush_invalidate_dcache_page_alias) + +/* + * void __invalidate_dcache_page_alias (addr, phys) + * a2 a3 + */ + +ENTRY(__invalidate_dcache_page_alias) + + entry sp, 16 + + movi a7, 0 # required for exception handler + addi a6, a3, (PAGE_KERNEL | _PAGE_HW_WRITE) + mov a4, a2 + wdtlb a6, a2 + dsync + + ___invalidate_dcache_page a2 a3 + + idtlb a4 + dsync + + retw + +ENDPROC(__invalidate_dcache_page_alias) +#endif + +ENTRY(__tlbtemp_mapping_itlb) + +#if (ICACHE_WAY_SIZE > PAGE_SIZE) + +ENTRY(__invalidate_icache_page_alias) + + entry sp, 16 + + addi a6, a3, (PAGE_KERNEL_EXEC | _PAGE_HW_WRITE) + mov a4, a2 + witlb a6, a2 + isync + + ___invalidate_icache_page a2 a3 + + iitlb a4 + isync + retw + +ENDPROC(__invalidate_icache_page_alias) + +#endif + +/* End of special treatment in tlb miss exception */ + +ENTRY(__tlbtemp_mapping_end) + +#endif /* CONFIG_MMU + +/* + * void __invalidate_icache_page(ulong start) + */ + +ENTRY(__invalidate_icache_page) + + entry sp, 16 + + ___invalidate_icache_page a2 a3 + isync + + retw + +ENDPROC(__invalidate_icache_page) + +/* + * void __invalidate_dcache_page(ulong start) + */ + +ENTRY(__invalidate_dcache_page) + + entry sp, 16 + + ___invalidate_dcache_page a2 a3 + dsync + + retw + +ENDPROC(__invalidate_dcache_page) + +/* + * void __flush_invalidate_dcache_page(ulong start) + */ + +ENTRY(__flush_invalidate_dcache_page) + + entry sp, 16 + + ___flush_invalidate_dcache_page a2 a3 + + dsync + retw + +ENDPROC(__flush_invalidate_dcache_page) + +/* + * void __flush_dcache_page(ulong start) + */ + +ENTRY(__flush_dcache_page) + + entry sp, 16 + + ___flush_dcache_page a2 a3 + + dsync + retw + +ENDPROC(__flush_dcache_page) + +/* + * void __invalidate_icache_range(ulong start, ulong size) + */ + +ENTRY(__invalidate_icache_range) + + entry sp, 16 + + ___invalidate_icache_range a2 a3 a4 + isync + + retw + +ENDPROC(__invalidate_icache_range) + +/* + * void __flush_invalidate_dcache_range(ulong start, ulong size) + */ + +ENTRY(__flush_invalidate_dcache_range) + + entry sp, 16 + + ___flush_invalidate_dcache_range a2 a3 a4 + dsync + + retw + +ENDPROC(__flush_invalidate_dcache_range) + +/* + * void _flush_dcache_range(ulong start, ulong size) + */ + +ENTRY(__flush_dcache_range) + + entry sp, 16 + + ___flush_dcache_range a2 a3 a4 + dsync + + retw + +ENDPROC(__flush_dcache_range) + +/* + * void _invalidate_dcache_range(ulong start, ulong size) + */ + +ENTRY(__invalidate_dcache_range) + + entry sp, 16 + + ___invalidate_dcache_range a2 a3 a4 + + retw + +ENDPROC(__invalidate_dcache_range) + +/* + * void _invalidate_icache_all(void) + */ + +ENTRY(__invalidate_icache_all) + + entry sp, 16 + + ___invalidate_icache_all a2 a3 + isync + + retw + +ENDPROC(__invalidate_icache_all) + +/* + * void _flush_invalidate_dcache_all(void) + */ + +ENTRY(__flush_invalidate_dcache_all) + + entry sp, 16 + + ___flush_invalidate_dcache_all a2 a3 + dsync + + retw + +ENDPROC(__flush_invalidate_dcache_all) + +/* + * void _invalidate_dcache_all(void) + */ + +ENTRY(__invalidate_dcache_all) + + entry sp, 16 + + ___invalidate_dcache_all a2 a3 + dsync + + retw + +ENDPROC(__invalidate_dcache_all) diff --git a/kernel/arch/xtensa/mm/mmu.c b/kernel/arch/xtensa/mm/mmu.c new file mode 100644 index 000000000..abe4513eb --- /dev/null +++ b/kernel/arch/xtensa/mm/mmu.c @@ -0,0 +1,110 @@ +/* + * xtensa mmu stuff + * + * Extracted from init.c + */ +#include <linux/bootmem.h> +#include <linux/percpu.h> +#include <linux/init.h> +#include <linux/string.h> +#include <linux/slab.h> +#include <linux/cache.h> + +#include <asm/tlb.h> +#include <asm/tlbflush.h> +#include <asm/mmu_context.h> +#include <asm/page.h> +#include <asm/initialize_mmu.h> +#include <asm/io.h> + +#if defined(CONFIG_HIGHMEM) +static void * __init init_pmd(unsigned long vaddr, unsigned long n_pages) +{ + pgd_t *pgd = pgd_offset_k(vaddr); + pmd_t *pmd = pmd_offset(pgd, vaddr); + pte_t *pte; + unsigned long i; + + n_pages = ALIGN(n_pages, PTRS_PER_PTE); + + pr_debug("%s: vaddr: 0x%08lx, n_pages: %ld\n", + __func__, vaddr, n_pages); + + pte = alloc_bootmem_low_pages(n_pages * sizeof(pte_t)); + + for (i = 0; i < n_pages; ++i) + pte_clear(NULL, 0, pte + i); + + for (i = 0; i < n_pages; i += PTRS_PER_PTE, ++pmd) { + pte_t *cur_pte = pte + i; + + BUG_ON(!pmd_none(*pmd)); + set_pmd(pmd, __pmd(((unsigned long)cur_pte) & PAGE_MASK)); + BUG_ON(cur_pte != pte_offset_kernel(pmd, 0)); + pr_debug("%s: pmd: 0x%p, pte: 0x%p\n", + __func__, pmd, cur_pte); + } + return pte; +} + +static void __init fixedrange_init(void) +{ + init_pmd(__fix_to_virt(0), __end_of_fixed_addresses); +} +#endif + +void __init paging_init(void) +{ + memset(swapper_pg_dir, 0, PAGE_SIZE); +#ifdef CONFIG_HIGHMEM + fixedrange_init(); + pkmap_page_table = init_pmd(PKMAP_BASE, LAST_PKMAP); + kmap_init(); +#endif +} + +/* + * Flush the mmu and reset associated register to default values. + */ +void init_mmu(void) +{ +#if !(XCHAL_HAVE_PTP_MMU && XCHAL_HAVE_SPANNING_WAY) + /* + * Writing zeros to the instruction and data TLBCFG special + * registers ensure that valid values exist in the register. + * + * For existing PGSZID<w> fields, zero selects the first element + * of the page-size array. For nonexistent PGSZID<w> fields, + * zero is the best value to write. Also, when changing PGSZID<w> + * fields, the corresponding TLB must be flushed. + */ + set_itlbcfg_register(0); + set_dtlbcfg_register(0); +#endif +#if XCHAL_HAVE_PTP_MMU && XCHAL_HAVE_SPANNING_WAY && defined(CONFIG_OF) + /* + * Update the IO area mapping in case xtensa_kio_paddr has changed + */ + write_dtlb_entry(__pte(xtensa_kio_paddr + CA_WRITEBACK), + XCHAL_KIO_CACHED_VADDR + 6); + write_itlb_entry(__pte(xtensa_kio_paddr + CA_WRITEBACK), + XCHAL_KIO_CACHED_VADDR + 6); + write_dtlb_entry(__pte(xtensa_kio_paddr + CA_BYPASS), + XCHAL_KIO_BYPASS_VADDR + 6); + write_itlb_entry(__pte(xtensa_kio_paddr + CA_BYPASS), + XCHAL_KIO_BYPASS_VADDR + 6); +#endif + + local_flush_tlb_all(); + + /* Set rasid register to a known value. */ + + set_rasid_register(ASID_INSERT(ASID_USER_FIRST)); + + /* Set PTEVADDR special register to the start of the page + * table, which is in kernel mappable space (ie. not + * statically mapped). This register's value is undefined on + * reset. + */ + set_ptevaddr_register(PGTABLE_START); +} diff --git a/kernel/arch/xtensa/mm/tlb.c b/kernel/arch/xtensa/mm/tlb.c new file mode 100644 index 000000000..5ece856c5 --- /dev/null +++ b/kernel/arch/xtensa/mm/tlb.c @@ -0,0 +1,278 @@ +/* + * arch/xtensa/mm/tlb.c + * + * Logic that manipulates the Xtensa MMU. Derived from MIPS. + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2001 - 2003 Tensilica Inc. + * + * Joe Taylor + * Chris Zankel <chris@zankel.net> + * Marc Gauthier + */ + +#include <linux/mm.h> +#include <asm/processor.h> +#include <asm/mmu_context.h> +#include <asm/tlbflush.h> +#include <asm/cacheflush.h> + + +static inline void __flush_itlb_all (void) +{ + int w, i; + + for (w = 0; w < ITLB_ARF_WAYS; w++) { + for (i = 0; i < (1 << XCHAL_ITLB_ARF_ENTRIES_LOG2); i++) { + int e = w + (i << PAGE_SHIFT); + invalidate_itlb_entry_no_isync(e); + } + } + asm volatile ("isync\n"); +} + +static inline void __flush_dtlb_all (void) +{ + int w, i; + + for (w = 0; w < DTLB_ARF_WAYS; w++) { + for (i = 0; i < (1 << XCHAL_DTLB_ARF_ENTRIES_LOG2); i++) { + int e = w + (i << PAGE_SHIFT); + invalidate_dtlb_entry_no_isync(e); + } + } + asm volatile ("isync\n"); +} + + +void local_flush_tlb_all(void) +{ + __flush_itlb_all(); + __flush_dtlb_all(); +} + +/* If mm is current, we simply assign the current task a new ASID, thus, + * invalidating all previous tlb entries. If mm is someone else's user mapping, + * wie invalidate the context, thus, when that user mapping is swapped in, + * a new context will be assigned to it. + */ + +void local_flush_tlb_mm(struct mm_struct *mm) +{ + int cpu = smp_processor_id(); + + if (mm == current->active_mm) { + unsigned long flags; + local_irq_save(flags); + mm->context.asid[cpu] = NO_CONTEXT; + activate_context(mm, cpu); + local_irq_restore(flags); + } else { + mm->context.asid[cpu] = NO_CONTEXT; + mm->context.cpu = -1; + } +} + + +#define _ITLB_ENTRIES (ITLB_ARF_WAYS << XCHAL_ITLB_ARF_ENTRIES_LOG2) +#define _DTLB_ENTRIES (DTLB_ARF_WAYS << XCHAL_DTLB_ARF_ENTRIES_LOG2) +#if _ITLB_ENTRIES > _DTLB_ENTRIES +# define _TLB_ENTRIES _ITLB_ENTRIES +#else +# define _TLB_ENTRIES _DTLB_ENTRIES +#endif + +void local_flush_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end) +{ + int cpu = smp_processor_id(); + struct mm_struct *mm = vma->vm_mm; + unsigned long flags; + + if (mm->context.asid[cpu] == NO_CONTEXT) + return; + +#if 0 + printk("[tlbrange<%02lx,%08lx,%08lx>]\n", + (unsigned long)mm->context.asid[cpu], start, end); +#endif + local_irq_save(flags); + + if (end-start + (PAGE_SIZE-1) <= _TLB_ENTRIES << PAGE_SHIFT) { + int oldpid = get_rasid_register(); + + set_rasid_register(ASID_INSERT(mm->context.asid[cpu])); + start &= PAGE_MASK; + if (vma->vm_flags & VM_EXEC) + while(start < end) { + invalidate_itlb_mapping(start); + invalidate_dtlb_mapping(start); + start += PAGE_SIZE; + } + else + while(start < end) { + invalidate_dtlb_mapping(start); + start += PAGE_SIZE; + } + + set_rasid_register(oldpid); + } else { + local_flush_tlb_mm(mm); + } + local_irq_restore(flags); +} + +void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page) +{ + int cpu = smp_processor_id(); + struct mm_struct* mm = vma->vm_mm; + unsigned long flags; + int oldpid; + + if (mm->context.asid[cpu] == NO_CONTEXT) + return; + + local_irq_save(flags); + + oldpid = get_rasid_register(); + set_rasid_register(ASID_INSERT(mm->context.asid[cpu])); + + if (vma->vm_flags & VM_EXEC) + invalidate_itlb_mapping(page); + invalidate_dtlb_mapping(page); + + set_rasid_register(oldpid); + + local_irq_restore(flags); +} + +void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) +{ + if (end > start && start >= TASK_SIZE && end <= PAGE_OFFSET && + end - start < _TLB_ENTRIES << PAGE_SHIFT) { + start &= PAGE_MASK; + while (start < end) { + invalidate_itlb_mapping(start); + invalidate_dtlb_mapping(start); + start += PAGE_SIZE; + } + } else { + local_flush_tlb_all(); + } +} + +#ifdef CONFIG_DEBUG_TLB_SANITY + +static unsigned get_pte_for_vaddr(unsigned vaddr) +{ + struct task_struct *task = get_current(); + struct mm_struct *mm = task->mm; + pgd_t *pgd; + pmd_t *pmd; + pte_t *pte; + + if (!mm) + mm = task->active_mm; + pgd = pgd_offset(mm, vaddr); + if (pgd_none_or_clear_bad(pgd)) + return 0; + pmd = pmd_offset(pgd, vaddr); + if (pmd_none_or_clear_bad(pmd)) + return 0; + pte = pte_offset_map(pmd, vaddr); + if (!pte) + return 0; + return pte_val(*pte); +} + +enum { + TLB_SUSPICIOUS = 1, + TLB_INSANE = 2, +}; + +static void tlb_insane(void) +{ + BUG_ON(1); +} + +static void tlb_suspicious(void) +{ + WARN_ON(1); +} + +/* + * Check that TLB entries with kernel ASID (1) have kernel VMA (>= TASK_SIZE), + * and TLB entries with user ASID (>=4) have VMA < TASK_SIZE. + * + * Check that valid TLB entries either have the same PA as the PTE, or PTE is + * marked as non-present. Non-present PTE and the page with non-zero refcount + * and zero mapcount is normal for batched TLB flush operation. Zero refcount + * means that the page was freed prematurely. Non-zero mapcount is unusual, + * but does not necessary means an error, thus marked as suspicious. + */ +static int check_tlb_entry(unsigned w, unsigned e, bool dtlb) +{ + unsigned tlbidx = w | (e << PAGE_SHIFT); + unsigned r0 = dtlb ? + read_dtlb_virtual(tlbidx) : read_itlb_virtual(tlbidx); + unsigned vpn = (r0 & PAGE_MASK) | (e << PAGE_SHIFT); + unsigned pte = get_pte_for_vaddr(vpn); + unsigned mm_asid = (get_rasid_register() >> 8) & ASID_MASK; + unsigned tlb_asid = r0 & ASID_MASK; + bool kernel = tlb_asid == 1; + int rc = 0; + + if (tlb_asid > 0 && ((vpn < TASK_SIZE) == kernel)) { + pr_err("%cTLB: way: %u, entry: %u, VPN %08x in %s PTE\n", + dtlb ? 'D' : 'I', w, e, vpn, + kernel ? "kernel" : "user"); + rc |= TLB_INSANE; + } + + if (tlb_asid == mm_asid) { + unsigned r1 = dtlb ? read_dtlb_translation(tlbidx) : + read_itlb_translation(tlbidx); + if ((pte ^ r1) & PAGE_MASK) { + pr_err("%cTLB: way: %u, entry: %u, mapping: %08x->%08x, PTE: %08x\n", + dtlb ? 'D' : 'I', w, e, r0, r1, pte); + if (pte == 0 || !pte_present(__pte(pte))) { + struct page *p = pfn_to_page(r1 >> PAGE_SHIFT); + pr_err("page refcount: %d, mapcount: %d\n", + page_count(p), + page_mapcount(p)); + if (!page_count(p)) + rc |= TLB_INSANE; + else if (page_mapped(p)) + rc |= TLB_SUSPICIOUS; + } else { + rc |= TLB_INSANE; + } + } + } + return rc; +} + +void check_tlb_sanity(void) +{ + unsigned long flags; + unsigned w, e; + int bug = 0; + + local_irq_save(flags); + for (w = 0; w < DTLB_ARF_WAYS; ++w) + for (e = 0; e < (1 << XCHAL_DTLB_ARF_ENTRIES_LOG2); ++e) + bug |= check_tlb_entry(w, e, true); + for (w = 0; w < ITLB_ARF_WAYS; ++w) + for (e = 0; e < (1 << XCHAL_ITLB_ARF_ENTRIES_LOG2); ++e) + bug |= check_tlb_entry(w, e, false); + if (bug & TLB_INSANE) + tlb_insane(); + if (bug & TLB_SUSPICIOUS) + tlb_suspicious(); + local_irq_restore(flags); +} + +#endif /* CONFIG_DEBUG_TLB_SANITY */ |