diff options
Diffstat (limited to 'kernel/arch/sh/mm/cache-sh5.c')
| -rw-r--r-- | kernel/arch/sh/mm/cache-sh5.c | 621 |
1 files changed, 621 insertions, 0 deletions
diff --git a/kernel/arch/sh/mm/cache-sh5.c b/kernel/arch/sh/mm/cache-sh5.c new file mode 100644 index 000000000..d1bffbcd9 --- /dev/null +++ b/kernel/arch/sh/mm/cache-sh5.c @@ -0,0 +1,621 @@ +/* + * arch/sh/mm/cache-sh5.c + * + * Copyright (C) 2000, 2001 Paolo Alberelli + * Copyright (C) 2002 Benedict Gaster + * Copyright (C) 2003 Richard Curnow + * Copyright (C) 2003 - 2008 Paul Mundt + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ +#include <linux/init.h> +#include <linux/mman.h> +#include <linux/mm.h> +#include <asm/tlb.h> +#include <asm/processor.h> +#include <asm/cache.h> +#include <asm/pgalloc.h> +#include <asm/uaccess.h> +#include <asm/mmu_context.h> + +extern void __weak sh4__flush_region_init(void); + +/* Wired TLB entry for the D-cache */ +static unsigned long long dtlb_cache_slot; + +/* + * The following group of functions deal with mapping and unmapping a + * temporary page into a DTLB slot that has been set aside for exclusive + * use. + */ +static inline void +sh64_setup_dtlb_cache_slot(unsigned long eaddr, unsigned long asid, + unsigned long paddr) +{ + local_irq_disable(); + sh64_setup_tlb_slot(dtlb_cache_slot, eaddr, asid, paddr); +} + +static inline void sh64_teardown_dtlb_cache_slot(void) +{ + sh64_teardown_tlb_slot(dtlb_cache_slot); + local_irq_enable(); +} + +static inline void sh64_icache_inv_all(void) +{ + unsigned long long addr, flag, data; + unsigned long flags; + + addr = ICCR0; + flag = ICCR0_ICI; + data = 0; + + /* Make this a critical section for safety (probably not strictly necessary.) */ + local_irq_save(flags); + + /* Without %1 it gets unexplicably wrong */ + __asm__ __volatile__ ( + "getcfg %3, 0, %0\n\t" + "or %0, %2, %0\n\t" + "putcfg %3, 0, %0\n\t" + "synci" + : "=&r" (data) + : "0" (data), "r" (flag), "r" (addr)); + + local_irq_restore(flags); +} + +static void sh64_icache_inv_kernel_range(unsigned long start, unsigned long end) +{ + /* Invalidate range of addresses [start,end] from the I-cache, where + * the addresses lie in the kernel superpage. */ + + unsigned long long ullend, addr, aligned_start; + aligned_start = (unsigned long long)(signed long long)(signed long) start; + addr = L1_CACHE_ALIGN(aligned_start); + ullend = (unsigned long long) (signed long long) (signed long) end; + + while (addr <= ullend) { + __asm__ __volatile__ ("icbi %0, 0" : : "r" (addr)); + addr += L1_CACHE_BYTES; + } +} + +static void sh64_icache_inv_user_page(struct vm_area_struct *vma, unsigned long eaddr) +{ + /* If we get called, we know that vma->vm_flags contains VM_EXEC. + Also, eaddr is page-aligned. */ + unsigned int cpu = smp_processor_id(); + unsigned long long addr, end_addr; + unsigned long flags = 0; + unsigned long running_asid, vma_asid; + addr = eaddr; + end_addr = addr + PAGE_SIZE; + + /* Check whether we can use the current ASID for the I-cache + invalidation. For example, if we're called via + access_process_vm->flush_cache_page->here, (e.g. when reading from + /proc), 'running_asid' will be that of the reader, not of the + victim. + + Also, note the risk that we might get pre-empted between the ASID + compare and blocking IRQs, and before we regain control, the + pid->ASID mapping changes. However, the whole cache will get + invalidated when the mapping is renewed, so the worst that can + happen is that the loop below ends up invalidating somebody else's + cache entries. + */ + + running_asid = get_asid(); + vma_asid = cpu_asid(cpu, vma->vm_mm); + if (running_asid != vma_asid) { + local_irq_save(flags); + switch_and_save_asid(vma_asid); + } + while (addr < end_addr) { + /* Worth unrolling a little */ + __asm__ __volatile__("icbi %0, 0" : : "r" (addr)); + __asm__ __volatile__("icbi %0, 32" : : "r" (addr)); + __asm__ __volatile__("icbi %0, 64" : : "r" (addr)); + __asm__ __volatile__("icbi %0, 96" : : "r" (addr)); + addr += 128; + } + if (running_asid != vma_asid) { + switch_and_save_asid(running_asid); + local_irq_restore(flags); + } +} + +static void sh64_icache_inv_user_page_range(struct mm_struct *mm, + unsigned long start, unsigned long end) +{ + /* Used for invalidating big chunks of I-cache, i.e. assume the range + is whole pages. If 'start' or 'end' is not page aligned, the code + is conservative and invalidates to the ends of the enclosing pages. + This is functionally OK, just a performance loss. */ + + /* See the comments below in sh64_dcache_purge_user_range() regarding + the choice of algorithm. However, for the I-cache option (2) isn't + available because there are no physical tags so aliases can't be + resolved. The icbi instruction has to be used through the user + mapping. Because icbi is cheaper than ocbp on a cache hit, it + would be cheaper to use the selective code for a large range than is + possible with the D-cache. Just assume 64 for now as a working + figure. + */ + int n_pages; + + if (!mm) + return; + + n_pages = ((end - start) >> PAGE_SHIFT); + if (n_pages >= 64) { + sh64_icache_inv_all(); + } else { + unsigned long aligned_start; + unsigned long eaddr; + unsigned long after_last_page_start; + unsigned long mm_asid, current_asid; + unsigned long flags = 0; + + mm_asid = cpu_asid(smp_processor_id(), mm); + current_asid = get_asid(); + + if (mm_asid != current_asid) { + /* Switch ASID and run the invalidate loop under cli */ + local_irq_save(flags); + switch_and_save_asid(mm_asid); + } + + aligned_start = start & PAGE_MASK; + after_last_page_start = PAGE_SIZE + ((end - 1) & PAGE_MASK); + + while (aligned_start < after_last_page_start) { + struct vm_area_struct *vma; + unsigned long vma_end; + vma = find_vma(mm, aligned_start); + if (!vma || (aligned_start <= vma->vm_end)) { + /* Avoid getting stuck in an error condition */ + aligned_start += PAGE_SIZE; + continue; + } + vma_end = vma->vm_end; + if (vma->vm_flags & VM_EXEC) { + /* Executable */ + eaddr = aligned_start; + while (eaddr < vma_end) { + sh64_icache_inv_user_page(vma, eaddr); + eaddr += PAGE_SIZE; + } + } + aligned_start = vma->vm_end; /* Skip to start of next region */ + } + + if (mm_asid != current_asid) { + switch_and_save_asid(current_asid); + local_irq_restore(flags); + } + } +} + +static void sh64_icache_inv_current_user_range(unsigned long start, unsigned long end) +{ + /* The icbi instruction never raises ITLBMISS. i.e. if there's not a + cache hit on the virtual tag the instruction ends there, without a + TLB lookup. */ + + unsigned long long aligned_start; + unsigned long long ull_end; + unsigned long long addr; + + ull_end = end; + + /* Just invalidate over the range using the natural addresses. TLB + miss handling will be OK (TBC). Since it's for the current process, + either we're already in the right ASID context, or the ASIDs have + been recycled since we were last active in which case we might just + invalidate another processes I-cache entries : no worries, just a + performance drop for him. */ + aligned_start = L1_CACHE_ALIGN(start); + addr = aligned_start; + while (addr < ull_end) { + __asm__ __volatile__ ("icbi %0, 0" : : "r" (addr)); + __asm__ __volatile__ ("nop"); + __asm__ __volatile__ ("nop"); + addr += L1_CACHE_BYTES; + } +} + +/* Buffer used as the target of alloco instructions to purge data from cache + sets by natural eviction. -- RPC */ +#define DUMMY_ALLOCO_AREA_SIZE ((L1_CACHE_BYTES << 10) + (1024 * 4)) +static unsigned char dummy_alloco_area[DUMMY_ALLOCO_AREA_SIZE] __cacheline_aligned = { 0, }; + +static void inline sh64_dcache_purge_sets(int sets_to_purge_base, int n_sets) +{ + /* Purge all ways in a particular block of sets, specified by the base + set number and number of sets. Can handle wrap-around, if that's + needed. */ + + int dummy_buffer_base_set; + unsigned long long eaddr, eaddr0, eaddr1; + int j; + int set_offset; + + dummy_buffer_base_set = ((int)&dummy_alloco_area & + cpu_data->dcache.entry_mask) >> + cpu_data->dcache.entry_shift; + set_offset = sets_to_purge_base - dummy_buffer_base_set; + + for (j = 0; j < n_sets; j++, set_offset++) { + set_offset &= (cpu_data->dcache.sets - 1); + eaddr0 = (unsigned long long)dummy_alloco_area + + (set_offset << cpu_data->dcache.entry_shift); + + /* + * Do one alloco which hits the required set per cache + * way. For write-back mode, this will purge the #ways + * resident lines. There's little point unrolling this + * loop because the allocos stall more if they're too + * close together. + */ + eaddr1 = eaddr0 + cpu_data->dcache.way_size * + cpu_data->dcache.ways; + + for (eaddr = eaddr0; eaddr < eaddr1; + eaddr += cpu_data->dcache.way_size) { + __asm__ __volatile__ ("alloco %0, 0" : : "r" (eaddr)); + __asm__ __volatile__ ("synco"); /* TAKum03020 */ + } + + eaddr1 = eaddr0 + cpu_data->dcache.way_size * + cpu_data->dcache.ways; + + for (eaddr = eaddr0; eaddr < eaddr1; + eaddr += cpu_data->dcache.way_size) { + /* + * Load from each address. Required because + * alloco is a NOP if the cache is write-through. + */ + if (test_bit(SH_CACHE_MODE_WT, &(cpu_data->dcache.flags))) + __raw_readb((unsigned long)eaddr); + } + } + + /* + * Don't use OCBI to invalidate the lines. That costs cycles + * directly. If the dummy block is just left resident, it will + * naturally get evicted as required. + */ +} + +/* + * Purge the entire contents of the dcache. The most efficient way to + * achieve this is to use alloco instructions on a region of unused + * memory equal in size to the cache, thereby causing the current + * contents to be discarded by natural eviction. The alternative, namely + * reading every tag, setting up a mapping for the corresponding page and + * doing an OCBP for the line, would be much more expensive. + */ +static void sh64_dcache_purge_all(void) +{ + + sh64_dcache_purge_sets(0, cpu_data->dcache.sets); +} + + +/* Assumes this address (+ (2**n_synbits) pages up from it) aren't used for + anything else in the kernel */ +#define MAGIC_PAGE0_START 0xffffffffec000000ULL + +/* Purge the physical page 'paddr' from the cache. It's known that any + * cache lines requiring attention have the same page colour as the the + * address 'eaddr'. + * + * This relies on the fact that the D-cache matches on physical tags when + * no virtual tag matches. So we create an alias for the original page + * and purge through that. (Alternatively, we could have done this by + * switching ASID to match the original mapping and purged through that, + * but that involves ASID switching cost + probably a TLBMISS + refill + * anyway.) + */ +static void sh64_dcache_purge_coloured_phy_page(unsigned long paddr, + unsigned long eaddr) +{ + unsigned long long magic_page_start; + unsigned long long magic_eaddr, magic_eaddr_end; + + magic_page_start = MAGIC_PAGE0_START + (eaddr & CACHE_OC_SYN_MASK); + + /* As long as the kernel is not pre-emptible, this doesn't need to be + under cli/sti. */ + sh64_setup_dtlb_cache_slot(magic_page_start, get_asid(), paddr); + + magic_eaddr = magic_page_start; + magic_eaddr_end = magic_eaddr + PAGE_SIZE; + + while (magic_eaddr < magic_eaddr_end) { + /* Little point in unrolling this loop - the OCBPs are blocking + and won't go any quicker (i.e. the loop overhead is parallel + to part of the OCBP execution.) */ + __asm__ __volatile__ ("ocbp %0, 0" : : "r" (magic_eaddr)); + magic_eaddr += L1_CACHE_BYTES; + } + + sh64_teardown_dtlb_cache_slot(); +} + +/* + * Purge a page given its physical start address, by creating a temporary + * 1 page mapping and purging across that. Even if we know the virtual + * address (& vma or mm) of the page, the method here is more elegant + * because it avoids issues of coping with page faults on the purge + * instructions (i.e. no special-case code required in the critical path + * in the TLB miss handling). + */ +static void sh64_dcache_purge_phy_page(unsigned long paddr) +{ + unsigned long long eaddr_start, eaddr, eaddr_end; + int i; + + /* As long as the kernel is not pre-emptible, this doesn't need to be + under cli/sti. */ + eaddr_start = MAGIC_PAGE0_START; + for (i = 0; i < (1 << CACHE_OC_N_SYNBITS); i++) { + sh64_setup_dtlb_cache_slot(eaddr_start, get_asid(), paddr); + + eaddr = eaddr_start; + eaddr_end = eaddr + PAGE_SIZE; + while (eaddr < eaddr_end) { + __asm__ __volatile__ ("ocbp %0, 0" : : "r" (eaddr)); + eaddr += L1_CACHE_BYTES; + } + + sh64_teardown_dtlb_cache_slot(); + eaddr_start += PAGE_SIZE; + } +} + +static void sh64_dcache_purge_user_pages(struct mm_struct *mm, + unsigned long addr, unsigned long end) +{ + pgd_t *pgd; + pud_t *pud; + pmd_t *pmd; + pte_t *pte; + pte_t entry; + spinlock_t *ptl; + unsigned long paddr; + + if (!mm) + return; /* No way to find physical address of page */ + + pgd = pgd_offset(mm, addr); + if (pgd_bad(*pgd)) + return; + + pud = pud_offset(pgd, addr); + if (pud_none(*pud) || pud_bad(*pud)) + return; + + pmd = pmd_offset(pud, addr); + if (pmd_none(*pmd) || pmd_bad(*pmd)) + return; + + pte = pte_offset_map_lock(mm, pmd, addr, &ptl); + do { + entry = *pte; + if (pte_none(entry) || !pte_present(entry)) + continue; + paddr = pte_val(entry) & PAGE_MASK; + sh64_dcache_purge_coloured_phy_page(paddr, addr); + } while (pte++, addr += PAGE_SIZE, addr != end); + pte_unmap_unlock(pte - 1, ptl); +} + +/* + * There are at least 5 choices for the implementation of this, with + * pros (+), cons(-), comments(*): + * + * 1. ocbp each line in the range through the original user's ASID + * + no lines spuriously evicted + * - tlbmiss handling (must either handle faults on demand => extra + * special-case code in tlbmiss critical path), or map the page in + * advance (=> flush_tlb_range in advance to avoid multiple hits) + * - ASID switching + * - expensive for large ranges + * + * 2. temporarily map each page in the range to a special effective + * address and ocbp through the temporary mapping; relies on the + * fact that SH-5 OCB* always do TLB lookup and match on ptags (they + * never look at the etags) + * + no spurious evictions + * - expensive for large ranges + * * surely cheaper than (1) + * + * 3. walk all the lines in the cache, check the tags, if a match + * occurs create a page mapping to ocbp the line through + * + no spurious evictions + * - tag inspection overhead + * - (especially for small ranges) + * - potential cost of setting up/tearing down page mapping for + * every line that matches the range + * * cost partly independent of range size + * + * 4. walk all the lines in the cache, check the tags, if a match + * occurs use 4 * alloco to purge the line (+3 other probably + * innocent victims) by natural eviction + * + no tlb mapping overheads + * - spurious evictions + * - tag inspection overhead + * + * 5. implement like flush_cache_all + * + no tag inspection overhead + * - spurious evictions + * - bad for small ranges + * + * (1) can be ruled out as more expensive than (2). (2) appears best + * for small ranges. The choice between (3), (4) and (5) for large + * ranges and the range size for the large/small boundary need + * benchmarking to determine. + * + * For now use approach (2) for small ranges and (5) for large ones. + */ +static void sh64_dcache_purge_user_range(struct mm_struct *mm, + unsigned long start, unsigned long end) +{ + int n_pages = ((end - start) >> PAGE_SHIFT); + + if (n_pages >= 64 || ((start ^ (end - 1)) & PMD_MASK)) { + sh64_dcache_purge_all(); + } else { + /* Small range, covered by a single page table page */ + start &= PAGE_MASK; /* should already be so */ + end = PAGE_ALIGN(end); /* should already be so */ + sh64_dcache_purge_user_pages(mm, start, end); + } +} + +/* + * Invalidate the entire contents of both caches, after writing back to + * memory any dirty data from the D-cache. + */ +static void sh5_flush_cache_all(void *unused) +{ + sh64_dcache_purge_all(); + sh64_icache_inv_all(); +} + +/* + * Invalidate an entire user-address space from both caches, after + * writing back dirty data (e.g. for shared mmap etc). + * + * This could be coded selectively by inspecting all the tags then + * doing 4*alloco on any set containing a match (as for + * flush_cache_range), but fork/exit/execve (where this is called from) + * are expensive anyway. + * + * Have to do a purge here, despite the comments re I-cache below. + * There could be odd-coloured dirty data associated with the mm still + * in the cache - if this gets written out through natural eviction + * after the kernel has reused the page there will be chaos. + * + * The mm being torn down won't ever be active again, so any Icache + * lines tagged with its ASID won't be visible for the rest of the + * lifetime of this ASID cycle. Before the ASID gets reused, there + * will be a flush_cache_all. Hence we don't need to touch the + * I-cache. This is similar to the lack of action needed in + * flush_tlb_mm - see fault.c. + */ +static void sh5_flush_cache_mm(void *unused) +{ + sh64_dcache_purge_all(); +} + +/* + * Invalidate (from both caches) the range [start,end) of virtual + * addresses from the user address space specified by mm, after writing + * back any dirty data. + * + * Note, 'end' is 1 byte beyond the end of the range to flush. + */ +static void sh5_flush_cache_range(void *args) +{ + struct flusher_data *data = args; + struct vm_area_struct *vma; + unsigned long start, end; + + vma = data->vma; + start = data->addr1; + end = data->addr2; + + sh64_dcache_purge_user_range(vma->vm_mm, start, end); + sh64_icache_inv_user_page_range(vma->vm_mm, start, end); +} + +/* + * Invalidate any entries in either cache for the vma within the user + * address space vma->vm_mm for the page starting at virtual address + * 'eaddr'. This seems to be used primarily in breaking COW. Note, + * the I-cache must be searched too in case the page in question is + * both writable and being executed from (e.g. stack trampolines.) + * + * Note, this is called with pte lock held. + */ +static void sh5_flush_cache_page(void *args) +{ + struct flusher_data *data = args; + struct vm_area_struct *vma; + unsigned long eaddr, pfn; + + vma = data->vma; + eaddr = data->addr1; + pfn = data->addr2; + + sh64_dcache_purge_phy_page(pfn << PAGE_SHIFT); + + if (vma->vm_flags & VM_EXEC) + sh64_icache_inv_user_page(vma, eaddr); +} + +static void sh5_flush_dcache_page(void *page) +{ + sh64_dcache_purge_phy_page(page_to_phys((struct page *)page)); + wmb(); +} + +/* + * Flush the range [start,end] of kernel virtual address space from + * the I-cache. The corresponding range must be purged from the + * D-cache also because the SH-5 doesn't have cache snooping between + * the caches. The addresses will be visible through the superpage + * mapping, therefore it's guaranteed that there no cache entries for + * the range in cache sets of the wrong colour. + */ +static void sh5_flush_icache_range(void *args) +{ + struct flusher_data *data = args; + unsigned long start, end; + + start = data->addr1; + end = data->addr2; + + __flush_purge_region((void *)start, end); + wmb(); + sh64_icache_inv_kernel_range(start, end); +} + +/* + * For the address range [start,end), write back the data from the + * D-cache and invalidate the corresponding region of the I-cache for the + * current process. Used to flush signal trampolines on the stack to + * make them executable. + */ +static void sh5_flush_cache_sigtramp(void *vaddr) +{ + unsigned long end = (unsigned long)vaddr + L1_CACHE_BYTES; + + __flush_wback_region(vaddr, L1_CACHE_BYTES); + wmb(); + sh64_icache_inv_current_user_range((unsigned long)vaddr, end); +} + +void __init sh5_cache_init(void) +{ + local_flush_cache_all = sh5_flush_cache_all; + local_flush_cache_mm = sh5_flush_cache_mm; + local_flush_cache_dup_mm = sh5_flush_cache_mm; + local_flush_cache_page = sh5_flush_cache_page; + local_flush_cache_range = sh5_flush_cache_range; + local_flush_dcache_page = sh5_flush_dcache_page; + local_flush_icache_range = sh5_flush_icache_range; + local_flush_cache_sigtramp = sh5_flush_cache_sigtramp; + + /* Reserve a slot for dcache colouring in the DTLB */ + dtlb_cache_slot = sh64_get_wired_dtlb_entry(); + + sh4__flush_region_init(); +} |