diff options
Diffstat (limited to 'kernel/arch/frv/mm/tlb-miss.S')
| -rw-r--r-- | kernel/arch/frv/mm/tlb-miss.S | 629 |
1 files changed, 629 insertions, 0 deletions
diff --git a/kernel/arch/frv/mm/tlb-miss.S b/kernel/arch/frv/mm/tlb-miss.S new file mode 100644 index 000000000..f3ac019bb --- /dev/null +++ b/kernel/arch/frv/mm/tlb-miss.S @@ -0,0 +1,629 @@ +/* tlb-miss.S: TLB miss handlers + * + * Copyright (C) 2004 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + */ + +#include <linux/sys.h> +#include <linux/linkage.h> +#include <asm/page.h> +#include <asm/pgtable.h> +#include <asm/spr-regs.h> + + .section .text..tlbmiss + .balign 4 + + .globl __entry_insn_mmu_miss +__entry_insn_mmu_miss: + break + nop + + .globl __entry_insn_mmu_exception +__entry_insn_mmu_exception: + break + nop + + .globl __entry_data_mmu_miss +__entry_data_mmu_miss: + break + nop + + .globl __entry_data_mmu_exception +__entry_data_mmu_exception: + break + nop + +############################################################################### +# +# handle a lookup failure of one sort or another in a kernel TLB handler +# On entry: +# GR29 - faulting address +# SCR2 - saved CCR +# +############################################################################### + .type __tlb_kernel_fault,@function +__tlb_kernel_fault: + # see if we're supposed to re-enable single-step mode upon return + sethi.p %hi(__break_tlb_miss_return_break),gr30 + setlo %lo(__break_tlb_miss_return_break),gr30 + movsg pcsr,gr31 + + subcc gr31,gr30,gr0,icc0 + beq icc0,#0,__tlb_kernel_fault_sstep + + movsg scr2,gr30 + movgs gr30,ccr + movgs gr29,scr2 /* save EAR0 value */ + sethi.p %hi(__kernel_current_task),gr29 + setlo %lo(__kernel_current_task),gr29 + ldi.p @(gr29,#0),gr29 /* restore GR29 */ + + bra __entry_kernel_handle_mmu_fault + + # we've got to re-enable single-stepping +__tlb_kernel_fault_sstep: + sethi.p %hi(__break_tlb_miss_real_return_info),gr30 + setlo %lo(__break_tlb_miss_real_return_info),gr30 + lddi @(gr30,0),gr30 + movgs gr30,pcsr + movgs gr31,psr + + movsg scr2,gr30 + movgs gr30,ccr + movgs gr29,scr2 /* save EAR0 value */ + sethi.p %hi(__kernel_current_task),gr29 + setlo %lo(__kernel_current_task),gr29 + ldi.p @(gr29,#0),gr29 /* restore GR29 */ + bra __entry_kernel_handle_mmu_fault_sstep + + .size __tlb_kernel_fault, .-__tlb_kernel_fault + +############################################################################### +# +# handle a lookup failure of one sort or another in a user TLB handler +# On entry: +# GR28 - faulting address +# SCR2 - saved CCR +# +############################################################################### + .type __tlb_user_fault,@function +__tlb_user_fault: + # see if we're supposed to re-enable single-step mode upon return + sethi.p %hi(__break_tlb_miss_return_break),gr30 + setlo %lo(__break_tlb_miss_return_break),gr30 + movsg pcsr,gr31 + subcc gr31,gr30,gr0,icc0 + beq icc0,#0,__tlb_user_fault_sstep + + movsg scr2,gr30 + movgs gr30,ccr + bra __entry_uspace_handle_mmu_fault + + # we've got to re-enable single-stepping +__tlb_user_fault_sstep: + sethi.p %hi(__break_tlb_miss_real_return_info),gr30 + setlo %lo(__break_tlb_miss_real_return_info),gr30 + lddi @(gr30,0),gr30 + movgs gr30,pcsr + movgs gr31,psr + movsg scr2,gr30 + movgs gr30,ccr + bra __entry_uspace_handle_mmu_fault_sstep + + .size __tlb_user_fault, .-__tlb_user_fault + +############################################################################### +# +# Kernel instruction TLB miss handler +# On entry: +# GR1 - kernel stack pointer +# GR28 - saved exception frame pointer +# GR29 - faulting address +# GR31 - EAR0 ^ SCR0 +# SCR0 - base of virtual range covered by cached PGE from last ITLB miss (or 0xffffffff) +# DAMR3 - mapped page directory +# DAMR4 - mapped page table as matched by SCR0 +# +############################################################################### + .globl __entry_kernel_insn_tlb_miss + .type __entry_kernel_insn_tlb_miss,@function +__entry_kernel_insn_tlb_miss: +#if 0 + sethi.p %hi(0xe1200004),gr30 + setlo %lo(0xe1200004),gr30 + st gr0,@(gr30,gr0) + sethi.p %hi(0xffc00100),gr30 + setlo %lo(0xffc00100),gr30 + sth gr30,@(gr30,gr0) + membar +#endif + + movsg ccr,gr30 /* save CCR */ + movgs gr30,scr2 + + # see if the cached page table mapping is appropriate + srlicc.p gr31,#26,gr0,icc0 + setlos 0x3ffc,gr30 + srli.p gr29,#12,gr31 /* use EAR0[25:14] as PTE index */ + bne icc0,#0,__itlb_k_PTD_miss + +__itlb_k_PTD_mapped: + # access the PTD with EAR0[25:14] + # - DAMLR4 points to the virtual address of the appropriate page table + # - the PTD holds 4096 PTEs + # - the PTD must be accessed uncached + # - the PTE must be marked accessed if it was valid + # + and gr31,gr30,gr31 + movsg damlr4,gr30 + add gr30,gr31,gr31 + ldi @(gr31,#0),gr30 /* fetch the PTE */ + andicc gr30,#_PAGE_PRESENT,gr0,icc0 + ori.p gr30,#_PAGE_ACCESSED,gr30 + beq icc0,#0,__tlb_kernel_fault /* jump if PTE invalid */ + sti.p gr30,@(gr31,#0) /* update the PTE */ + andi gr30,#~_PAGE_ACCESSED,gr30 + + # we're using IAMR1 as an extra TLB entry + # - punt the entry here (if valid) to the real TLB and then replace with the new PTE + # - need to check DAMR1 lest we cause an multiple-DAT-hit exception + # - IAMPR1 has no WP bit, and we mustn't lose WP information + movsg iampr1,gr31 + andicc gr31,#xAMPRx_V,gr0,icc0 + setlos.p 0xfffff000,gr31 + beq icc0,#0,__itlb_k_nopunt /* punt not required */ + + movsg iamlr1,gr31 + movgs gr31,tplr /* set TPLR.CXN */ + tlbpr gr31,gr0,#4,#0 /* delete matches from TLB, IAMR1, DAMR1 */ + + movsg dampr1,gr31 + ori gr31,#xAMPRx_V,gr31 /* entry was invalidated by tlbpr #4 */ + movgs gr31,tppr + movsg iamlr1,gr31 /* set TPLR.CXN */ + movgs gr31,tplr + tlbpr gr31,gr0,#2,#0 /* save to the TLB */ + movsg tpxr,gr31 /* check the TLB write error flag */ + andicc.p gr31,#TPXR_E,gr0,icc0 + setlos #0xfffff000,gr31 + bne icc0,#0,__tlb_kernel_fault + +__itlb_k_nopunt: + + # assemble the new TLB entry + and gr29,gr31,gr29 + movsg cxnr,gr31 + or gr29,gr31,gr29 + movgs gr29,iamlr1 /* xAMLR = address | context number */ + movgs gr30,iampr1 + movgs gr29,damlr1 + movgs gr30,dampr1 + + # return, restoring registers + movsg scr2,gr30 + movgs gr30,ccr + sethi.p %hi(__kernel_current_task),gr29 + setlo %lo(__kernel_current_task),gr29 + ldi @(gr29,#0),gr29 + rett #0 + beq icc0,#3,0 /* prevent icache prefetch */ + + # the PTE we want wasn't in the PTD we have mapped, so we need to go looking for a more + # appropriate page table and map that instead + # - access the PGD with EAR0[31:26] + # - DAMLR3 points to the virtual address of the page directory + # - the PGD holds 64 PGEs and each PGE/PME points to a set of page tables +__itlb_k_PTD_miss: + srli gr29,#26,gr31 /* calculate PGE offset */ + slli gr31,#8,gr31 /* and clear bottom bits */ + + movsg damlr3,gr30 + ld @(gr31,gr30),gr30 /* access the PGE */ + + andicc.p gr30,#_PAGE_PRESENT,gr0,icc0 + andicc gr30,#xAMPRx_SS,gr0,icc1 + + # map this PTD instead and record coverage address + ori.p gr30,#xAMPRx_L|xAMPRx_SS_16Kb|xAMPRx_S|xAMPRx_C|xAMPRx_V,gr30 + beq icc0,#0,__tlb_kernel_fault /* jump if PGE not present */ + slli.p gr31,#18,gr31 + bne icc1,#0,__itlb_k_bigpage + movgs gr30,dampr4 + movgs gr31,scr0 + + # we can now resume normal service + setlos 0x3ffc,gr30 + srli.p gr29,#12,gr31 /* use EAR0[25:14] as PTE index */ + bra __itlb_k_PTD_mapped + +__itlb_k_bigpage: + break + nop + + .size __entry_kernel_insn_tlb_miss, .-__entry_kernel_insn_tlb_miss + +############################################################################### +# +# Kernel data TLB miss handler +# On entry: +# GR1 - kernel stack pointer +# GR28 - saved exception frame pointer +# GR29 - faulting address +# GR31 - EAR0 ^ SCR1 +# SCR1 - base of virtual range covered by cached PGE from last DTLB miss (or 0xffffffff) +# DAMR3 - mapped page directory +# DAMR5 - mapped page table as matched by SCR1 +# +############################################################################### + .globl __entry_kernel_data_tlb_miss + .type __entry_kernel_data_tlb_miss,@function +__entry_kernel_data_tlb_miss: +#if 0 + sethi.p %hi(0xe1200004),gr30 + setlo %lo(0xe1200004),gr30 + st gr0,@(gr30,gr0) + sethi.p %hi(0xffc00100),gr30 + setlo %lo(0xffc00100),gr30 + sth gr30,@(gr30,gr0) + membar +#endif + + movsg ccr,gr30 /* save CCR */ + movgs gr30,scr2 + + # see if the cached page table mapping is appropriate + srlicc.p gr31,#26,gr0,icc0 + setlos 0x3ffc,gr30 + srli.p gr29,#12,gr31 /* use EAR0[25:14] as PTE index */ + bne icc0,#0,__dtlb_k_PTD_miss + +__dtlb_k_PTD_mapped: + # access the PTD with EAR0[25:14] + # - DAMLR5 points to the virtual address of the appropriate page table + # - the PTD holds 4096 PTEs + # - the PTD must be accessed uncached + # - the PTE must be marked accessed if it was valid + # + and gr31,gr30,gr31 + movsg damlr5,gr30 + add gr30,gr31,gr31 + ldi @(gr31,#0),gr30 /* fetch the PTE */ + andicc gr30,#_PAGE_PRESENT,gr0,icc0 + ori.p gr30,#_PAGE_ACCESSED,gr30 + beq icc0,#0,__tlb_kernel_fault /* jump if PTE invalid */ + sti.p gr30,@(gr31,#0) /* update the PTE */ + andi gr30,#~_PAGE_ACCESSED,gr30 + + # we're using DAMR1 as an extra TLB entry + # - punt the entry here (if valid) to the real TLB and then replace with the new PTE + # - need to check IAMR1 lest we cause an multiple-DAT-hit exception + movsg dampr1,gr31 + andicc gr31,#xAMPRx_V,gr0,icc0 + setlos.p 0xfffff000,gr31 + beq icc0,#0,__dtlb_k_nopunt /* punt not required */ + + movsg damlr1,gr31 + movgs gr31,tplr /* set TPLR.CXN */ + tlbpr gr31,gr0,#4,#0 /* delete matches from TLB, IAMR1, DAMR1 */ + + movsg dampr1,gr31 + ori gr31,#xAMPRx_V,gr31 /* entry was invalidated by tlbpr #4 */ + movgs gr31,tppr + movsg damlr1,gr31 /* set TPLR.CXN */ + movgs gr31,tplr + tlbpr gr31,gr0,#2,#0 /* save to the TLB */ + movsg tpxr,gr31 /* check the TLB write error flag */ + andicc.p gr31,#TPXR_E,gr0,icc0 + setlos #0xfffff000,gr31 + bne icc0,#0,__tlb_kernel_fault + +__dtlb_k_nopunt: + + # assemble the new TLB entry + and gr29,gr31,gr29 + movsg cxnr,gr31 + or gr29,gr31,gr29 + movgs gr29,iamlr1 /* xAMLR = address | context number */ + movgs gr30,iampr1 + movgs gr29,damlr1 + movgs gr30,dampr1 + + # return, restoring registers + movsg scr2,gr30 + movgs gr30,ccr + sethi.p %hi(__kernel_current_task),gr29 + setlo %lo(__kernel_current_task),gr29 + ldi @(gr29,#0),gr29 + rett #0 + beq icc0,#3,0 /* prevent icache prefetch */ + + # the PTE we want wasn't in the PTD we have mapped, so we need to go looking for a more + # appropriate page table and map that instead + # - access the PGD with EAR0[31:26] + # - DAMLR3 points to the virtual address of the page directory + # - the PGD holds 64 PGEs and each PGE/PME points to a set of page tables +__dtlb_k_PTD_miss: + srli gr29,#26,gr31 /* calculate PGE offset */ + slli gr31,#8,gr31 /* and clear bottom bits */ + + movsg damlr3,gr30 + ld @(gr31,gr30),gr30 /* access the PGE */ + + andicc.p gr30,#_PAGE_PRESENT,gr0,icc0 + andicc gr30,#xAMPRx_SS,gr0,icc1 + + # map this PTD instead and record coverage address + ori.p gr30,#xAMPRx_L|xAMPRx_SS_16Kb|xAMPRx_S|xAMPRx_C|xAMPRx_V,gr30 + beq icc0,#0,__tlb_kernel_fault /* jump if PGE not present */ + slli.p gr31,#18,gr31 + bne icc1,#0,__dtlb_k_bigpage + movgs gr30,dampr5 + movgs gr31,scr1 + + # we can now resume normal service + setlos 0x3ffc,gr30 + srli.p gr29,#12,gr31 /* use EAR0[25:14] as PTE index */ + bra __dtlb_k_PTD_mapped + +__dtlb_k_bigpage: + break + nop + + .size __entry_kernel_data_tlb_miss, .-__entry_kernel_data_tlb_miss + +############################################################################### +# +# Userspace instruction TLB miss handler (with PGE prediction) +# On entry: +# GR28 - faulting address +# GR31 - EAR0 ^ SCR0 +# SCR0 - base of virtual range covered by cached PGE from last ITLB miss (or 0xffffffff) +# DAMR3 - mapped page directory +# DAMR4 - mapped page table as matched by SCR0 +# +############################################################################### + .globl __entry_user_insn_tlb_miss + .type __entry_user_insn_tlb_miss,@function +__entry_user_insn_tlb_miss: +#if 0 + sethi.p %hi(0xe1200004),gr30 + setlo %lo(0xe1200004),gr30 + st gr0,@(gr30,gr0) + sethi.p %hi(0xffc00100),gr30 + setlo %lo(0xffc00100),gr30 + sth gr30,@(gr30,gr0) + membar +#endif + + movsg ccr,gr30 /* save CCR */ + movgs gr30,scr2 + + # see if the cached page table mapping is appropriate + srlicc.p gr31,#26,gr0,icc0 + setlos 0x3ffc,gr30 + srli.p gr28,#12,gr31 /* use EAR0[25:14] as PTE index */ + bne icc0,#0,__itlb_u_PTD_miss + +__itlb_u_PTD_mapped: + # access the PTD with EAR0[25:14] + # - DAMLR4 points to the virtual address of the appropriate page table + # - the PTD holds 4096 PTEs + # - the PTD must be accessed uncached + # - the PTE must be marked accessed if it was valid + # + and gr31,gr30,gr31 + movsg damlr4,gr30 + add gr30,gr31,gr31 + ldi @(gr31,#0),gr30 /* fetch the PTE */ + andicc gr30,#_PAGE_PRESENT,gr0,icc0 + ori.p gr30,#_PAGE_ACCESSED,gr30 + beq icc0,#0,__tlb_user_fault /* jump if PTE invalid */ + sti.p gr30,@(gr31,#0) /* update the PTE */ + andi gr30,#~_PAGE_ACCESSED,gr30 + + # we're using IAMR1/DAMR1 as an extra TLB entry + # - punt the entry here (if valid) to the real TLB and then replace with the new PTE + movsg dampr1,gr31 + andicc gr31,#xAMPRx_V,gr0,icc0 + setlos.p 0xfffff000,gr31 + beq icc0,#0,__itlb_u_nopunt /* punt not required */ + + movsg dampr1,gr31 + movgs gr31,tppr + movsg damlr1,gr31 /* set TPLR.CXN */ + movgs gr31,tplr + tlbpr gr31,gr0,#2,#0 /* save to the TLB */ + movsg tpxr,gr31 /* check the TLB write error flag */ + andicc.p gr31,#TPXR_E,gr0,icc0 + setlos #0xfffff000,gr31 + bne icc0,#0,__tlb_user_fault + +__itlb_u_nopunt: + + # assemble the new TLB entry + and gr28,gr31,gr28 + movsg cxnr,gr31 + or gr28,gr31,gr28 + movgs gr28,iamlr1 /* xAMLR = address | context number */ + movgs gr30,iampr1 + movgs gr28,damlr1 + movgs gr30,dampr1 + + # return, restoring registers + movsg scr2,gr30 + movgs gr30,ccr + rett #0 + beq icc0,#3,0 /* prevent icache prefetch */ + + # the PTE we want wasn't in the PTD we have mapped, so we need to go looking for a more + # appropriate page table and map that instead + # - access the PGD with EAR0[31:26] + # - DAMLR3 points to the virtual address of the page directory + # - the PGD holds 64 PGEs and each PGE/PME points to a set of page tables +__itlb_u_PTD_miss: + srli gr28,#26,gr31 /* calculate PGE offset */ + slli gr31,#8,gr31 /* and clear bottom bits */ + + movsg damlr3,gr30 + ld @(gr31,gr30),gr30 /* access the PGE */ + + andicc.p gr30,#_PAGE_PRESENT,gr0,icc0 + andicc gr30,#xAMPRx_SS,gr0,icc1 + + # map this PTD instead and record coverage address + ori.p gr30,#xAMPRx_L|xAMPRx_SS_16Kb|xAMPRx_S|xAMPRx_C|xAMPRx_V,gr30 + beq icc0,#0,__tlb_user_fault /* jump if PGE not present */ + slli.p gr31,#18,gr31 + bne icc1,#0,__itlb_u_bigpage + movgs gr30,dampr4 + movgs gr31,scr0 + + # we can now resume normal service + setlos 0x3ffc,gr30 + srli.p gr28,#12,gr31 /* use EAR0[25:14] as PTE index */ + bra __itlb_u_PTD_mapped + +__itlb_u_bigpage: + break + nop + + .size __entry_user_insn_tlb_miss, .-__entry_user_insn_tlb_miss + +############################################################################### +# +# Userspace data TLB miss handler +# On entry: +# GR28 - faulting address +# GR31 - EAR0 ^ SCR1 +# SCR1 - base of virtual range covered by cached PGE from last DTLB miss (or 0xffffffff) +# DAMR3 - mapped page directory +# DAMR5 - mapped page table as matched by SCR1 +# +############################################################################### + .globl __entry_user_data_tlb_miss + .type __entry_user_data_tlb_miss,@function +__entry_user_data_tlb_miss: +#if 0 + sethi.p %hi(0xe1200004),gr30 + setlo %lo(0xe1200004),gr30 + st gr0,@(gr30,gr0) + sethi.p %hi(0xffc00100),gr30 + setlo %lo(0xffc00100),gr30 + sth gr30,@(gr30,gr0) + membar +#endif + + movsg ccr,gr30 /* save CCR */ + movgs gr30,scr2 + + # see if the cached page table mapping is appropriate + srlicc.p gr31,#26,gr0,icc0 + setlos 0x3ffc,gr30 + srli.p gr28,#12,gr31 /* use EAR0[25:14] as PTE index */ + bne icc0,#0,__dtlb_u_PTD_miss + +__dtlb_u_PTD_mapped: + # access the PTD with EAR0[25:14] + # - DAMLR5 points to the virtual address of the appropriate page table + # - the PTD holds 4096 PTEs + # - the PTD must be accessed uncached + # - the PTE must be marked accessed if it was valid + # + and gr31,gr30,gr31 + movsg damlr5,gr30 + +__dtlb_u_using_iPTD: + add gr30,gr31,gr31 + ldi @(gr31,#0),gr30 /* fetch the PTE */ + andicc gr30,#_PAGE_PRESENT,gr0,icc0 + ori.p gr30,#_PAGE_ACCESSED,gr30 + beq icc0,#0,__tlb_user_fault /* jump if PTE invalid */ + sti.p gr30,@(gr31,#0) /* update the PTE */ + andi gr30,#~_PAGE_ACCESSED,gr30 + + # we're using DAMR1 as an extra TLB entry + # - punt the entry here (if valid) to the real TLB and then replace with the new PTE + movsg dampr1,gr31 + andicc gr31,#xAMPRx_V,gr0,icc0 + setlos.p 0xfffff000,gr31 + beq icc0,#0,__dtlb_u_nopunt /* punt not required */ + + movsg dampr1,gr31 + movgs gr31,tppr + movsg damlr1,gr31 /* set TPLR.CXN */ + movgs gr31,tplr + tlbpr gr31,gr0,#2,#0 /* save to the TLB */ + movsg tpxr,gr31 /* check the TLB write error flag */ + andicc.p gr31,#TPXR_E,gr0,icc0 + setlos #0xfffff000,gr31 + bne icc0,#0,__tlb_user_fault + +__dtlb_u_nopunt: + + # assemble the new TLB entry + and gr28,gr31,gr28 + movsg cxnr,gr31 + or gr28,gr31,gr28 + movgs gr28,iamlr1 /* xAMLR = address | context number */ + movgs gr30,iampr1 + movgs gr28,damlr1 + movgs gr30,dampr1 + + # return, restoring registers + movsg scr2,gr30 + movgs gr30,ccr + rett #0 + beq icc0,#3,0 /* prevent icache prefetch */ + + # the PTE we want wasn't in the PTD we have mapped, so we need to go looking for a more + # appropriate page table and map that instead + # - first of all, check the insn PGE cache - we may well get a hit there + # - access the PGD with EAR0[31:26] + # - DAMLR3 points to the virtual address of the page directory + # - the PGD holds 64 PGEs and each PGE/PME points to a set of page tables +__dtlb_u_PTD_miss: + movsg scr0,gr31 /* consult the insn-PGE-cache key */ + xor gr28,gr31,gr31 + srlicc gr31,#26,gr0,icc0 + srli gr28,#12,gr31 /* use EAR0[25:14] as PTE index */ + bne icc0,#0,__dtlb_u_iPGE_miss + + # what we're looking for is covered by the insn-PGE-cache + setlos 0x3ffc,gr30 + and gr31,gr30,gr31 + movsg damlr4,gr30 + bra __dtlb_u_using_iPTD + +__dtlb_u_iPGE_miss: + srli gr28,#26,gr31 /* calculate PGE offset */ + slli gr31,#8,gr31 /* and clear bottom bits */ + + movsg damlr3,gr30 + ld @(gr31,gr30),gr30 /* access the PGE */ + + andicc.p gr30,#_PAGE_PRESENT,gr0,icc0 + andicc gr30,#xAMPRx_SS,gr0,icc1 + + # map this PTD instead and record coverage address + ori.p gr30,#xAMPRx_L|xAMPRx_SS_16Kb|xAMPRx_S|xAMPRx_C|xAMPRx_V,gr30 + beq icc0,#0,__tlb_user_fault /* jump if PGE not present */ + slli.p gr31,#18,gr31 + bne icc1,#0,__dtlb_u_bigpage + movgs gr30,dampr5 + movgs gr31,scr1 + + # we can now resume normal service + setlos 0x3ffc,gr30 + srli.p gr28,#12,gr31 /* use EAR0[25:14] as PTE index */ + bra __dtlb_u_PTD_mapped + +__dtlb_u_bigpage: + break + nop + + .size __entry_user_data_tlb_miss, .-__entry_user_data_tlb_miss |