/* $OpenBSD: tlbhandler.S,v 1.54 2022/12/11 05:07:25 visa Exp $ */ /* * Copyright (c) 1995-2004 Opsycon AB (www.opsycon.se / www.opsycon.com) * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * */ /* * This code handles TLB exceptions and updates. */ #include #include #include #include #include #include #include #ifdef CPU_LOONGSON2 #include #endif #include "assym.h" .set mips3 .set noreorder /* Default reorder mode */ /*---------------------------------------------------------------- tlb_miss * Low level TLB exception handler. TLB and XTLB code are * identical at the moment. * This code is not copied to the exception area, instead * trampolines branching to the appropriate code are built. */ .set noat .globl xtlb_miss .ent xtlb_miss, 0 xtlb_miss: dmfc0 k0, COP_0_BAD_VADDR bltz k0, _k_miss # kernel address space PTR_SRL k0, k0, SEGSHIFT sltiu k1, k0, PMAP_SEGTABSIZE beqz k1, _inv_seg # wrong if outside pm_segtab nop GET_CPU_INFO(k1, k0) PTR_L k1, CI_CURPROCPADDR(k1) dmfc0 k0, COP_0_BAD_VADDR PTR_SRL k0, k0, SEGSHIFT PTR_SLL k0, k0, LOGREGSZ PTR_L k1, PCB_SEGTAB(k1) PTR_ADDU k1, k1, k0 PTR_L k1, 0(k1) # get pointer to page directory dmfc0 k0, COP_0_BAD_VADDR PTR_SRL k0, k0, (DIRSHIFT - LOGREGSZ) beqz k1, _inv_seg andi k0, k0, (NPDEPG - 1) << LOGREGSZ PTR_ADDU k1, k1, k0 PTR_L k1, 0(k1) # get pointer to page table dmfc0 k0, COP_0_BAD_VADDR PTR_SRL k0, k0, PGSHIFT - PTE_LOG beqz k1, _inv_seg andi k0, k0, ((NPTEPG >> 1) - 1) << (PTE_LOG + 1) PTR_ADDU k1, k1, k0 PTE_LOAD k0, 0(k1) PTE_LOAD k1, PTE_OFFS(k1) PTE_CLEAR_SWBITS(k0) PTE_CLEAR_SWBITS(k1) dmtc0 k0, COP_0_TLB_LO0 dmtc0 k1, COP_0_TLB_LO1 TLB_HAZARD tlbwr # update TLB TLB_HAZARD #ifdef CPU_LOONGSON2 li k0, COP_0_DIAG_ITLB_CLEAR | COP_0_DIAG_BTB_CLEAR | COP_0_DIAG_RAS_DISABLE dmtc0 k0, COP_0_DIAG #endif ERET _k_miss: dmfc0 k0, COP_0_BAD_VADDR # must reload. b k_tlb_miss # kernel tlbmiss. nop _inv_seg: # No page table for this segment, # or processing is too complex to be # done here: invoke C code. mfc0 k0, COP_0_STATUS_REG andi k0, SR_KSU_USER bne k0, zero, go_u_general nop go_k_general: j k_general nop go_u_general: #ifdef CPU_LOONGSON2 /* * To work around a branch prediction issue on earlier LS2F * chips, it is necessary to clear the BTB upon * userland->kernel boundaries. */ li k0, COP_0_DIAG_BTB_CLEAR | COP_0_DIAG_RAS_DISABLE dmtc0 k0, COP_0_DIAG #endif j u_general nop .end xtlb_miss .set at /*---------------------------------------------------------------- k_tlb_inv * Handle a TLB invalid exception from kernel mode in kernel * space. This happens when we have a TLB match but an invalid * entry. Try to reload. */ NLEAF(k_tlb_inv, 0) .set noat #ifdef CPU_OCTEON .set push .set arch=octeon /* Check if the fault was caused by an instruction fetch. */ dmfc0 k0, COP_0_CAUSE_REG bbit0 k0, CR_BR_DELAY_SHIFT, 1f # fault in a branch delay slot? dmfc0 k1, COP_0_EXC_PC daddu k1, k1, 4 # adjust for the slot 1: dmfc0 k0, COP_0_BAD_VADDR beq k0, k1, go_k_general .set pop #else dmfc0 k0, COP_0_BAD_VADDR # get the fault address #endif LA k1, (VM_MIN_KERNEL_ADDRESS) # compute index PTR_SUBU k0, k0, k1 lw k1, Sysmapsize # index within range? PTR_SRL k0, k0, PGSHIFT sltu k1, k0, k1 beq k1, zero, sys_stk_chk # No. check for valid stack nop PTR_L k1, Sysmap PTR_SLL k0, k0, PTE_LOG # compute offset from index TLB_HAZARD tlbp # Probe the invalid entry TLB_HAZARD # necessary? PTR_ADDU k1, k1, k0 and k0, k0, PTE_OFFS # check even/odd page bne k0, zero, k_tlb_inv_odd nop mfc0 k0, COP_0_TLB_INDEX bltz k0, sys_stk_chk # probe fail PTE_LOAD k0, 0(k1) # get PTE entry PTE_CLEAR_SWBITS(k0) dmtc0 k0, COP_0_TLB_LO0 # load PTE entry and k0, k0, PG_V # check for valid entry beq k0, zero, go_k_general # PTE invalid PTE_LOAD k0, PTE_OFFS(k1) # get odd PTE entry PTE_CLEAR_SWBITS(k0) dmtc0 k0, COP_0_TLB_LO1 # load PTE entry TLB_HAZARD tlbwi # write TLB TLB_HAZARD #ifdef CPU_LOONGSON2 li k0, COP_0_DIAG_ITLB_CLEAR | COP_0_DIAG_BTB_CLEAR | COP_0_DIAG_RAS_DISABLE dmtc0 k0, COP_0_DIAG #endif ERET k_tlb_inv_odd: mfc0 k0, COP_0_TLB_INDEX bltz k0, sys_stk_chk # probe fail PTE_LOAD k0, 0(k1) # get PTE entry PTE_CLEAR_SWBITS(k0) dmtc0 k0, COP_0_TLB_LO1 # save PTE entry and k0, k0, PG_V # check for valid entry beq k0, zero, go_k_general # PTE invalid PTE_LOAD k0, -PTE_OFFS(k1) # get even PTE entry PTE_CLEAR_SWBITS(k0) dmtc0 k0, COP_0_TLB_LO0 # save PTE entry TLB_HAZARD tlbwi # update TLB TLB_HAZARD #ifdef CPU_LOONGSON2 li k0, COP_0_DIAG_ITLB_CLEAR | COP_0_DIAG_BTB_CLEAR | COP_0_DIAG_RAS_DISABLE dmtc0 k0, COP_0_DIAG #endif ERET END(k_tlb_inv) /*---------------------------------------------------------------- k_tlb_miss * * Handle a TLB miss exception from kernel mode in kernel space. * We must check that this is coming from kernel mode. If not * it's a bad address from user mode so handle properly. * Load up the correct entry contents from the kernel map. * k0 has bad address. */ NLEAF(k_tlb_miss, 0) .set noat mfc0 k1, COP_0_STATUS_REG andi k1, SR_KSU_USER bne k1, zero, go_u_general LA k1, (VM_MIN_KERNEL_ADDRESS) # compute index # (safe if expands to > 1 insn) PTR_SUBU k0, k0, k1 lw k1, Sysmapsize # index within range? PTR_SRL k0, k0, PGSHIFT sltu k1, k0, k1 beq k1, zero, sys_stk_chk # No. check for valid stack PTR_SRL k0, k0, 1 PTR_L k1, Sysmap PTR_SLL k0, k0, (PTE_LOG + 1) # compute offset from index PTR_ADDU k1, k1, k0 PTE_LOAD k0, 0(k1) # get PTE entry PTE_LOAD k1, PTE_OFFS(k1) # get odd PTE entry PTE_CLEAR_SWBITS(k0) dmtc0 k0, COP_0_TLB_LO0 # load PTE entry PTE_CLEAR_SWBITS(k1) dmtc0 k1, COP_0_TLB_LO1 # load PTE entry TLB_HAZARD tlbwr # write TLB TLB_HAZARD #ifdef CPU_LOONGSON2 li k0, COP_0_DIAG_ITLB_CLEAR | COP_0_DIAG_BTB_CLEAR | COP_0_DIAG_RAS_DISABLE dmtc0 k0, COP_0_DIAG #endif ERET sys_stk_chk: GET_CPU_INFO(k1, k0) PTR_L k1, CI_CURPROCPADDR(k1) PTR_SUBU k0, sp, k1 # check to see if we have a sltiu k0, k0, 2048 # valid kernel stack beqz k0, go_k_general # yes, handle. nop LA a0, start-FRAMESZ(CF_SZ)-4*REGSZ # set sp to a valid place #ifdef __mips_n64 mfc0 a4, COP_0_STATUS_REG mfc0 a5, COP_0_CAUSE_REG move a6, sp #else mfc0 a2, COP_0_STATUS_REG mfc0 a3, COP_0_CAUSE_REG REG_S a2, CF_ARGSZ+0*REGSZ(sp) REG_S a3, CF_ARGSZ+1*REGSZ(sp) PTR_S sp, CF_ARGSZ+2*REGSZ(a0) #endif move sp, a0 dmfc0 a3, COP_0_BAD_VADDR move a2, ra dmfc0 a1, COP_0_EXC_PC LA a0, 1f jal printf nop LA sp, start-FRAMESZ(CF_SZ) # set sp to a valid place #ifdef DDB LA a1, db_printf LA a0, 2f jal trapDump nop #endif PANIC("kernel stack overflow") /*noreturn*/ .data 1: .asciiz "\rktlbmiss: PC %p RA %p ADR %p\nSR %p CR %p SP %p\n" 2: .asciiz "stack ovf" .text .set at END(k_tlb_miss) /*---------------------------------------------------------------- tlb_flush * Flush the "random" entries from the TLB. * Uses "wired" register to determine what register to start with. * Arg "tlbsize" is the number of entries to flush. */ LEAF(tlb_flush, 0) mfc0 v1, COP_0_STATUS_REG # Save the status register. ori v0, v1, SR_INT_ENAB xori v0, v0, SR_INT_ENAB mtc0 v0, COP_0_STATUS_REG # Disable interrupts MTC0_SR_IE_HAZARD mfc0 ta1, COP_0_TLB_WIRED LA v0, CKSEG0_BASE # invalid address dmfc0 ta0, COP_0_TLB_HI # Save the PID #ifdef CPU_OCTEON PTR_SLL ta2, ta1, PAGE_SHIFT + 1 PTR_ADDU v0, v0, ta2 #else dmtc0 v0, COP_0_TLB_HI # Mark entry high as invalid #endif dmtc0 zero, COP_0_TLB_LO0 # Zero out low entry0. dmtc0 zero, COP_0_TLB_LO1 # Zero out low entry1. mtc0 zero, COP_0_TLB_PG_MASK # Zero out mask entry. /* * Align the starting value (ta1) and the upper bound (a0). */ 1: mtc0 ta1, COP_0_TLB_INDEX # Set the index register. #ifdef CPU_OCTEON dmtc0 v0, COP_0_TLB_HI # Mark entry high as invalid li ta2, 2 * PAGE_SIZE PTR_ADDU v0, v0, ta2 #endif addu ta1, ta1, 1 # Increment index. TLB_HAZARD tlbwi # Write the TLB entry. TLB_HAZARD bne ta1, a0, 1b nop #ifdef CPU_LOONGSON2 li v0, COP_0_DIAG_ITLB_CLEAR | COP_0_DIAG_BTB_CLEAR | COP_0_DIAG_RAS_DISABLE dmtc0 v0, COP_0_DIAG #endif dmtc0 ta0, COP_0_TLB_HI # Restore the PID li a0, TLB_PAGE_MASK mtc0 a0, COP_0_TLB_PG_MASK # Restore default mask value. mtc0 v1, COP_0_STATUS_REG # Restore the status register MTC0_SR_IE_HAZARD j ra nop END(tlb_flush) /*--------------------------------------------------------------- tlb_flush_addr * Flush any TLB entries for the given address and TLB PID. */ LEAF(tlb_flush_addr, 0) mfc0 v1, COP_0_STATUS_REG # Save the status register. ori v0, v1, SR_INT_ENAB xori v0, v0, SR_INT_ENAB mtc0 v0, COP_0_STATUS_REG # Disable interrupts MTC0_SR_IE_HAZARD dli v0, (PG_HVPN | PG_ASID_MASK) and a0, a0, v0 # Make sure valid hi value. dmfc0 ta0, COP_0_TLB_HI # Get current PID dmtc0 a0, COP_0_TLB_HI # look for addr & PID TLB_HAZARD tlbp # Probe for the entry. TLB_HAZARD # necessary? LA ta1, CKSEG0_BASE # Load invalid entry. mfc0 v0, COP_0_TLB_INDEX # See what we got bltz v0, 1f # index < 0 => !found nop #ifdef CPU_OCTEON PTR_SLL ta2, v0, PAGE_SHIFT + 1 PTR_ADDU ta1, ta1, ta2 #endif dmtc0 ta1, COP_0_TLB_HI # Mark entry high as invalid dmtc0 zero, COP_0_TLB_LO0 # Zero out low entry. dmtc0 zero, COP_0_TLB_LO1 # Zero out low entry. TLB_HAZARD tlbwi TLB_HAZARD #ifdef CPU_LOONGSON2 li v0, COP_0_DIAG_ITLB_CLEAR | COP_0_DIAG_BTB_CLEAR | COP_0_DIAG_RAS_DISABLE dmtc0 v0, COP_0_DIAG #endif 1: dmtc0 ta0, COP_0_TLB_HI # restore PID mtc0 v1, COP_0_STATUS_REG # Restore the status register MTC0_SR_IE_HAZARD j ra nop END(tlb_flush_addr) /*---------------------------------------------------------------- tlb_update * Update the TLB if highreg is found; otherwise, enter the data. */ LEAF(tlb_update, 0) mfc0 v1, COP_0_STATUS_REG # Save the status register. ori v0, v1, SR_INT_ENAB xori v0, v0, SR_INT_ENAB mtc0 v0, COP_0_STATUS_REG # Disable interrupts MTC0_SR_IE_HAZARD and ta1, a0, PG_ODDPG # ta1 = Even/Odd flag dli v0, (PG_HVPN | PG_ASID_MASK) and a0, a0, v0 dmfc0 ta0, COP_0_TLB_HI # Save current PID dmtc0 a0, COP_0_TLB_HI # Init high reg and a2, a1, PG_G # Copy global bit li a3, TLB_PAGE_MASK TLB_HAZARD tlbp # Probe for the entry. TLB_HAZARD # necessary? PTE_CLEAR_SWBITS(a1) mfc0 v0, COP_0_TLB_INDEX # See what we got bne ta1, zero, 2f # Decide even odd nop # EVEN bltz v0, 1f # index < 0 => !found nop TLB_HAZARD tlbr # update, read entry first TLB_HAZARD # necessary? dmtc0 a1, COP_0_TLB_LO0 # init low reg0. TLB_HAZARD tlbwi # update slot found TLB_HAZARD b 4f li v0, 1 1: mtc0 a3, COP_0_TLB_PG_MASK # init mask. dmtc0 a0, COP_0_TLB_HI # init high reg. dmtc0 a1, COP_0_TLB_LO0 # init low reg0. dmtc0 a2, COP_0_TLB_LO1 # init low reg1. TLB_HAZARD tlbwr # enter into a random slot TLB_HAZARD b 4f li v0, 0 # ODD 2: nop bltz v0, 3f # index < 0 => !found nop TLB_HAZARD tlbr # read the entry first TLB_HAZARD # necessary? dmtc0 a1, COP_0_TLB_LO1 # init low reg1. TLB_HAZARD tlbwi # update slot found TLB_HAZARD b 4f li v0, 1 3: mtc0 a3, COP_0_TLB_PG_MASK # init mask. dmtc0 a0, COP_0_TLB_HI # init high reg. dmtc0 a2, COP_0_TLB_LO0 # init low reg0. dmtc0 a1, COP_0_TLB_LO1 # init low reg1. TLB_HAZARD tlbwr # enter into a random slot TLB_HAZARD 4: #ifdef CPU_LOONGSON2 li v0, COP_0_DIAG_ITLB_CLEAR | COP_0_DIAG_BTB_CLEAR | COP_0_DIAG_RAS_DISABLE dmtc0 v0, COP_0_DIAG #endif dmtc0 ta0, COP_0_TLB_HI # restore PID mtc0 v1, COP_0_STATUS_REG # Restore the status register MTC0_SR_IE_HAZARD j ra li v0, 0 END(tlb_update) /*---------------------------------------------------------------- tlb_read * Read the TLB entry. */ LEAF(tlb_read, 0) mfc0 v1, COP_0_STATUS_REG # Save the status register. ori v0, v1, SR_INT_ENAB xori v0, v0, SR_INT_ENAB mtc0 v0, COP_0_STATUS_REG # Disable interrupts MTC0_SR_IE_HAZARD dmfc0 v0, COP_0_TLB_HI # Get current PID mtc0 a0, COP_0_TLB_INDEX # Set the index register TLB_HAZARD tlbr # Read from the TLB TLB_HAZARD # necessary? mfc0 ta0, COP_0_TLB_PG_MASK # fetch the size dmfc0 ta1, COP_0_TLB_HI # fetch the hi entry dmfc0 ta2, COP_0_TLB_LO0 # See what we got dmfc0 ta3, COP_0_TLB_LO1 # See what we got dmtc0 v0, COP_0_TLB_HI # restore PID nop nop nop # wait for PID active li a0, TLB_PAGE_MASK mtc0 a0, COP_0_TLB_PG_MASK # Restore default mask value. mtc0 v1, COP_0_STATUS_REG # Restore the status register MTC0_SR_IE_HAZARD sd ta0, 0(a1) sd ta1, 8(a1) sd ta2, 16(a1) j ra sd ta3, 24(a1) END(tlb_read) /*---------------------------------------------------------------- tlb_get_pid * Read the tlb pid value. */ LEAF(tlb_get_pid, 0) dmfc0 v0, COP_0_TLB_HI # get PID li v1, PG_ASID_MASK # mask off PID j ra and v0, v0, v1 # mask off PID END(tlb_get_pid) /*---------------------------------------------------------------- tlb_set_pid * Write the given pid into the TLB pid reg. */ LEAF(tlb_set_pid, 0) dmtc0 a0, COP_0_TLB_HI # Write the hi reg value TLB_HAZARD j ra nop END(tlb_set_pid) /*---------------------------------------------------------------- tlb_set_wired * Write the given value into the TLB wired reg. */ LEAF(tlb_set_wired, 0) mtc0 a0, COP_0_TLB_WIRED j ra nop END(tlb_set_wired) /* * Initialize the TLB page mask. */ LEAF(tlb_set_page_mask, 0) mtc0 a0, COP_0_TLB_PG_MASK TLB_HAZARD j ra nop END(tlb_set_page_mask)