Loading...
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 | /* * TLB exception handling code for r4k. * * Copyright (C) 1994, 1995, 1996 by Ralf Baechle and Andreas Busse * * Multi-cpu abstraction and reworking: * Copyright (C) 1996 David S. Miller (dm@engr.sgi.com) * * Carsten Langgaard, carstenl@mips.com * Copyright (C) 2000 MIPS Technologies, Inc. All rights reserved. */ #include <linux/init.h> #include <linux/config.h> #include <asm/asm.h> #include <asm/offset.h> #include <asm/cachectl.h> #include <asm/fpregdef.h> #include <asm/mipsregs.h> #include <asm/page.h> #include <asm/pgtable-bits.h> #include <asm/processor.h> #include <asm/regdef.h> #include <asm/stackframe.h> #include <asm/war.h> #define TLB_OPTIMIZE /* If you are paranoid, disable this. */ #ifdef CONFIG_64BIT_PHYS_ADDR #define PTE_L ld #define PTE_S sd #define PTE_SRL dsrl #define P_MTC0 dmtc0 #define PTE_SIZE 8 #define PTEP_INDX_MSK 0xff0 #define PTE_INDX_MSK 0xff8 #define PTE_INDX_SHIFT 9 #else #define PTE_L lw #define PTE_S sw #define PTE_SRL srl #define P_MTC0 mtc0 #define PTE_SIZE 4 #define PTEP_INDX_MSK 0xff8 #define PTE_INDX_MSK 0xffc #define PTE_INDX_SHIFT 10 #endif /* * ABUSE of CPP macros 101. * * After this macro runs, the pte faulted on is * in register PTE, a ptr into the table in which * the pte belongs is in PTR. */ #ifdef CONFIG_SMP #define GET_PGD(scratch, ptr) \ mfc0 ptr, CP0_CONTEXT; \ la scratch, pgd_current;\ srl ptr, 23; \ sll ptr, 2; \ addu ptr, scratch, ptr; \ lw ptr, (ptr); #else #define GET_PGD(scratch, ptr) \ lw ptr, pgd_current; #endif #define LOAD_PTE(pte, ptr) \ GET_PGD(pte, ptr) \ mfc0 pte, CP0_BADVADDR; \ srl pte, pte, _PGDIR_SHIFT; \ sll pte, pte, 2; \ addu ptr, ptr, pte; \ mfc0 pte, CP0_BADVADDR; \ lw ptr, (ptr); \ srl pte, pte, PTE_INDX_SHIFT; \ and pte, pte, PTE_INDX_MSK; \ addu ptr, ptr, pte; \ PTE_L pte, (ptr); /* This places the even/odd pte pair in the page * table at PTR into ENTRYLO0 and ENTRYLO1 using * TMP as a scratch register. */ #define PTE_RELOAD(ptr, tmp) \ ori ptr, ptr, PTE_SIZE; \ xori ptr, ptr, PTE_SIZE; \ PTE_L tmp, PTE_SIZE(ptr); \ PTE_L ptr, 0(ptr); \ PTE_SRL tmp, tmp, 6; \ P_MTC0 tmp, CP0_ENTRYLO1; \ PTE_SRL ptr, ptr, 6; \ P_MTC0 ptr, CP0_ENTRYLO0; #define DO_FAULT(write) \ .set noat; \ SAVE_ALL; \ mfc0 a2, CP0_BADVADDR; \ KMODE; \ .set at; \ move a0, sp; \ jal do_page_fault; \ li a1, write; \ j ret_from_exception; \ nop; \ .set noat; /* Check is PTE is present, if not then jump to LABEL. * PTR points to the page table where this PTE is located, * when the macro is done executing PTE will be restored * with it's original value. */ #define PTE_PRESENT(pte, ptr, label) \ andi pte, pte, (_PAGE_PRESENT | _PAGE_READ); \ xori pte, pte, (_PAGE_PRESENT | _PAGE_READ); \ bnez pte, label; \ PTE_L pte, (ptr); /* Make PTE valid, store result in PTR. */ #define PTE_MAKEVALID(pte, ptr) \ ori pte, pte, (_PAGE_VALID | _PAGE_ACCESSED); \ PTE_S pte, (ptr); /* Check if PTE can be written to, if not branch to LABEL. * Regardless restore PTE with value from PTR when done. */ #define PTE_WRITABLE(pte, ptr, label) \ andi pte, pte, (_PAGE_PRESENT | _PAGE_WRITE); \ xori pte, pte, (_PAGE_PRESENT | _PAGE_WRITE); \ bnez pte, label; \ PTE_L pte, (ptr); /* Make PTE writable, update software status bits as well, * then store at PTR. */ #define PTE_MAKEWRITE(pte, ptr) \ ori pte, pte, (_PAGE_ACCESSED | _PAGE_MODIFIED | \ _PAGE_VALID | _PAGE_DIRTY); \ PTE_S pte, (ptr); __INIT #ifdef CONFIG_64BIT_PHYS_ADDR #define GET_PTE_OFF(reg) #elif CONFIG_CPU_VR41XX #define GET_PTE_OFF(reg) srl reg, reg, 3 #else #define GET_PTE_OFF(reg) srl reg, reg, 1 #endif /* * These handlers much be written in a relocatable manner * because based upon the cpu type an arbitrary one of the * following pieces of code will be copied to the KSEG0 * vector location. */ /* TLB refill, EXL == 0, R4xx0, non-R4600 version */ .set noreorder .set noat LEAF(except_vec0_r4000) .set mips3 GET_PGD(k0, k1) # get pgd pointer mfc0 k0, CP0_BADVADDR # Get faulting address srl k0, k0, _PGDIR_SHIFT # get pgd only bits sll k0, k0, 2 addu k1, k1, k0 # add in pgd offset mfc0 k0, CP0_CONTEXT # get context reg lw k1, (k1) GET_PTE_OFF(k0) # get pte offset and k0, k0, PTEP_INDX_MSK addu k1, k1, k0 # add in offset PTE_L k0, 0(k1) # get even pte PTE_L k1, PTE_SIZE(k1) # get odd pte PTE_SRL k0, k0, 6 # convert to entrylo0 P_MTC0 k0, CP0_ENTRYLO0 # load it PTE_SRL k1, k1, 6 # convert to entrylo1 P_MTC0 k1, CP0_ENTRYLO1 # load it b 1f tlbwr # write random tlb entry 1: nop eret # return from trap END(except_vec0_r4000) /* TLB refill, EXL == 0, R4600 version */ LEAF(except_vec0_r4600) .set mips3 GET_PGD(k0, k1) # get pgd pointer mfc0 k0, CP0_BADVADDR srl k0, k0, _PGDIR_SHIFT sll k0, k0, 2 # log2(sizeof(pgd_t) addu k1, k1, k0 mfc0 k0, CP0_CONTEXT lw k1, (k1) GET_PTE_OFF(k0) # get pte offset and k0, k0, PTEP_INDX_MSK addu k1, k1, k0 PTE_L k0, 0(k1) PTE_L k1, PTE_SIZE(k1) PTE_SRL k0, k0, 6 P_MTC0 k0, CP0_ENTRYLO0 PTE_SRL k1, k1, 6 P_MTC0 k1, CP0_ENTRYLO1 nop tlbwr nop eret END(except_vec0_r4600) /* TLB refill, EXL == 0, R52x0 "Nevada" version */ /* * This version has a bug workaround for the Nevada. It seems * as if under certain circumstances the move from cp0_context * might produce a bogus result when the mfc0 instruction and * it's consumer are in a different cacheline or a load instruction, * probably any memory reference, is between them. This is * potencially slower than the R4000 version, so we use this * special version. */ .set noreorder .set noat LEAF(except_vec0_nevada) .set mips3 mfc0 k0, CP0_BADVADDR # Get faulting address srl k0, k0, _PGDIR_SHIFT # get pgd only bits lw k1, pgd_current # get pgd pointer sll k0, k0, 2 # log2(sizeof(pgd_t) addu k1, k1, k0 # add in pgd offset lw k1, (k1) mfc0 k0, CP0_CONTEXT # get context reg GET_PTE_OFF(k0) # get pte offset and k0, k0, PTEP_INDX_MSK addu k1, k1, k0 # add in offset PTE_L k0, 0(k1) # get even pte PTE_L k1, PTE_SIZE(k1) # get odd pte PTE_SRL k0, k0, 6 # convert to entrylo0 P_MTC0 k0, CP0_ENTRYLO0 # load it PTE_SRL k1, k1, 6 # convert to entrylo1 P_MTC0 k1, CP0_ENTRYLO1 # load it nop # QED specified nops nop tlbwr # write random tlb entry nop # traditional nop eret # return from trap END(except_vec0_nevada) /* TLB refill, EXL == 0, SB1 with M3 errata handling version */ LEAF(except_vec0_sb1) #if BCM1250_M3_WAR mfc0 k0, CP0_BADVADDR mfc0 k1, CP0_ENTRYHI xor k0, k1 srl k0, k0, PAGE_SHIFT+1 bnez k0, 1f #endif GET_PGD(k0, k1) # get pgd pointer mfc0 k0, CP0_BADVADDR # Get faulting address srl k0, k0, _PGDIR_SHIFT # get pgd only bits sll k0, k0, 2 addu k1, k1, k0 # add in pgd offset mfc0 k0, CP0_CONTEXT # get context reg lw k1, (k1) GET_PTE_OFF(k0) # get pte offset and k0, k0, PTEP_INDX_MSK addu k1, k1, k0 # add in offset PTE_L k0, 0(k1) # get even pte PTE_L k1, PTE_SIZE(k1) # get odd pte PTE_SRL k0, k0, 6 # convert to entrylo0 P_MTC0 k0, CP0_ENTRYLO0 # load it PTE_SRL k1, k1, 6 # convert to entrylo1 P_MTC0 k1, CP0_ENTRYLO1 # load it tlbwr # write random tlb entry 1: eret # return from trap END(except_vec0_sb1) /* TLB refill, EXL == 0, R4[40]00/R5000 badvaddr hwbug version */ LEAF(except_vec0_r45k_bvahwbug) .set mips3 GET_PGD(k0, k1) # get pgd pointer mfc0 k0, CP0_BADVADDR srl k0, k0, _PGDIR_SHIFT sll k0, k0, 2 # log2(sizeof(pgd_t) addu k1, k1, k0 mfc0 k0, CP0_CONTEXT lw k1, (k1) #ifndef CONFIG_64BIT_PHYS_ADDR srl k0, k0, 1 #endif and k0, k0, PTEP_INDX_MSK addu k1, k1, k0 PTE_L k0, 0(k1) PTE_L k1, PTE_SIZE(k1) nop /* XXX */ tlbp PTE_SRL k0, k0, 6 P_MTC0 k0, CP0_ENTRYLO0 PTE_SRL k1, k1, 6 mfc0 k0, CP0_INDEX P_MTC0 k1, CP0_ENTRYLO1 bltzl k0, 1f tlbwr 1: nop eret END(except_vec0_r45k_bvahwbug) #ifdef CONFIG_SMP /* TLB refill, EXL == 0, R4000 MP badvaddr hwbug version */ LEAF(except_vec0_r4k_mphwbug) .set mips3 GET_PGD(k0, k1) # get pgd pointer mfc0 k0, CP0_BADVADDR srl k0, k0, _PGDIR_SHIFT sll k0, k0, 2 # log2(sizeof(pgd_t) addu k1, k1, k0 mfc0 k0, CP0_CONTEXT lw k1, (k1) #ifndef CONFIG_64BIT_PHYS_ADDR srl k0, k0, 1 #endif and k0, k0, PTEP_INDX_MSK addu k1, k1, k0 PTE_L k0, 0(k1) PTE_L k1, PTE_SIZE(k1) nop /* XXX */ tlbp PTE_SRL k0, k0, 6 P_MTC0 k0, CP0_ENTRYLO0 PTE_SRL k1, k1, 6 mfc0 k0, CP0_INDEX P_MTC0 k1, CP0_ENTRYLO1 bltzl k0, 1f tlbwr 1: nop eret END(except_vec0_r4k_mphwbug) #endif /* TLB refill, EXL == 0, R4000 UP 250MHZ entrylo[01] hwbug version */ LEAF(except_vec0_r4k_250MHZhwbug) .set mips3 GET_PGD(k0, k1) # get pgd pointer mfc0 k0, CP0_BADVADDR srl k0, k0, _PGDIR_SHIFT sll k0, k0, 2 # log2(sizeof(pgd_t) addu k1, k1, k0 mfc0 k0, CP0_CONTEXT lw k1, (k1) #ifndef CONFIG_64BIT_PHYS_ADDR srl k0, k0, 1 #endif and k0, k0, PTEP_INDX_MSK addu k1, k1, k0 PTE_L k0, 0(k1) PTE_L k1, PTE_SIZE(k1) PTE_SRL k0, k0, 6 P_MTC0 zero, CP0_ENTRYLO0 P_MTC0 k0, CP0_ENTRYLO0 PTE_SRL k1, k1, 6 P_MTC0 zero, CP0_ENTRYLO1 P_MTC0 k1, CP0_ENTRYLO1 b 1f tlbwr 1: nop eret END(except_vec0_r4k_250MHZhwbug) #ifdef CONFIG_SMP /* TLB refill, EXL == 0, R4000 MP 250MHZ entrylo[01]+badvaddr bug version */ LEAF(except_vec0_r4k_MP250MHZhwbug) .set mips3 GET_PGD(k0, k1) # get pgd pointer mfc0 k0, CP0_BADVADDR srl k0, k0, _PGDIR_SHIFT sll k0, k0, 2 # log2(sizeof(pgd_t) addu k1, k1, k0 mfc0 k0, CP0_CONTEXT lw k1, (k1) #ifndef CONFIG_64BIT_PHYS_ADDR srl k0, k0, 1 #endif and k0, k0, PTEP_INDX_MSK addu k1, k1, k0 PTE_L k0, 0(k1) PTE_L k1, PTE_SIZE(k1) nop /* XXX */ tlbp PTE_SRL k0, k0, 6 P_MTC0 zero, CP0_ENTRYLO0 P_MTC0 k0, CP0_ENTRYLO0 mfc0 k0, CP0_INDEX PTE_SRL k1, k1, 6 P_MTC0 zero, CP0_ENTRYLO1 P_MTC0 k1, CP0_ENTRYLO1 bltzl k0, 1f tlbwr 1: nop eret END(except_vec0_r4k_MP250MHZhwbug) #endif __FINIT .set noreorder /* * From the IDT errata for the QED RM5230 (Nevada), processor revision 1.0: * 2. A timing hazard exists for the TLBP instruction. * * stalling_instruction * TLBP * * The JTLB is being read for the TLBP throughout the stall generated by the * previous instruction. This is not really correct as the stalling instruction * can modify the address used to access the JTLB. The failure symptom is that * the TLBP instruction will use an address created for the stalling instruction * and not the address held in C0_ENHI and thus report the wrong results. * * The software work-around is to not allow the instruction preceding the TLBP * to stall - make it an NOP or some other instruction guaranteed not to stall. * * Errata 2 will not be fixed. This errata is also on the R5000. * * As if we MIPS hackers wouldn't know how to nop pipelines happy ... */ #define R5K_HAZARD nop /* * Note for many R4k variants tlb probes cannot be executed out * of the instruction cache else you get bogus results. */ .align 5 NESTED(handle_tlbl, PT_SIZE, sp) .set noat #if BCM1250_M3_WAR mfc0 k0, CP0_BADVADDR mfc0 k1, CP0_ENTRYHI xor k0, k1 srl k0, k0, PAGE_SHIFT+1 beqz k0, 1f nop .set mips3 eret .set mips0 1: #endif invalid_tlbl: #ifdef TLB_OPTIMIZE /* Test present bit in entry. */ LOAD_PTE(k0, k1) R5K_HAZARD tlbp PTE_PRESENT(k0, k1, nopage_tlbl) PTE_MAKEVALID(k0, k1) PTE_RELOAD(k1, k0) nop b 1f tlbwi 1: nop .set mips3 eret .set mips0 #endif nopage_tlbl: DO_FAULT(0) END(handle_tlbl) .align 5 NESTED(handle_tlbs, PT_SIZE, sp) .set noat #ifdef TLB_OPTIMIZE .set mips3 li k0,0 LOAD_PTE(k0, k1) R5K_HAZARD tlbp # find faulting entry PTE_WRITABLE(k0, k1, nopage_tlbs) PTE_MAKEWRITE(k0, k1) PTE_RELOAD(k1, k0) nop b 1f tlbwi 1: nop .set mips3 eret .set mips0 #endif nopage_tlbs: DO_FAULT(1) END(handle_tlbs) .align 5 NESTED(handle_mod, PT_SIZE, sp) .set noat #ifdef TLB_OPTIMIZE .set mips3 LOAD_PTE(k0, k1) R5K_HAZARD tlbp # find faulting entry andi k0, k0, _PAGE_WRITE beqz k0, nowrite_mod PTE_L k0, (k1) /* Present and writable bits set, set accessed and dirty bits. */ PTE_MAKEWRITE(k0, k1) /* Now reload the entry into the tlb. */ PTE_RELOAD(k1, k0) nop b 1f tlbwi 1: nop .set mips3 eret .set mips0 #endif nowrite_mod: DO_FAULT(1) END(handle_mod) |