Loading...
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 | /* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_BOOK3S_64_HASH_H #define _ASM_POWERPC_BOOK3S_64_HASH_H #ifdef __KERNEL__ #include <asm/asm-const.h> /* * Common bits between 4K and 64K pages in a linux-style PTE. * Additional bits may be defined in pgtable-hash64-*.h * */ #define H_PTE_NONE_MASK _PAGE_HPTEFLAGS #ifdef CONFIG_PPC_64K_PAGES #include <asm/book3s/64/hash-64k.h> #else #include <asm/book3s/64/hash-4k.h> #endif #define H_PTRS_PER_PTE (1 << H_PTE_INDEX_SIZE) #define H_PTRS_PER_PMD (1 << H_PMD_INDEX_SIZE) #define H_PTRS_PER_PUD (1 << H_PUD_INDEX_SIZE) /* Bits to set in a PMD/PUD/PGD entry valid bit*/ #define HASH_PMD_VAL_BITS (0x8000000000000000UL) #define HASH_PUD_VAL_BITS (0x8000000000000000UL) #define HASH_PGD_VAL_BITS (0x8000000000000000UL) /* * Size of EA range mapped by our pagetables. */ #define H_PGTABLE_EADDR_SIZE (H_PTE_INDEX_SIZE + H_PMD_INDEX_SIZE + \ H_PUD_INDEX_SIZE + H_PGD_INDEX_SIZE + PAGE_SHIFT) #define H_PGTABLE_RANGE (ASM_CONST(1) << H_PGTABLE_EADDR_SIZE) /* * Top 2 bits are ignored in page table walk. */ #define EA_MASK (~(0xcUL << 60)) /* * We store the slot details in the second half of page table. * Increase the pud level table so that hugetlb ptes can be stored * at pud level. */ #if defined(CONFIG_HUGETLB_PAGE) && defined(CONFIG_PPC_64K_PAGES) #define H_PUD_CACHE_INDEX (H_PUD_INDEX_SIZE + 1) #else #define H_PUD_CACHE_INDEX (H_PUD_INDEX_SIZE) #endif /* * +------------------------------+ * | | * | | * | | * +------------------------------+ Kernel virtual map end (0xc00e000000000000) * | | * | | * | 512TB/16TB of vmemmap | * | | * | | * +------------------------------+ Kernel vmemmap start * | | * | 512TB/16TB of IO map | * | | * +------------------------------+ Kernel IO map start * | | * | 512TB/16TB of vmap | * | | * +------------------------------+ Kernel virt start (0xc008000000000000) * | | * | | * | | * +------------------------------+ Kernel linear (0xc.....) */ #define H_VMALLOC_START H_KERN_VIRT_START #define H_VMALLOC_SIZE H_KERN_MAP_SIZE #define H_VMALLOC_END (H_VMALLOC_START + H_VMALLOC_SIZE) #define H_KERN_IO_START H_VMALLOC_END #define H_KERN_IO_SIZE H_KERN_MAP_SIZE #define H_KERN_IO_END (H_KERN_IO_START + H_KERN_IO_SIZE) #define H_VMEMMAP_START H_KERN_IO_END #define H_VMEMMAP_SIZE H_KERN_MAP_SIZE #define H_VMEMMAP_END (H_VMEMMAP_START + H_VMEMMAP_SIZE) #define NON_LINEAR_REGION_ID(ea) ((((unsigned long)ea - H_KERN_VIRT_START) >> REGION_SHIFT) + 2) /* * Region IDs */ #define USER_REGION_ID 0 #define LINEAR_MAP_REGION_ID 1 #define VMALLOC_REGION_ID NON_LINEAR_REGION_ID(H_VMALLOC_START) #define IO_REGION_ID NON_LINEAR_REGION_ID(H_KERN_IO_START) #define VMEMMAP_REGION_ID NON_LINEAR_REGION_ID(H_VMEMMAP_START) #define INVALID_REGION_ID (VMEMMAP_REGION_ID + 1) /* * Defines the address of the vmemap area, in its own region on * hash table CPUs. */ /* PTEIDX nibble */ #define _PTEIDX_SECONDARY 0x8 #define _PTEIDX_GROUP_IX 0x7 #define H_PMD_BAD_BITS (PTE_TABLE_SIZE-1) #define H_PUD_BAD_BITS (PMD_TABLE_SIZE-1) #ifndef __ASSEMBLY__ static inline int get_region_id(unsigned long ea) { int region_id; int id = (ea >> 60UL); if (id == 0) return USER_REGION_ID; if (id != (PAGE_OFFSET >> 60)) return INVALID_REGION_ID; if (ea < H_KERN_VIRT_START) return LINEAR_MAP_REGION_ID; BUILD_BUG_ON(NON_LINEAR_REGION_ID(H_VMALLOC_START) != 2); region_id = NON_LINEAR_REGION_ID(ea); return region_id; } static inline int hash__pmd_same(pmd_t pmd_a, pmd_t pmd_b) { return (((pmd_raw(pmd_a) ^ pmd_raw(pmd_b)) & ~cpu_to_be64(_PAGE_HPTEFLAGS)) == 0); } #define hash__pmd_bad(pmd) (pmd_val(pmd) & H_PMD_BAD_BITS) /* * pud comparison that will work with both pte and page table pointer. */ static inline int hash__pud_same(pud_t pud_a, pud_t pud_b) { return (((pud_raw(pud_a) ^ pud_raw(pud_b)) & ~cpu_to_be64(_PAGE_HPTEFLAGS)) == 0); } #define hash__pud_bad(pud) (pud_val(pud) & H_PUD_BAD_BITS) static inline int hash__p4d_bad(p4d_t p4d) { return (p4d_val(p4d) == 0); } #ifdef CONFIG_STRICT_KERNEL_RWX extern void hash__mark_rodata_ro(void); extern void hash__mark_initmem_nx(void); #endif extern void hpte_need_flush(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned long pte, int huge); unsigned long htab_convert_pte_flags(unsigned long pteflags, unsigned long flags); /* Atomic PTE updates */ static inline unsigned long hash__pte_update(struct mm_struct *mm, unsigned long addr, pte_t *ptep, unsigned long clr, unsigned long set, int huge) { __be64 old_be, tmp_be; unsigned long old; __asm__ __volatile__( "1: ldarx %0,0,%3 # pte_update\n\ and. %1,%0,%6\n\ bne- 1b \n\ andc %1,%0,%4 \n\ or %1,%1,%7\n\ stdcx. %1,0,%3 \n\ bne- 1b" : "=&r" (old_be), "=&r" (tmp_be), "=m" (*ptep) : "r" (ptep), "r" (cpu_to_be64(clr)), "m" (*ptep), "r" (cpu_to_be64(H_PAGE_BUSY)), "r" (cpu_to_be64(set)) : "cc" ); /* huge pages use the old page table lock */ if (!huge) assert_pte_locked(mm, addr); old = be64_to_cpu(old_be); if (old & H_PAGE_HASHPTE) hpte_need_flush(mm, addr, ptep, old, huge); return old; } /* Set the dirty and/or accessed bits atomically in a linux PTE, this * function doesn't need to flush the hash entry */ static inline void hash__ptep_set_access_flags(pte_t *ptep, pte_t entry) { __be64 old, tmp, val, mask; mask = cpu_to_be64(_PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_READ | _PAGE_WRITE | _PAGE_EXEC | _PAGE_SOFT_DIRTY); val = pte_raw(entry) & mask; __asm__ __volatile__( "1: ldarx %0,0,%4\n\ and. %1,%0,%6\n\ bne- 1b \n\ or %0,%3,%0\n\ stdcx. %0,0,%4\n\ bne- 1b" :"=&r" (old), "=&r" (tmp), "=m" (*ptep) :"r" (val), "r" (ptep), "m" (*ptep), "r" (cpu_to_be64(H_PAGE_BUSY)) :"cc"); } static inline int hash__pte_same(pte_t pte_a, pte_t pte_b) { return (((pte_raw(pte_a) ^ pte_raw(pte_b)) & ~cpu_to_be64(_PAGE_HPTEFLAGS)) == 0); } static inline int hash__pte_none(pte_t pte) { return (pte_val(pte) & ~H_PTE_NONE_MASK) == 0; } unsigned long pte_get_hash_gslot(unsigned long vpn, unsigned long shift, int ssize, real_pte_t rpte, unsigned int subpg_index); /* This low level function performs the actual PTE insertion * Setting the PTE depends on the MMU type and other factors. It's * an horrible mess that I'm not going to try to clean up now but * I'm keeping it in one place rather than spread around */ static inline void hash__set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte, int percpu) { /* * Anything else just stores the PTE normally. That covers all 64-bit * cases, and 32-bit non-hash with 32-bit PTEs. */ *ptep = pte; } #ifdef CONFIG_TRANSPARENT_HUGEPAGE extern void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, unsigned long old_pmd); #else static inline void hpte_do_hugepage_flush(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, unsigned long old_pmd) { WARN(1, "%s called with THP disabled\n", __func__); } #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ int hash__map_kernel_page(unsigned long ea, unsigned long pa, pgprot_t prot); extern int __meminit hash__vmemmap_create_mapping(unsigned long start, unsigned long page_size, unsigned long phys); extern void hash__vmemmap_remove_mapping(unsigned long start, unsigned long page_size); int hash__create_section_mapping(unsigned long start, unsigned long end, int nid, pgprot_t prot); int hash__remove_section_mapping(unsigned long start, unsigned long end); void hash__kernel_map_pages(struct page *page, int numpages, int enable); #endif /* !__ASSEMBLY__ */ #endif /* __KERNEL__ */ #endif /* _ASM_POWERPC_BOOK3S_64_HASH_H */ |