Loading...
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 | // SPDX-License-Identifier: GPL-2.0 /* * In-kernel vector facility support functions * * Copyright IBM Corp. 2015 * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> */ #include <linux/kernel.h> #include <linux/cpu.h> #include <linux/sched.h> #include <asm/fpu/types.h> #include <asm/fpu/api.h> asm(".include \"asm/vx-insn.h\"\n"); void __kernel_fpu_begin(struct kernel_fpu *state, u32 flags) { /* * Limit the save to the FPU/vector registers already * in use by the previous context */ flags &= state->mask; if (flags & KERNEL_FPC) /* Save floating point control */ asm volatile("stfpc %0" : "=Q" (state->fpc)); if (!MACHINE_HAS_VX) { if (flags & KERNEL_VXR_V0V7) { /* Save floating-point registers */ asm volatile("std 0,%0" : "=Q" (state->fprs[0])); asm volatile("std 1,%0" : "=Q" (state->fprs[1])); asm volatile("std 2,%0" : "=Q" (state->fprs[2])); asm volatile("std 3,%0" : "=Q" (state->fprs[3])); asm volatile("std 4,%0" : "=Q" (state->fprs[4])); asm volatile("std 5,%0" : "=Q" (state->fprs[5])); asm volatile("std 6,%0" : "=Q" (state->fprs[6])); asm volatile("std 7,%0" : "=Q" (state->fprs[7])); asm volatile("std 8,%0" : "=Q" (state->fprs[8])); asm volatile("std 9,%0" : "=Q" (state->fprs[9])); asm volatile("std 10,%0" : "=Q" (state->fprs[10])); asm volatile("std 11,%0" : "=Q" (state->fprs[11])); asm volatile("std 12,%0" : "=Q" (state->fprs[12])); asm volatile("std 13,%0" : "=Q" (state->fprs[13])); asm volatile("std 14,%0" : "=Q" (state->fprs[14])); asm volatile("std 15,%0" : "=Q" (state->fprs[15])); } return; } /* Test and save vector registers */ asm volatile ( /* * Test if any vector register must be saved and, if so, * test if all register can be saved. */ " la 1,%[vxrs]\n" /* load save area */ " tmll %[m],30\n" /* KERNEL_VXR */ " jz 7f\n" /* no work -> done */ " jo 5f\n" /* -> save V0..V31 */ /* * Test for special case KERNEL_FPU_MID only. In this * case a vstm V8..V23 is the best instruction */ " chi %[m],12\n" /* KERNEL_VXR_MID */ " jne 0f\n" /* -> save V8..V23 */ " VSTM 8,23,128,1\n" /* vstm %v8,%v23,128(%r1) */ " j 7f\n" /* Test and save the first half of 16 vector registers */ "0: tmll %[m],6\n" /* KERNEL_VXR_LOW */ " jz 3f\n" /* -> KERNEL_VXR_HIGH */ " jo 2f\n" /* 11 -> save V0..V15 */ " brc 2,1f\n" /* 10 -> save V8..V15 */ " VSTM 0,7,0,1\n" /* vstm %v0,%v7,0(%r1) */ " j 3f\n" "1: VSTM 8,15,128,1\n" /* vstm %v8,%v15,128(%r1) */ " j 3f\n" "2: VSTM 0,15,0,1\n" /* vstm %v0,%v15,0(%r1) */ /* Test and save the second half of 16 vector registers */ "3: tmll %[m],24\n" /* KERNEL_VXR_HIGH */ " jz 7f\n" " jo 6f\n" /* 11 -> save V16..V31 */ " brc 2,4f\n" /* 10 -> save V24..V31 */ " VSTM 16,23,256,1\n" /* vstm %v16,%v23,256(%r1) */ " j 7f\n" "4: VSTM 24,31,384,1\n" /* vstm %v24,%v31,384(%r1) */ " j 7f\n" "5: VSTM 0,15,0,1\n" /* vstm %v0,%v15,0(%r1) */ "6: VSTM 16,31,256,1\n" /* vstm %v16,%v31,256(%r1) */ "7:" : [vxrs] "=Q" (*(struct vx_array *) &state->vxrs) : [m] "d" (flags) : "1", "cc"); } EXPORT_SYMBOL(__kernel_fpu_begin); void __kernel_fpu_end(struct kernel_fpu *state, u32 flags) { /* * Limit the restore to the FPU/vector registers of the * previous context that have been overwritte by the * current context */ flags &= state->mask; if (flags & KERNEL_FPC) /* Restore floating-point controls */ asm volatile("lfpc %0" : : "Q" (state->fpc)); if (!MACHINE_HAS_VX) { if (flags & KERNEL_VXR_V0V7) { /* Restore floating-point registers */ asm volatile("ld 0,%0" : : "Q" (state->fprs[0])); asm volatile("ld 1,%0" : : "Q" (state->fprs[1])); asm volatile("ld 2,%0" : : "Q" (state->fprs[2])); asm volatile("ld 3,%0" : : "Q" (state->fprs[3])); asm volatile("ld 4,%0" : : "Q" (state->fprs[4])); asm volatile("ld 5,%0" : : "Q" (state->fprs[5])); asm volatile("ld 6,%0" : : "Q" (state->fprs[6])); asm volatile("ld 7,%0" : : "Q" (state->fprs[7])); asm volatile("ld 8,%0" : : "Q" (state->fprs[8])); asm volatile("ld 9,%0" : : "Q" (state->fprs[9])); asm volatile("ld 10,%0" : : "Q" (state->fprs[10])); asm volatile("ld 11,%0" : : "Q" (state->fprs[11])); asm volatile("ld 12,%0" : : "Q" (state->fprs[12])); asm volatile("ld 13,%0" : : "Q" (state->fprs[13])); asm volatile("ld 14,%0" : : "Q" (state->fprs[14])); asm volatile("ld 15,%0" : : "Q" (state->fprs[15])); } return; } /* Test and restore (load) vector registers */ asm volatile ( /* * Test if any vector register must be loaded and, if so, * test if all registers can be loaded at once. */ " la 1,%[vxrs]\n" /* load restore area */ " tmll %[m],30\n" /* KERNEL_VXR */ " jz 7f\n" /* no work -> done */ " jo 5f\n" /* -> restore V0..V31 */ /* * Test for special case KERNEL_FPU_MID only. In this * case a vlm V8..V23 is the best instruction */ " chi %[m],12\n" /* KERNEL_VXR_MID */ " jne 0f\n" /* -> restore V8..V23 */ " VLM 8,23,128,1\n" /* vlm %v8,%v23,128(%r1) */ " j 7f\n" /* Test and restore the first half of 16 vector registers */ "0: tmll %[m],6\n" /* KERNEL_VXR_LOW */ " jz 3f\n" /* -> KERNEL_VXR_HIGH */ " jo 2f\n" /* 11 -> restore V0..V15 */ " brc 2,1f\n" /* 10 -> restore V8..V15 */ " VLM 0,7,0,1\n" /* vlm %v0,%v7,0(%r1) */ " j 3f\n" "1: VLM 8,15,128,1\n" /* vlm %v8,%v15,128(%r1) */ " j 3f\n" "2: VLM 0,15,0,1\n" /* vlm %v0,%v15,0(%r1) */ /* Test and restore the second half of 16 vector registers */ "3: tmll %[m],24\n" /* KERNEL_VXR_HIGH */ " jz 7f\n" " jo 6f\n" /* 11 -> restore V16..V31 */ " brc 2,4f\n" /* 10 -> restore V24..V31 */ " VLM 16,23,256,1\n" /* vlm %v16,%v23,256(%r1) */ " j 7f\n" "4: VLM 24,31,384,1\n" /* vlm %v24,%v31,384(%r1) */ " j 7f\n" "5: VLM 0,15,0,1\n" /* vlm %v0,%v15,0(%r1) */ "6: VLM 16,31,256,1\n" /* vlm %v16,%v31,256(%r1) */ "7:" : [vxrs] "=Q" (*(struct vx_array *) &state->vxrs) : [m] "d" (flags) : "1", "cc"); } EXPORT_SYMBOL(__kernel_fpu_end); |