1/* 2 * ICSWX and ACOP Management 3 * 4 * Copyright (C) 2011 Anton Blanchard, IBM Corp. <anton@samba.org> 5 * 6 * This program is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU General Public License 8 * as published by the Free Software Foundation; either version 9 * 2 of the License, or (at your option) any later version. 10 * 11 */ 12 13#include <linux/sched.h> 14#include <linux/kernel.h> 15#include <linux/errno.h> 16#include <linux/types.h> 17#include <linux/mm.h> 18#include <linux/spinlock.h> 19#include <linux/module.h> 20#include <linux/uaccess.h> 21 22#include "icswx.h" 23 24/* 25 * The processor and its L2 cache cause the icswx instruction to 26 * generate a COP_REQ transaction on PowerBus. The transaction has no 27 * address, and the processor does not perform an MMU access to 28 * authenticate the transaction. The command portion of the PowerBus 29 * COP_REQ transaction includes the LPAR_ID (LPID) and the coprocessor 30 * Process ID (PID), which the coprocessor compares to the authorized 31 * LPID and PID held in the coprocessor, to determine if the process 32 * is authorized to generate the transaction. The data of the COP_REQ 33 * transaction is 128-byte or less in size and is placed in cacheable 34 * memory on a 128-byte cache line boundary. 35 * 36 * The task to use a coprocessor should use use_cop() to mark the use 37 * of the Coprocessor Type (CT) and context switching. On a server 38 * class processor, the PID register is used only for coprocessor 39 * management + * and so a coprocessor PID is allocated before 40 * executing icswx + * instruction. Drop_cop() is used to free the 41 * coprocessor PID. 42 * 43 * Example: 44 * Host Fabric Interface (HFI) is a PowerPC network coprocessor. 45 * Each HFI have multiple windows. Each HFI window serves as a 46 * network device sending to and receiving from HFI network. 47 * HFI immediate send function uses icswx instruction. The immediate 48 * send function allows small (single cache-line) packets be sent 49 * without using the regular HFI send FIFO and doorbell, which are 50 * much slower than immediate send. 51 * 52 * For each task intending to use HFI immediate send, the HFI driver 53 * calls use_cop() to obtain a coprocessor PID for the task. 54 * The HFI driver then allocate a free HFI window and save the 55 * coprocessor PID to the HFI window to allow the task to use the 56 * HFI window. 57 * 58 * The HFI driver repeatedly creates immediate send packets and 59 * issues icswx instruction to send data through the HFI window. 60 * The HFI compares the coprocessor PID in the CPU PID register 61 * to the PID held in the HFI window to determine if the transaction 62 * is allowed. 63 * 64 * When the task to release the HFI window, the HFI driver calls 65 * drop_cop() to release the coprocessor PID. 66 */ 67 68void switch_cop(struct mm_struct *next) 69{ 70#ifdef CONFIG_PPC_ICSWX_PID 71 mtspr(SPRN_PID, next->context.cop_pid); 72#endif 73 mtspr(SPRN_ACOP, next->context.acop); 74} 75 76/** 77 * Start using a coprocessor. 78 * @acop: mask of coprocessor to be used. 79 * @mm: The mm the coprocessor to associate with. Most likely current mm. 80 * 81 * Return a positive PID if successful. Negative errno otherwise. 82 * The returned PID will be fed to the coprocessor to determine if an 83 * icswx transaction is authenticated. 84 */ 85int use_cop(unsigned long acop, struct mm_struct *mm) 86{ 87 int ret; 88 89 if (!cpu_has_feature(CPU_FTR_ICSWX)) 90 return -ENODEV; 91 92 if (!mm || !acop) 93 return -EINVAL; 94 95 /* The page_table_lock ensures mm_users won't change under us */ 96 spin_lock(&mm->page_table_lock); 97 spin_lock(mm->context.cop_lockp); 98 99 ret = get_cop_pid(mm); 100 if (ret < 0) 101 goto out; 102 103 /* update acop */ 104 mm->context.acop |= acop; 105 106 sync_cop(mm); 107 108 /* 109 * If this is a threaded process then there might be other threads 110 * running. We need to send an IPI to force them to pick up any 111 * change in PID and ACOP. 112 */ 113 if (atomic_read(&mm->mm_users) > 1) 114 smp_call_function(sync_cop, mm, 1); 115 116out: 117 spin_unlock(mm->context.cop_lockp); 118 spin_unlock(&mm->page_table_lock); 119 120 return ret; 121} 122EXPORT_SYMBOL_GPL(use_cop); 123 124/** 125 * Stop using a coprocessor. 126 * @acop: mask of coprocessor to be stopped. 127 * @mm: The mm the coprocessor associated with. 128 */ 129void drop_cop(unsigned long acop, struct mm_struct *mm) 130{ 131 int free_pid; 132 133 if (!cpu_has_feature(CPU_FTR_ICSWX)) 134 return; 135 136 if (WARN_ON_ONCE(!mm)) 137 return; 138 139 /* The page_table_lock ensures mm_users won't change under us */ 140 spin_lock(&mm->page_table_lock); 141 spin_lock(mm->context.cop_lockp); 142 143 mm->context.acop &= ~acop; 144 145 free_pid = disable_cop_pid(mm); 146 sync_cop(mm); 147 148 /* 149 * If this is a threaded process then there might be other threads 150 * running. We need to send an IPI to force them to pick up any 151 * change in PID and ACOP. 152 */ 153 if (atomic_read(&mm->mm_users) > 1) 154 smp_call_function(sync_cop, mm, 1); 155 156 if (free_pid != COP_PID_NONE) 157 free_cop_pid(free_pid); 158 159 spin_unlock(mm->context.cop_lockp); 160 spin_unlock(&mm->page_table_lock); 161} 162EXPORT_SYMBOL_GPL(drop_cop); 163 164static int acop_use_cop(int ct) 165{ 166 /* There is no alternate policy, yet */ 167 return -1; 168} 169 170/* 171 * Get the instruction word at the NIP 172 */ 173static u32 acop_get_inst(struct pt_regs *regs) 174{ 175 u32 inst; 176 u32 __user *p; 177 178 p = (u32 __user *)regs->nip; 179 if (!access_ok(VERIFY_READ, p, sizeof(*p))) 180 return 0; 181 182 if (__get_user(inst, p)) 183 return 0; 184 185 return inst; 186} 187 188/** 189 * @regs: regsiters at time of interrupt 190 * @address: storage address 191 * @error_code: Fault code, usually the DSISR or ESR depending on 192 * processor type 193 * 194 * Return 0 if we are able to resolve the data storage fault that 195 * results from a CT miss in the ACOP register. 196 */ 197int acop_handle_fault(struct pt_regs *regs, unsigned long address, 198 unsigned long error_code) 199{ 200 int ct; 201 u32 inst = 0; 202 203 if (!cpu_has_feature(CPU_FTR_ICSWX)) { 204 pr_info("No coprocessors available"); 205 _exception(SIGILL, regs, ILL_ILLOPN, address); 206 } 207 208 if (!user_mode(regs)) { 209 /* this could happen if the HV denies the 210 * kernel access, for now we just die */ 211 die("ICSWX from kernel failed", regs, SIGSEGV); 212 } 213 214 /* Some implementations leave us a hint for the CT */ 215 ct = ICSWX_GET_CT_HINT(error_code); 216 if (ct < 0) { 217 /* we have to peek at the instruction word to figure out CT */ 218 u32 ccw; 219 u32 rs; 220 221 inst = acop_get_inst(regs); 222 if (inst == 0) 223 return -1; 224 225 rs = (inst >> (31 - 10)) & 0x1f; 226 ccw = regs->gpr[rs]; 227 ct = (ccw >> 16) & 0x3f; 228 } 229 230 /* 231 * We could be here because another thread has enabled acop 232 * but the ACOP register has yet to be updated. 233 * 234 * This should have been taken care of by the IPI to sync all 235 * the threads (see smp_call_function(sync_cop, mm, 1)), but 236 * that could take forever if there are a significant amount 237 * of threads. 238 * 239 * Given the number of threads on some of these systems, 240 * perhaps this is the best way to sync ACOP rather than whack 241 * every thread with an IPI. 242 */ 243 if ((acop_copro_type_bit(ct) & current->active_mm->context.acop) != 0) { 244 sync_cop(current->active_mm); 245 return 0; 246 } 247 248 /* check for alternate policy */ 249 if (!acop_use_cop(ct)) 250 return 0; 251 252 /* at this point the CT is unknown to the system */ 253 pr_warn("%s[%d]: Coprocessor %d is unavailable\n", 254 current->comm, current->pid, ct); 255 256 /* get inst if we don't already have it */ 257 if (inst == 0) { 258 inst = acop_get_inst(regs); 259 if (inst == 0) 260 return -1; 261 } 262 263 /* Check if the instruction is the "record form" */ 264 if (inst & 1) { 265 /* 266 * the instruction is "record" form so we can reject 267 * using CR0 268 */ 269 regs->ccr &= ~(0xful << 28); 270 regs->ccr |= ICSWX_RC_NOT_FOUND << 28; 271 272 /* Move on to the next instruction */ 273 regs->nip += 4; 274 } else { 275 /* 276 * There is no architected mechanism to report a bad 277 * CT so we could either SIGILL or report nothing. 278 * Since the non-record version should only bu used 279 * for "hints" or "don't care" we should probably do 280 * nothing. However, I could see how some people 281 * might want an SIGILL so it here if you want it. 282 */ 283#ifdef CONFIG_PPC_ICSWX_USE_SIGILL 284 _exception(SIGILL, regs, ILL_ILLOPN, address); 285#else 286 regs->nip += 4; 287#endif 288 } 289 290 return 0; 291} 292EXPORT_SYMBOL_GPL(acop_handle_fault); 293