arch/arm64/kernel/fpsimd.c

/* [<][>][^][v][top][bottom][index][help] */
This source file includes following definitions.
__get_cpu_fpsimd_context
get_cpu_fpsimd_context
__put_cpu_fpsimd_context
put_cpu_fpsimd_context
have_cpu_fpsimd_context
__sve_free
sve_free
task_fpsimd_load
fpsimd_save
find_supported_vector_length
sve_proc_do_default_vl
sve_sysctl_init
sve_sysctl_init
arm64_cpu_to_le128
arm64_cpu_to_le128
__fpsimd_to_sve
fpsimd_to_sve
sve_to_fpsimd
sve_state_size
sve_alloc
fpsimd_sync_to_sve
sve_sync_to_fpsimd
sve_sync_from_fpsimd_zeropad
sve_set_vector_length
sve_prctl_status
sve_set_current_vl
sve_get_current_vl
sve_probe_vqs
sve_init_vq_map
sve_update_vq_map
sve_verify_vq_map
sve_efi_setup
sve_kernel_enable
read_zcr_features
sve_setup
fpsimd_release_task
do_sve_acc
do_fpsimd_acc
do_fpsimd_exc
fpsimd_thread_switch
fpsimd_flush_thread
fpsimd_preserve_current_state
fpsimd_signal_preserve_current_state
fpsimd_bind_task_to_cpu
fpsimd_bind_state_to_cpu
fpsimd_restore_current_state
fpsimd_update_current_state
fpsimd_flush_task_state
fpsimd_flush_cpu_state
fpsimd_save_and_flush_cpu_state
kernel_neon_begin
kernel_neon_end
__efi_fpsimd_begin
__efi_fpsimd_end
fpsimd_cpu_pm_notifier
fpsimd_pm_init
fpsimd_pm_init
fpsimd_cpu_dead
fpsimd_hotplug_init
fpsimd_hotplug_init
fpsimd_init
   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * FP/SIMD context switching and fault handling
   4  *
   5  * Copyright (C) 2012 ARM Ltd.
   6  * Author: Catalin Marinas <catalin.marinas@arm.com>
   7  */
   8 
   9 #include <linux/bitmap.h>
  10 #include <linux/bitops.h>
  11 #include <linux/bottom_half.h>
  12 #include <linux/bug.h>
  13 #include <linux/cache.h>
  14 #include <linux/compat.h>
  15 #include <linux/cpu.h>
  16 #include <linux/cpu_pm.h>
  17 #include <linux/kernel.h>
  18 #include <linux/linkage.h>
  19 #include <linux/irqflags.h>
  20 #include <linux/init.h>
  21 #include <linux/percpu.h>
  22 #include <linux/prctl.h>
  23 #include <linux/preempt.h>
  24 #include <linux/ptrace.h>
  25 #include <linux/sched/signal.h>
  26 #include <linux/sched/task_stack.h>
  27 #include <linux/signal.h>
  28 #include <linux/slab.h>
  29 #include <linux/stddef.h>
  30 #include <linux/sysctl.h>
  31 #include <linux/swab.h>
  32 
  33 #include <asm/esr.h>
  34 #include <asm/fpsimd.h>
  35 #include <asm/cpufeature.h>
  36 #include <asm/cputype.h>
  37 #include <asm/processor.h>
  38 #include <asm/simd.h>
  39 #include <asm/sigcontext.h>
  40 #include <asm/sysreg.h>
  41 #include <asm/traps.h>
  42 #include <asm/virt.h>
  43 
  44 #define FPEXC_IOF       (1 << 0)
  45 #define FPEXC_DZF       (1 << 1)
  46 #define FPEXC_OFF       (1 << 2)
  47 #define FPEXC_UFF       (1 << 3)
  48 #define FPEXC_IXF       (1 << 4)
  49 #define FPEXC_IDF       (1 << 7)
  50 
  51 /*
  52  * (Note: in this discussion, statements about FPSIMD apply equally to SVE.)
  53  *
  54  * In order to reduce the number of times the FPSIMD state is needlessly saved
  55  * and restored, we need to keep track of two things:
  56  * (a) for each task, we need to remember which CPU was the last one to have
  57  *     the task's FPSIMD state loaded into its FPSIMD registers;
  58  * (b) for each CPU, we need to remember which task's userland FPSIMD state has
  59  *     been loaded into its FPSIMD registers most recently, or whether it has
  60  *     been used to perform kernel mode NEON in the meantime.
  61  *
  62  * For (a), we add a fpsimd_cpu field to thread_struct, which gets updated to
  63  * the id of the current CPU every time the state is loaded onto a CPU. For (b),
  64  * we add the per-cpu variable 'fpsimd_last_state' (below), which contains the
  65  * address of the userland FPSIMD state of the task that was loaded onto the CPU
  66  * the most recently, or NULL if kernel mode NEON has been performed after that.
  67  *
  68  * With this in place, we no longer have to restore the next FPSIMD state right
  69  * when switching between tasks. Instead, we can defer this check to userland
  70  * resume, at which time we verify whether the CPU's fpsimd_last_state and the
  71  * task's fpsimd_cpu are still mutually in sync. If this is the case, we
  72  * can omit the FPSIMD restore.
  73  *
  74  * As an optimization, we use the thread_info flag TIF_FOREIGN_FPSTATE to
  75  * indicate whether or not the userland FPSIMD state of the current task is
  76  * present in the registers. The flag is set unless the FPSIMD registers of this
  77  * CPU currently contain the most recent userland FPSIMD state of the current
  78  * task.
  79  *
  80  * In order to allow softirq handlers to use FPSIMD, kernel_neon_begin() may
  81  * save the task's FPSIMD context back to task_struct from softirq context.
  82  * To prevent this from racing with the manipulation of the task's FPSIMD state
  83  * from task context and thereby corrupting the state, it is necessary to
  84  * protect any manipulation of a task's fpsimd_state or TIF_FOREIGN_FPSTATE
  85  * flag with {, __}get_cpu_fpsimd_context(). This will still allow softirqs to
  86  * run but prevent them to use FPSIMD.
  87  *
  88  * For a certain task, the sequence may look something like this:
  89  * - the task gets scheduled in; if both the task's fpsimd_cpu field
  90  *   contains the id of the current CPU, and the CPU's fpsimd_last_state per-cpu
  91  *   variable points to the task's fpsimd_state, the TIF_FOREIGN_FPSTATE flag is
  92  *   cleared, otherwise it is set;
  93  *
  94  * - the task returns to userland; if TIF_FOREIGN_FPSTATE is set, the task's
  95  *   userland FPSIMD state is copied from memory to the registers, the task's
  96  *   fpsimd_cpu field is set to the id of the current CPU, the current
  97  *   CPU's fpsimd_last_state pointer is set to this task's fpsimd_state and the
  98  *   TIF_FOREIGN_FPSTATE flag is cleared;
  99  *
 100  * - the task executes an ordinary syscall; upon return to userland, the
 101  *   TIF_FOREIGN_FPSTATE flag will still be cleared, so no FPSIMD state is
 102  *   restored;
 103  *
 104  * - the task executes a syscall which executes some NEON instructions; this is
 105  *   preceded by a call to kernel_neon_begin(), which copies the task's FPSIMD
 106  *   register contents to memory, clears the fpsimd_last_state per-cpu variable
 107  *   and sets the TIF_FOREIGN_FPSTATE flag;
 108  *
 109  * - the task gets preempted after kernel_neon_end() is called; as we have not
 110  *   returned from the 2nd syscall yet, TIF_FOREIGN_FPSTATE is still set so
 111  *   whatever is in the FPSIMD registers is not saved to memory, but discarded.
 112  */
 113 struct fpsimd_last_state_struct {
 114         struct user_fpsimd_state *st;
 115         void *sve_state;
 116         unsigned int sve_vl;
 117 };
 118 
 119 static DEFINE_PER_CPU(struct fpsimd_last_state_struct, fpsimd_last_state);
 120 
 121 /* Default VL for tasks that don't set it explicitly: */
 122 static int sve_default_vl = -1;
 123 
 124 #ifdef CONFIG_ARM64_SVE
 125 
 126 /* Maximum supported vector length across all CPUs (initially poisoned) */
 127 int __ro_after_init sve_max_vl = SVE_VL_MIN;
 128 int __ro_after_init sve_max_virtualisable_vl = SVE_VL_MIN;
 129 
 130 /*
 131  * Set of available vector lengths,
 132  * where length vq encoded as bit __vq_to_bit(vq):
 133  */
 134 __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
 135 /* Set of vector lengths present on at least one cpu: */
 136 static __ro_after_init DECLARE_BITMAP(sve_vq_partial_map, SVE_VQ_MAX);
 137 
 138 static void __percpu *efi_sve_state;
 139 
 140 #else /* ! CONFIG_ARM64_SVE */
 141 
 142 /* Dummy declaration for code that will be optimised out: */
 143 extern __ro_after_init DECLARE_BITMAP(sve_vq_map, SVE_VQ_MAX);
 144 extern __ro_after_init DECLARE_BITMAP(sve_vq_partial_map, SVE_VQ_MAX);
 145 extern void __percpu *efi_sve_state;
 146 
 147 #endif /* ! CONFIG_ARM64_SVE */
 148 
 149 DEFINE_PER_CPU(bool, fpsimd_context_busy);
 150 EXPORT_PER_CPU_SYMBOL(fpsimd_context_busy);
 151 
 152 static void __get_cpu_fpsimd_context(void)
 153 {
 154         bool busy = __this_cpu_xchg(fpsimd_context_busy, true);
 155 
 156         WARN_ON(busy);
 157 }
 158 
 159 /*
 160  * Claim ownership of the CPU FPSIMD context for use by the calling context.
 161  *
 162  * The caller may freely manipulate the FPSIMD context metadata until
 163  * put_cpu_fpsimd_context() is called.
 164  *
 165  * The double-underscore version must only be called if you know the task
 166  * can't be preempted.
 167  */
 168 static void get_cpu_fpsimd_context(void)
 169 {
 170         preempt_disable();
 171         __get_cpu_fpsimd_context();
 172 }
 173 
 174 static void __put_cpu_fpsimd_context(void)
 175 {
 176         bool busy = __this_cpu_xchg(fpsimd_context_busy, false);
 177 
 178         WARN_ON(!busy); /* No matching get_cpu_fpsimd_context()? */
 179 }
 180 
 181 /*
 182  * Release the CPU FPSIMD context.
 183  *
 184  * Must be called from a context in which get_cpu_fpsimd_context() was
 185  * previously called, with no call to put_cpu_fpsimd_context() in the
 186  * meantime.
 187  */
 188 static void put_cpu_fpsimd_context(void)
 189 {
 190         __put_cpu_fpsimd_context();
 191         preempt_enable();
 192 }
 193 
 194 static bool have_cpu_fpsimd_context(void)
 195 {
 196         return !preemptible() && __this_cpu_read(fpsimd_context_busy);
 197 }
 198 
 199 /*
 200  * Call __sve_free() directly only if you know task can't be scheduled
 201  * or preempted.
 202  */
 203 static void __sve_free(struct task_struct *task)
 204 {
 205         kfree(task->thread.sve_state);
 206         task->thread.sve_state = NULL;
 207 }
 208 
 209 static void sve_free(struct task_struct *task)
 210 {
 211         WARN_ON(test_tsk_thread_flag(task, TIF_SVE));
 212 
 213         __sve_free(task);
 214 }
 215 
 216 /*
 217  * TIF_SVE controls whether a task can use SVE without trapping while
 218  * in userspace, and also the way a task's FPSIMD/SVE state is stored
 219  * in thread_struct.
 220  *
 221  * The kernel uses this flag to track whether a user task is actively
 222  * using SVE, and therefore whether full SVE register state needs to
 223  * be tracked.  If not, the cheaper FPSIMD context handling code can
 224  * be used instead of the more costly SVE equivalents.
 225  *
 226  *  * TIF_SVE set:
 227  *
 228  *    The task can execute SVE instructions while in userspace without
 229  *    trapping to the kernel.
 230  *
 231  *    When stored, Z0-Z31 (incorporating Vn in bits[127:0] or the
 232  *    corresponding Zn), P0-P15 and FFR are encoded in in
 233  *    task->thread.sve_state, formatted appropriately for vector
 234  *    length task->thread.sve_vl.
 235  *
 236  *    task->thread.sve_state must point to a valid buffer at least
 237  *    sve_state_size(task) bytes in size.
 238  *
 239  *    During any syscall, the kernel may optionally clear TIF_SVE and
 240  *    discard the vector state except for the FPSIMD subset.
 241  *
 242  *  * TIF_SVE clear:
 243  *
 244  *    An attempt by the user task to execute an SVE instruction causes
 245  *    do_sve_acc() to be called, which does some preparation and then
 246  *    sets TIF_SVE.
 247  *
 248  *    When stored, FPSIMD registers V0-V31 are encoded in
 249  *    task->thread.uw.fpsimd_state; bits [max : 128] for each of Z0-Z31 are
 250  *    logically zero but not stored anywhere; P0-P15 and FFR are not
 251  *    stored and have unspecified values from userspace's point of
 252  *    view.  For hygiene purposes, the kernel zeroes them on next use,
 253  *    but userspace is discouraged from relying on this.
 254  *
 255  *    task->thread.sve_state does not need to be non-NULL, valid or any
 256  *    particular size: it must not be dereferenced.
 257  *
 258  *  * FPSR and FPCR are always stored in task->thread.uw.fpsimd_state
 259  *    irrespective of whether TIF_SVE is clear or set, since these are
 260  *    not vector length dependent.
 261  */
 262 
 263 /*
 264  * Update current's FPSIMD/SVE registers from thread_struct.
 265  *
 266  * This function should be called only when the FPSIMD/SVE state in
 267  * thread_struct is known to be up to date, when preparing to enter
 268  * userspace.
 269  */
 270 static void task_fpsimd_load(void)
 271 {
 272         WARN_ON(!system_supports_fpsimd());
 273         WARN_ON(!have_cpu_fpsimd_context());
 274 
 275         if (system_supports_sve() && test_thread_flag(TIF_SVE))
 276                 sve_load_state(sve_pffr(&current->thread),
 277                                &current->thread.uw.fpsimd_state.fpsr,
 278                                sve_vq_from_vl(current->thread.sve_vl) - 1);
 279         else
 280                 fpsimd_load_state(&current->thread.uw.fpsimd_state);
 281 }
 282 
 283 /*
 284  * Ensure FPSIMD/SVE storage in memory for the loaded context is up to
 285  * date with respect to the CPU registers.
 286  */
 287 static void fpsimd_save(void)
 288 {
 289         struct fpsimd_last_state_struct const *last =
 290                 this_cpu_ptr(&fpsimd_last_state);
 291         /* set by fpsimd_bind_task_to_cpu() or fpsimd_bind_state_to_cpu() */
 292 
 293         WARN_ON(!system_supports_fpsimd());
 294         WARN_ON(!have_cpu_fpsimd_context());
 295 
 296         if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
 297                 if (system_supports_sve() && test_thread_flag(TIF_SVE)) {
 298                         if (WARN_ON(sve_get_vl() != last->sve_vl)) {
 299                                 /*
 300                                  * Can't save the user regs, so current would
 301                                  * re-enter user with corrupt state.
 302                                  * There's no way to recover, so kill it:
 303                                  */
 304                                 force_signal_inject(SIGKILL, SI_KERNEL, 0);
 305                                 return;
 306                         }
 307 
 308                         sve_save_state((char *)last->sve_state +
 309                                                 sve_ffr_offset(last->sve_vl),
 310                                        &last->st->fpsr);
 311                 } else
 312                         fpsimd_save_state(last->st);
 313         }
 314 }
 315 
 316 /*
 317  * All vector length selection from userspace comes through here.
 318  * We're on a slow path, so some sanity-checks are included.
 319  * If things go wrong there's a bug somewhere, but try to fall back to a
 320  * safe choice.
 321  */
 322 static unsigned int find_supported_vector_length(unsigned int vl)
 323 {
 324         int bit;
 325         int max_vl = sve_max_vl;
 326 
 327         if (WARN_ON(!sve_vl_valid(vl)))
 328                 vl = SVE_VL_MIN;
 329 
 330         if (WARN_ON(!sve_vl_valid(max_vl)))
 331                 max_vl = SVE_VL_MIN;
 332 
 333         if (vl > max_vl)
 334                 vl = max_vl;
 335 
 336         bit = find_next_bit(sve_vq_map, SVE_VQ_MAX,
 337                             __vq_to_bit(sve_vq_from_vl(vl)));
 338         return sve_vl_from_vq(__bit_to_vq(bit));
 339 }
 340 
 341 #ifdef CONFIG_SYSCTL
 342 
 343 static int sve_proc_do_default_vl(struct ctl_table *table, int write,
 344                                   void __user *buffer, size_t *lenp,
 345                                   loff_t *ppos)
 346 {
 347         int ret;
 348         int vl = sve_default_vl;
 349         struct ctl_table tmp_table = {
 350                 .data = &vl,
 351                 .maxlen = sizeof(vl),
 352         };
 353 
 354         ret = proc_dointvec(&tmp_table, write, buffer, lenp, ppos);
 355         if (ret || !write)
 356                 return ret;
 357 
 358         /* Writing -1 has the special meaning "set to max": */
 359         if (vl == -1)
 360                 vl = sve_max_vl;
 361 
 362         if (!sve_vl_valid(vl))
 363                 return -EINVAL;
 364 
 365         sve_default_vl = find_supported_vector_length(vl);
 366         return 0;
 367 }
 368 
 369 static struct ctl_table sve_default_vl_table[] = {
 370         {
 371                 .procname       = "sve_default_vector_length",
 372                 .mode           = 0644,
 373                 .proc_handler   = sve_proc_do_default_vl,
 374         },
 375         { }
 376 };
 377 
 378 static int __init sve_sysctl_init(void)
 379 {
 380         if (system_supports_sve())
 381                 if (!register_sysctl("abi", sve_default_vl_table))
 382                         return -EINVAL;
 383 
 384         return 0;
 385 }
 386 
 387 #else /* ! CONFIG_SYSCTL */
 388 static int __init sve_sysctl_init(void) { return 0; }
 389 #endif /* ! CONFIG_SYSCTL */
 390 
 391 #define ZREG(sve_state, vq, n) ((char *)(sve_state) +           \
 392         (SVE_SIG_ZREG_OFFSET(vq, n) - SVE_SIG_REGS_OFFSET))
 393 
 394 #ifdef CONFIG_CPU_BIG_ENDIAN
 395 static __uint128_t arm64_cpu_to_le128(__uint128_t x)
 396 {
 397         u64 a = swab64(x);
 398         u64 b = swab64(x >> 64);
 399 
 400         return ((__uint128_t)a << 64) | b;
 401 }
 402 #else
 403 static __uint128_t arm64_cpu_to_le128(__uint128_t x)
 404 {
 405         return x;
 406 }
 407 #endif
 408 
 409 #define arm64_le128_to_cpu(x) arm64_cpu_to_le128(x)
 410 
 411 static void __fpsimd_to_sve(void *sst, struct user_fpsimd_state const *fst,
 412                             unsigned int vq)
 413 {
 414         unsigned int i;
 415         __uint128_t *p;
 416 
 417         for (i = 0; i < SVE_NUM_ZREGS; ++i) {
 418                 p = (__uint128_t *)ZREG(sst, vq, i);
 419                 *p = arm64_cpu_to_le128(fst->vregs[i]);
 420         }
 421 }
 422 
 423 /*
 424  * Transfer the FPSIMD state in task->thread.uw.fpsimd_state to
 425  * task->thread.sve_state.
 426  *
 427  * Task can be a non-runnable task, or current.  In the latter case,
 428  * the caller must have ownership of the cpu FPSIMD context before calling
 429  * this function.
 430  * task->thread.sve_state must point to at least sve_state_size(task)
 431  * bytes of allocated kernel memory.
 432  * task->thread.uw.fpsimd_state must be up to date before calling this
 433  * function.
 434  */
 435 static void fpsimd_to_sve(struct task_struct *task)
 436 {
 437         unsigned int vq;
 438         void *sst = task->thread.sve_state;
 439         struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;
 440 
 441         if (!system_supports_sve())
 442                 return;
 443 
 444         vq = sve_vq_from_vl(task->thread.sve_vl);
 445         __fpsimd_to_sve(sst, fst, vq);
 446 }
 447 
 448 /*
 449  * Transfer the SVE state in task->thread.sve_state to
 450  * task->thread.uw.fpsimd_state.
 451  *
 452  * Task can be a non-runnable task, or current.  In the latter case,
 453  * the caller must have ownership of the cpu FPSIMD context before calling
 454  * this function.
 455  * task->thread.sve_state must point to at least sve_state_size(task)
 456  * bytes of allocated kernel memory.
 457  * task->thread.sve_state must be up to date before calling this function.
 458  */
 459 static void sve_to_fpsimd(struct task_struct *task)
 460 {
 461         unsigned int vq;
 462         void const *sst = task->thread.sve_state;
 463         struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state;
 464         unsigned int i;
 465         __uint128_t const *p;
 466 
 467         if (!system_supports_sve())
 468                 return;
 469 
 470         vq = sve_vq_from_vl(task->thread.sve_vl);
 471         for (i = 0; i < SVE_NUM_ZREGS; ++i) {
 472                 p = (__uint128_t const *)ZREG(sst, vq, i);
 473                 fst->vregs[i] = arm64_le128_to_cpu(*p);
 474         }
 475 }
 476 
 477 #ifdef CONFIG_ARM64_SVE
 478 
 479 /*
 480  * Return how many bytes of memory are required to store the full SVE
 481  * state for task, given task's currently configured vector length.
 482  */
 483 size_t sve_state_size(struct task_struct const *task)
 484 {
 485         return SVE_SIG_REGS_SIZE(sve_vq_from_vl(task->thread.sve_vl));
 486 }
 487 
 488 /*
 489  * Ensure that task->thread.sve_state is allocated and sufficiently large.
 490  *
 491  * This function should be used only in preparation for replacing
 492  * task->thread.sve_state with new data.  The memory is always zeroed
 493  * here to prevent stale data from showing through: this is done in
 494  * the interest of testability and predictability: except in the
 495  * do_sve_acc() case, there is no ABI requirement to hide stale data
 496  * written previously be task.
 497  */
 498 void sve_alloc(struct task_struct *task)
 499 {
 500         if (task->thread.sve_state) {
 501                 memset(task->thread.sve_state, 0, sve_state_size(current));
 502                 return;
 503         }
 504 
 505         /* This is a small allocation (maximum ~8KB) and Should Not Fail. */
 506         task->thread.sve_state =
 507                 kzalloc(sve_state_size(task), GFP_KERNEL);
 508 
 509         /*
 510          * If future SVE revisions can have larger vectors though,
 511          * this may cease to be true:
 512          */
 513         BUG_ON(!task->thread.sve_state);
 514 }
 515 
 516 
 517 /*
 518  * Ensure that task->thread.sve_state is up to date with respect to
 519  * the user task, irrespective of when SVE is in use or not.
 520  *
 521  * This should only be called by ptrace.  task must be non-runnable.
 522  * task->thread.sve_state must point to at least sve_state_size(task)
 523  * bytes of allocated kernel memory.
 524  */
 525 void fpsimd_sync_to_sve(struct task_struct *task)
 526 {
 527         if (!test_tsk_thread_flag(task, TIF_SVE))
 528                 fpsimd_to_sve(task);
 529 }
 530 
 531 /*
 532  * Ensure that task->thread.uw.fpsimd_state is up to date with respect to
 533  * the user task, irrespective of whether SVE is in use or not.
 534  *
 535  * This should only be called by ptrace.  task must be non-runnable.
 536  * task->thread.sve_state must point to at least sve_state_size(task)
 537  * bytes of allocated kernel memory.
 538  */
 539 void sve_sync_to_fpsimd(struct task_struct *task)
 540 {
 541         if (test_tsk_thread_flag(task, TIF_SVE))
 542                 sve_to_fpsimd(task);
 543 }
 544 
 545 /*
 546  * Ensure that task->thread.sve_state is up to date with respect to
 547  * the task->thread.uw.fpsimd_state.
 548  *
 549  * This should only be called by ptrace to merge new FPSIMD register
 550  * values into a task for which SVE is currently active.
 551  * task must be non-runnable.
 552  * task->thread.sve_state must point to at least sve_state_size(task)
 553  * bytes of allocated kernel memory.
 554  * task->thread.uw.fpsimd_state must already have been initialised with
 555  * the new FPSIMD register values to be merged in.
 556  */
 557 void sve_sync_from_fpsimd_zeropad(struct task_struct *task)
 558 {
 559         unsigned int vq;
 560         void *sst = task->thread.sve_state;
 561         struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state;
 562 
 563         if (!test_tsk_thread_flag(task, TIF_SVE))
 564                 return;
 565 
 566         vq = sve_vq_from_vl(task->thread.sve_vl);
 567 
 568         memset(sst, 0, SVE_SIG_REGS_SIZE(vq));
 569         __fpsimd_to_sve(sst, fst, vq);
 570 }
 571 
 572 int sve_set_vector_length(struct task_struct *task,
 573                           unsigned long vl, unsigned long flags)
 574 {
 575         if (flags & ~(unsigned long)(PR_SVE_VL_INHERIT |
 576                                      PR_SVE_SET_VL_ONEXEC))
 577                 return -EINVAL;
 578 
 579         if (!sve_vl_valid(vl))
 580                 return -EINVAL;
 581 
 582         /*
 583          * Clamp to the maximum vector length that VL-agnostic SVE code can
 584          * work with.  A flag may be assigned in the future to allow setting
 585          * of larger vector lengths without confusing older software.
 586          */
 587         if (vl > SVE_VL_ARCH_MAX)
 588                 vl = SVE_VL_ARCH_MAX;
 589 
 590         vl = find_supported_vector_length(vl);
 591 
 592         if (flags & (PR_SVE_VL_INHERIT |
 593                      PR_SVE_SET_VL_ONEXEC))
 594                 task->thread.sve_vl_onexec = vl;
 595         else
 596                 /* Reset VL to system default on next exec: */
 597                 task->thread.sve_vl_onexec = 0;
 598 
 599         /* Only actually set the VL if not deferred: */
 600         if (flags & PR_SVE_SET_VL_ONEXEC)
 601                 goto out;
 602 
 603         if (vl == task->thread.sve_vl)
 604                 goto out;
 605 
 606         /*
 607          * To ensure the FPSIMD bits of the SVE vector registers are preserved,
 608          * write any live register state back to task_struct, and convert to a
 609          * non-SVE thread.
 610          */
 611         if (task == current) {
 612                 get_cpu_fpsimd_context();
 613 
 614                 fpsimd_save();
 615         }
 616 
 617         fpsimd_flush_task_state(task);
 618         if (test_and_clear_tsk_thread_flag(task, TIF_SVE))
 619                 sve_to_fpsimd(task);
 620 
 621         if (task == current)
 622                 put_cpu_fpsimd_context();
 623 
 624         /*
 625          * Force reallocation of task SVE state to the correct size
 626          * on next use:
 627          */
 628         sve_free(task);
 629 
 630         task->thread.sve_vl = vl;
 631 
 632 out:
 633         update_tsk_thread_flag(task, TIF_SVE_VL_INHERIT,
 634                                flags & PR_SVE_VL_INHERIT);
 635 
 636         return 0;
 637 }
 638 
 639 /*
 640  * Encode the current vector length and flags for return.
 641  * This is only required for prctl(): ptrace has separate fields
 642  *
 643  * flags are as for sve_set_vector_length().
 644  */
 645 static int sve_prctl_status(unsigned long flags)
 646 {
 647         int ret;
 648 
 649         if (flags & PR_SVE_SET_VL_ONEXEC)
 650                 ret = current->thread.sve_vl_onexec;
 651         else
 652                 ret = current->thread.sve_vl;
 653 
 654         if (test_thread_flag(TIF_SVE_VL_INHERIT))
 655                 ret |= PR_SVE_VL_INHERIT;
 656 
 657         return ret;
 658 }
 659 
 660 /* PR_SVE_SET_VL */
 661 int sve_set_current_vl(unsigned long arg)
 662 {
 663         unsigned long vl, flags;
 664         int ret;
 665 
 666         vl = arg & PR_SVE_VL_LEN_MASK;
 667         flags = arg & ~vl;
 668 
 669         if (!system_supports_sve())
 670                 return -EINVAL;
 671 
 672         ret = sve_set_vector_length(current, vl, flags);
 673         if (ret)
 674                 return ret;
 675 
 676         return sve_prctl_status(flags);
 677 }
 678 
 679 /* PR_SVE_GET_VL */
 680 int sve_get_current_vl(void)
 681 {
 682         if (!system_supports_sve())
 683                 return -EINVAL;
 684 
 685         return sve_prctl_status(0);
 686 }
 687 
 688 static void sve_probe_vqs(DECLARE_BITMAP(map, SVE_VQ_MAX))
 689 {
 690         unsigned int vq, vl;
 691         unsigned long zcr;
 692 
 693         bitmap_zero(map, SVE_VQ_MAX);
 694 
 695         zcr = ZCR_ELx_LEN_MASK;
 696         zcr = read_sysreg_s(SYS_ZCR_EL1) & ~zcr;
 697 
 698         for (vq = SVE_VQ_MAX; vq >= SVE_VQ_MIN; --vq) {
 699                 write_sysreg_s(zcr | (vq - 1), SYS_ZCR_EL1); /* self-syncing */
 700                 vl = sve_get_vl();
 701                 vq = sve_vq_from_vl(vl); /* skip intervening lengths */
 702                 set_bit(__vq_to_bit(vq), map);
 703         }
 704 }
 705 
 706 /*
 707  * Initialise the set of known supported VQs for the boot CPU.
 708  * This is called during kernel boot, before secondary CPUs are brought up.
 709  */
 710 void __init sve_init_vq_map(void)
 711 {
 712         sve_probe_vqs(sve_vq_map);
 713         bitmap_copy(sve_vq_partial_map, sve_vq_map, SVE_VQ_MAX);
 714 }
 715 
 716 /*
 717  * If we haven't committed to the set of supported VQs yet, filter out
 718  * those not supported by the current CPU.
 719  * This function is called during the bring-up of early secondary CPUs only.
 720  */
 721 void sve_update_vq_map(void)
 722 {
 723         DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
 724 
 725         sve_probe_vqs(tmp_map);
 726         bitmap_and(sve_vq_map, sve_vq_map, tmp_map, SVE_VQ_MAX);
 727         bitmap_or(sve_vq_partial_map, sve_vq_partial_map, tmp_map, SVE_VQ_MAX);
 728 }
 729 
 730 /*
 731  * Check whether the current CPU supports all VQs in the committed set.
 732  * This function is called during the bring-up of late secondary CPUs only.
 733  */
 734 int sve_verify_vq_map(void)
 735 {
 736         DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
 737         unsigned long b;
 738 
 739         sve_probe_vqs(tmp_map);
 740 
 741         bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX);
 742         if (bitmap_intersects(tmp_map, sve_vq_map, SVE_VQ_MAX)) {
 743                 pr_warn("SVE: cpu%d: Required vector length(s) missing\n",
 744                         smp_processor_id());
 745                 return -EINVAL;
 746         }
 747 
 748         if (!IS_ENABLED(CONFIG_KVM) || !is_hyp_mode_available())
 749                 return 0;
 750 
 751         /*
 752          * For KVM, it is necessary to ensure that this CPU doesn't
 753          * support any vector length that guests may have probed as
 754          * unsupported.
 755          */
 756 
 757         /* Recover the set of supported VQs: */
 758         bitmap_complement(tmp_map, tmp_map, SVE_VQ_MAX);
 759         /* Find VQs supported that are not globally supported: */
 760         bitmap_andnot(tmp_map, tmp_map, sve_vq_map, SVE_VQ_MAX);
 761 
 762         /* Find the lowest such VQ, if any: */
 763         b = find_last_bit(tmp_map, SVE_VQ_MAX);
 764         if (b >= SVE_VQ_MAX)
 765                 return 0; /* no mismatches */
 766 
 767         /*
 768          * Mismatches above sve_max_virtualisable_vl are fine, since
 769          * no guest is allowed to configure ZCR_EL2.LEN to exceed this:
 770          */
 771         if (sve_vl_from_vq(__bit_to_vq(b)) <= sve_max_virtualisable_vl) {
 772                 pr_warn("SVE: cpu%d: Unsupported vector length(s) present\n",
 773                         smp_processor_id());
 774                 return -EINVAL;
 775         }
 776 
 777         return 0;
 778 }
 779 
 780 static void __init sve_efi_setup(void)
 781 {
 782         if (!IS_ENABLED(CONFIG_EFI))
 783                 return;
 784 
 785         /*
 786          * alloc_percpu() warns and prints a backtrace if this goes wrong.
 787          * This is evidence of a crippled system and we are returning void,
 788          * so no attempt is made to handle this situation here.
 789          */
 790         if (!sve_vl_valid(sve_max_vl))
 791                 goto fail;
 792 
 793         efi_sve_state = __alloc_percpu(
 794                 SVE_SIG_REGS_SIZE(sve_vq_from_vl(sve_max_vl)), SVE_VQ_BYTES);
 795         if (!efi_sve_state)
 796                 goto fail;
 797 
 798         return;
 799 
 800 fail:
 801         panic("Cannot allocate percpu memory for EFI SVE save/restore");
 802 }
 803 
 804 /*
 805  * Enable SVE for EL1.
 806  * Intended for use by the cpufeatures code during CPU boot.
 807  */
 808 void sve_kernel_enable(const struct arm64_cpu_capabilities *__always_unused p)
 809 {
 810         write_sysreg(read_sysreg(CPACR_EL1) | CPACR_EL1_ZEN_EL1EN, CPACR_EL1);
 811         isb();
 812 }
 813 
 814 /*
 815  * Read the pseudo-ZCR used by cpufeatures to identify the supported SVE
 816  * vector length.
 817  *
 818  * Use only if SVE is present.
 819  * This function clobbers the SVE vector length.
 820  */
 821 u64 read_zcr_features(void)
 822 {
 823         u64 zcr;
 824         unsigned int vq_max;
 825 
 826         /*
 827          * Set the maximum possible VL, and write zeroes to all other
 828          * bits to see if they stick.
 829          */
 830         sve_kernel_enable(NULL);
 831         write_sysreg_s(ZCR_ELx_LEN_MASK, SYS_ZCR_EL1);
 832 
 833         zcr = read_sysreg_s(SYS_ZCR_EL1);
 834         zcr &= ~(u64)ZCR_ELx_LEN_MASK; /* find sticky 1s outside LEN field */
 835         vq_max = sve_vq_from_vl(sve_get_vl());
 836         zcr |= vq_max - 1; /* set LEN field to maximum effective value */
 837 
 838         return zcr;
 839 }
 840 
 841 void __init sve_setup(void)
 842 {
 843         u64 zcr;
 844         DECLARE_BITMAP(tmp_map, SVE_VQ_MAX);
 845         unsigned long b;
 846 
 847         if (!system_supports_sve())
 848                 return;
 849 
 850         /*
 851          * The SVE architecture mandates support for 128-bit vectors,
 852          * so sve_vq_map must have at least SVE_VQ_MIN set.
 853          * If something went wrong, at least try to patch it up:
 854          */
 855         if (WARN_ON(!test_bit(__vq_to_bit(SVE_VQ_MIN), sve_vq_map)))
 856                 set_bit(__vq_to_bit(SVE_VQ_MIN), sve_vq_map);
 857 
 858         zcr = read_sanitised_ftr_reg(SYS_ZCR_EL1);
 859         sve_max_vl = sve_vl_from_vq((zcr & ZCR_ELx_LEN_MASK) + 1);
 860 
 861         /*
 862          * Sanity-check that the max VL we determined through CPU features
 863          * corresponds properly to sve_vq_map.  If not, do our best:
 864          */
 865         if (WARN_ON(sve_max_vl != find_supported_vector_length(sve_max_vl)))
 866                 sve_max_vl = find_supported_vector_length(sve_max_vl);
 867 
 868         /*
 869          * For the default VL, pick the maximum supported value <= 64.
 870          * VL == 64 is guaranteed not to grow the signal frame.
 871          */
 872         sve_default_vl = find_supported_vector_length(64);
 873 
 874         bitmap_andnot(tmp_map, sve_vq_partial_map, sve_vq_map,
 875                       SVE_VQ_MAX);
 876 
 877         b = find_last_bit(tmp_map, SVE_VQ_MAX);
 878         if (b >= SVE_VQ_MAX)
 879                 /* No non-virtualisable VLs found */
 880                 sve_max_virtualisable_vl = SVE_VQ_MAX;
 881         else if (WARN_ON(b == SVE_VQ_MAX - 1))
 882                 /* No virtualisable VLs?  This is architecturally forbidden. */
 883                 sve_max_virtualisable_vl = SVE_VQ_MIN;
 884         else /* b + 1 < SVE_VQ_MAX */
 885                 sve_max_virtualisable_vl = sve_vl_from_vq(__bit_to_vq(b + 1));
 886 
 887         if (sve_max_virtualisable_vl > sve_max_vl)
 888                 sve_max_virtualisable_vl = sve_max_vl;
 889 
 890         pr_info("SVE: maximum available vector length %u bytes per vector\n",
 891                 sve_max_vl);
 892         pr_info("SVE: default vector length %u bytes per vector\n",
 893                 sve_default_vl);
 894 
 895         /* KVM decides whether to support mismatched systems. Just warn here: */
 896         if (sve_max_virtualisable_vl < sve_max_vl)
 897                 pr_warn("SVE: unvirtualisable vector lengths present\n");
 898 
 899         sve_efi_setup();
 900 }
 901 
 902 /*
 903  * Called from the put_task_struct() path, which cannot get here
 904  * unless dead_task is really dead and not schedulable.
 905  */
 906 void fpsimd_release_task(struct task_struct *dead_task)
 907 {
 908         __sve_free(dead_task);
 909 }
 910 
 911 #endif /* CONFIG_ARM64_SVE */
 912 
 913 /*
 914  * Trapped SVE access
 915  *
 916  * Storage is allocated for the full SVE state, the current FPSIMD
 917  * register contents are migrated across, and TIF_SVE is set so that
 918  * the SVE access trap will be disabled the next time this task
 919  * reaches ret_to_user.
 920  *
 921  * TIF_SVE should be clear on entry: otherwise, task_fpsimd_load()
 922  * would have disabled the SVE access trap for userspace during
 923  * ret_to_user, making an SVE access trap impossible in that case.
 924  */
 925 asmlinkage void do_sve_acc(unsigned int esr, struct pt_regs *regs)
 926 {
 927         /* Even if we chose not to use SVE, the hardware could still trap: */
 928         if (unlikely(!system_supports_sve()) || WARN_ON(is_compat_task())) {
 929                 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc);
 930                 return;
 931         }
 932 
 933         sve_alloc(current);
 934 
 935         get_cpu_fpsimd_context();
 936 
 937         fpsimd_save();
 938 
 939         /* Force ret_to_user to reload the registers: */
 940         fpsimd_flush_task_state(current);
 941 
 942         fpsimd_to_sve(current);
 943         if (test_and_set_thread_flag(TIF_SVE))
 944                 WARN_ON(1); /* SVE access shouldn't have trapped */
 945 
 946         put_cpu_fpsimd_context();
 947 }
 948 
 949 /*
 950  * Trapped FP/ASIMD access.
 951  */
 952 asmlinkage void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs)
 953 {
 954         /* TODO: implement lazy context saving/restoring */
 955         WARN_ON(1);
 956 }
 957 
 958 /*
 959  * Raise a SIGFPE for the current process.
 960  */
 961 asmlinkage void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs)
 962 {
 963         unsigned int si_code = FPE_FLTUNK;
 964 
 965         if (esr & ESR_ELx_FP_EXC_TFV) {
 966                 if (esr & FPEXC_IOF)
 967                         si_code = FPE_FLTINV;
 968                 else if (esr & FPEXC_DZF)
 969                         si_code = FPE_FLTDIV;
 970                 else if (esr & FPEXC_OFF)
 971                         si_code = FPE_FLTOVF;
 972                 else if (esr & FPEXC_UFF)
 973                         si_code = FPE_FLTUND;
 974                 else if (esr & FPEXC_IXF)
 975                         si_code = FPE_FLTRES;
 976         }
 977 
 978         send_sig_fault(SIGFPE, si_code,
 979                        (void __user *)instruction_pointer(regs),
 980                        current);
 981 }
 982 
 983 void fpsimd_thread_switch(struct task_struct *next)
 984 {
 985         bool wrong_task, wrong_cpu;
 986 
 987         if (!system_supports_fpsimd())
 988                 return;
 989 
 990         __get_cpu_fpsimd_context();
 991 
 992         /* Save unsaved fpsimd state, if any: */
 993         fpsimd_save();
 994 
 995         /*
 996          * Fix up TIF_FOREIGN_FPSTATE to correctly describe next's
 997          * state.  For kernel threads, FPSIMD registers are never loaded
 998          * and wrong_task and wrong_cpu will always be true.
 999          */
1000         wrong_task = __this_cpu_read(fpsimd_last_state.st) !=
1001                                         &next->thread.uw.fpsimd_state;
1002         wrong_cpu = next->thread.fpsimd_cpu != smp_processor_id();
1003 
1004         update_tsk_thread_flag(next, TIF_FOREIGN_FPSTATE,
1005                                wrong_task || wrong_cpu);
1006 
1007         __put_cpu_fpsimd_context();
1008 }
1009 
1010 void fpsimd_flush_thread(void)
1011 {
1012         int vl, supported_vl;
1013 
1014         if (!system_supports_fpsimd())
1015                 return;
1016 
1017         get_cpu_fpsimd_context();
1018 
1019         fpsimd_flush_task_state(current);
1020         memset(&current->thread.uw.fpsimd_state, 0,
1021                sizeof(current->thread.uw.fpsimd_state));
1022 
1023         if (system_supports_sve()) {
1024                 clear_thread_flag(TIF_SVE);
1025                 sve_free(current);
1026 
1027                 /*
1028                  * Reset the task vector length as required.
1029                  * This is where we ensure that all user tasks have a valid
1030                  * vector length configured: no kernel task can become a user
1031                  * task without an exec and hence a call to this function.
1032                  * By the time the first call to this function is made, all
1033                  * early hardware probing is complete, so sve_default_vl
1034                  * should be valid.
1035                  * If a bug causes this to go wrong, we make some noise and
1036                  * try to fudge thread.sve_vl to a safe value here.
1037                  */
1038                 vl = current->thread.sve_vl_onexec ?
1039                         current->thread.sve_vl_onexec : sve_default_vl;
1040 
1041                 if (WARN_ON(!sve_vl_valid(vl)))
1042                         vl = SVE_VL_MIN;
1043 
1044                 supported_vl = find_supported_vector_length(vl);
1045                 if (WARN_ON(supported_vl != vl))
1046                         vl = supported_vl;
1047 
1048                 current->thread.sve_vl = vl;
1049 
1050                 /*
1051                  * If the task is not set to inherit, ensure that the vector
1052                  * length will be reset by a subsequent exec:
1053                  */
1054                 if (!test_thread_flag(TIF_SVE_VL_INHERIT))
1055                         current->thread.sve_vl_onexec = 0;
1056         }
1057 
1058         put_cpu_fpsimd_context();
1059 }
1060 
1061 /*
1062  * Save the userland FPSIMD state of 'current' to memory, but only if the state
1063  * currently held in the registers does in fact belong to 'current'
1064  */
1065 void fpsimd_preserve_current_state(void)
1066 {
1067         if (!system_supports_fpsimd())
1068                 return;
1069 
1070         get_cpu_fpsimd_context();
1071         fpsimd_save();
1072         put_cpu_fpsimd_context();
1073 }
1074 
1075 /*
1076  * Like fpsimd_preserve_current_state(), but ensure that
1077  * current->thread.uw.fpsimd_state is updated so that it can be copied to
1078  * the signal frame.
1079  */
1080 void fpsimd_signal_preserve_current_state(void)
1081 {
1082         fpsimd_preserve_current_state();
1083         if (system_supports_sve() && test_thread_flag(TIF_SVE))
1084                 sve_to_fpsimd(current);
1085 }
1086 
1087 /*
1088  * Associate current's FPSIMD context with this cpu
1089  * The caller must have ownership of the cpu FPSIMD context before calling
1090  * this function.
1091  */
1092 void fpsimd_bind_task_to_cpu(void)
1093 {
1094         struct fpsimd_last_state_struct *last =
1095                 this_cpu_ptr(&fpsimd_last_state);
1096 
1097         WARN_ON(!system_supports_fpsimd());
1098         last->st = &current->thread.uw.fpsimd_state;
1099         last->sve_state = current->thread.sve_state;
1100         last->sve_vl = current->thread.sve_vl;
1101         current->thread.fpsimd_cpu = smp_processor_id();
1102 
1103         if (system_supports_sve()) {
1104                 /* Toggle SVE trapping for userspace if needed */
1105                 if (test_thread_flag(TIF_SVE))
1106                         sve_user_enable();
1107                 else
1108                         sve_user_disable();
1109 
1110                 /* Serialised by exception return to user */
1111         }
1112 }
1113 
1114 void fpsimd_bind_state_to_cpu(struct user_fpsimd_state *st, void *sve_state,
1115                               unsigned int sve_vl)
1116 {
1117         struct fpsimd_last_state_struct *last =
1118                 this_cpu_ptr(&fpsimd_last_state);
1119 
1120         WARN_ON(!system_supports_fpsimd());
1121         WARN_ON(!in_softirq() && !irqs_disabled());
1122 
1123         last->st = st;
1124         last->sve_state = sve_state;
1125         last->sve_vl = sve_vl;
1126 }
1127 
1128 /*
1129  * Load the userland FPSIMD state of 'current' from memory, but only if the
1130  * FPSIMD state already held in the registers is /not/ the most recent FPSIMD
1131  * state of 'current'
1132  */
1133 void fpsimd_restore_current_state(void)
1134 {
1135         /*
1136          * For the tasks that were created before we detected the absence of
1137          * FP/SIMD, the TIF_FOREIGN_FPSTATE could be set via fpsimd_thread_switch(),
1138          * e.g, init. This could be then inherited by the children processes.
1139          * If we later detect that the system doesn't support FP/SIMD,
1140          * we must clear the flag for  all the tasks to indicate that the
1141          * FPSTATE is clean (as we can't have one) to avoid looping for ever in
1142          * do_notify_resume().
1143          */
1144         if (!system_supports_fpsimd()) {
1145                 clear_thread_flag(TIF_FOREIGN_FPSTATE);
1146                 return;
1147         }
1148 
1149         get_cpu_fpsimd_context();
1150 
1151         if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) {
1152                 task_fpsimd_load();
1153                 fpsimd_bind_task_to_cpu();
1154         }
1155 
1156         put_cpu_fpsimd_context();
1157 }
1158 
1159 /*
1160  * Load an updated userland FPSIMD state for 'current' from memory and set the
1161  * flag that indicates that the FPSIMD register contents are the most recent
1162  * FPSIMD state of 'current'
1163  */
1164 void fpsimd_update_current_state(struct user_fpsimd_state const *state)
1165 {
1166         if (WARN_ON(!system_supports_fpsimd()))
1167                 return;
1168 
1169         get_cpu_fpsimd_context();
1170 
1171         current->thread.uw.fpsimd_state = *state;
1172         if (system_supports_sve() && test_thread_flag(TIF_SVE))
1173                 fpsimd_to_sve(current);
1174 
1175         task_fpsimd_load();
1176         fpsimd_bind_task_to_cpu();
1177 
1178         clear_thread_flag(TIF_FOREIGN_FPSTATE);
1179 
1180         put_cpu_fpsimd_context();
1181 }
1182 
1183 /*
1184  * Invalidate live CPU copies of task t's FPSIMD state
1185  *
1186  * This function may be called with preemption enabled.  The barrier()
1187  * ensures that the assignment to fpsimd_cpu is visible to any
1188  * preemption/softirq that could race with set_tsk_thread_flag(), so
1189  * that TIF_FOREIGN_FPSTATE cannot be spuriously re-cleared.
1190  *
1191  * The final barrier ensures that TIF_FOREIGN_FPSTATE is seen set by any
1192  * subsequent code.
1193  */
1194 void fpsimd_flush_task_state(struct task_struct *t)
1195 {
1196         t->thread.fpsimd_cpu = NR_CPUS;
1197         /*
1198          * If we don't support fpsimd, bail out after we have
1199          * reset the fpsimd_cpu for this task and clear the
1200          * FPSTATE.
1201          */
1202         if (!system_supports_fpsimd())
1203                 return;
1204         barrier();
1205         set_tsk_thread_flag(t, TIF_FOREIGN_FPSTATE);
1206 
1207         barrier();
1208 }
1209 
1210 /*
1211  * Invalidate any task's FPSIMD state that is present on this cpu.
1212  * The FPSIMD context should be acquired with get_cpu_fpsimd_context()
1213  * before calling this function.
1214  */
1215 static void fpsimd_flush_cpu_state(void)
1216 {
1217         WARN_ON(!system_supports_fpsimd());
1218         __this_cpu_write(fpsimd_last_state.st, NULL);
1219         set_thread_flag(TIF_FOREIGN_FPSTATE);
1220 }
1221 
1222 /*
1223  * Save the FPSIMD state to memory and invalidate cpu view.
1224  * This function must be called with preemption disabled.
1225  */
1226 void fpsimd_save_and_flush_cpu_state(void)
1227 {
1228         if (!system_supports_fpsimd())
1229                 return;
1230         WARN_ON(preemptible());
1231         __get_cpu_fpsimd_context();
1232         fpsimd_save();
1233         fpsimd_flush_cpu_state();
1234         __put_cpu_fpsimd_context();
1235 }
1236 
1237 #ifdef CONFIG_KERNEL_MODE_NEON
1238 
1239 /*
1240  * Kernel-side NEON support functions
1241  */
1242 
1243 /*
1244  * kernel_neon_begin(): obtain the CPU FPSIMD registers for use by the calling
1245  * context
1246  *
1247  * Must not be called unless may_use_simd() returns true.
1248  * Task context in the FPSIMD registers is saved back to memory as necessary.
1249  *
1250  * A matching call to kernel_neon_end() must be made before returning from the
1251  * calling context.
1252  *
1253  * The caller may freely use the FPSIMD registers until kernel_neon_end() is
1254  * called.
1255  */
1256 void kernel_neon_begin(void)
1257 {
1258         if (WARN_ON(!system_supports_fpsimd()))
1259                 return;
1260 
1261         BUG_ON(!may_use_simd());
1262 
1263         get_cpu_fpsimd_context();
1264 
1265         /* Save unsaved fpsimd state, if any: */
1266         fpsimd_save();
1267 
1268         /* Invalidate any task state remaining in the fpsimd regs: */
1269         fpsimd_flush_cpu_state();
1270 }
1271 EXPORT_SYMBOL(kernel_neon_begin);
1272 
1273 /*
1274  * kernel_neon_end(): give the CPU FPSIMD registers back to the current task
1275  *
1276  * Must be called from a context in which kernel_neon_begin() was previously
1277  * called, with no call to kernel_neon_end() in the meantime.
1278  *
1279  * The caller must not use the FPSIMD registers after this function is called,
1280  * unless kernel_neon_begin() is called again in the meantime.
1281  */
1282 void kernel_neon_end(void)
1283 {
1284         if (!system_supports_fpsimd())
1285                 return;
1286 
1287         put_cpu_fpsimd_context();
1288 }
1289 EXPORT_SYMBOL(kernel_neon_end);
1290 
1291 #ifdef CONFIG_EFI
1292 
1293 static DEFINE_PER_CPU(struct user_fpsimd_state, efi_fpsimd_state);
1294 static DEFINE_PER_CPU(bool, efi_fpsimd_state_used);
1295 static DEFINE_PER_CPU(bool, efi_sve_state_used);
1296 
1297 /*
1298  * EFI runtime services support functions
1299  *
1300  * The ABI for EFI runtime services allows EFI to use FPSIMD during the call.
1301  * This means that for EFI (and only for EFI), we have to assume that FPSIMD
1302  * is always used rather than being an optional accelerator.
1303  *
1304  * These functions provide the necessary support for ensuring FPSIMD
1305  * save/restore in the contexts from which EFI is used.
1306  *
1307  * Do not use them for any other purpose -- if tempted to do so, you are
1308  * either doing something wrong or you need to propose some refactoring.
1309  */
1310 
1311 /*
1312  * __efi_fpsimd_begin(): prepare FPSIMD for making an EFI runtime services call
1313  */
1314 void __efi_fpsimd_begin(void)
1315 {
1316         if (!system_supports_fpsimd())
1317                 return;
1318 
1319         WARN_ON(preemptible());
1320 
1321         if (may_use_simd()) {
1322                 kernel_neon_begin();
1323         } else {
1324                 /*
1325                  * If !efi_sve_state, SVE can't be in use yet and doesn't need
1326                  * preserving:
1327                  */
1328                 if (system_supports_sve() && likely(efi_sve_state)) {
1329                         char *sve_state = this_cpu_ptr(efi_sve_state);
1330 
1331                         __this_cpu_write(efi_sve_state_used, true);
1332 
1333                         sve_save_state(sve_state + sve_ffr_offset(sve_max_vl),
1334                                        &this_cpu_ptr(&efi_fpsimd_state)->fpsr);
1335                 } else {
1336                         fpsimd_save_state(this_cpu_ptr(&efi_fpsimd_state));
1337                 }
1338 
1339                 __this_cpu_write(efi_fpsimd_state_used, true);
1340         }
1341 }
1342 
1343 /*
1344  * __efi_fpsimd_end(): clean up FPSIMD after an EFI runtime services call
1345  */
1346 void __efi_fpsimd_end(void)
1347 {
1348         if (!system_supports_fpsimd())
1349                 return;
1350 
1351         if (!__this_cpu_xchg(efi_fpsimd_state_used, false)) {
1352                 kernel_neon_end();
1353         } else {
1354                 if (system_supports_sve() &&
1355                     likely(__this_cpu_read(efi_sve_state_used))) {
1356                         char const *sve_state = this_cpu_ptr(efi_sve_state);
1357 
1358                         sve_load_state(sve_state + sve_ffr_offset(sve_max_vl),
1359                                        &this_cpu_ptr(&efi_fpsimd_state)->fpsr,
1360                                        sve_vq_from_vl(sve_get_vl()) - 1);
1361 
1362                         __this_cpu_write(efi_sve_state_used, false);
1363                 } else {
1364                         fpsimd_load_state(this_cpu_ptr(&efi_fpsimd_state));
1365                 }
1366         }
1367 }
1368 
1369 #endif /* CONFIG_EFI */
1370 
1371 #endif /* CONFIG_KERNEL_MODE_NEON */
1372 
1373 #ifdef CONFIG_CPU_PM
1374 static int fpsimd_cpu_pm_notifier(struct notifier_block *self,
1375                                   unsigned long cmd, void *v)
1376 {
1377         switch (cmd) {
1378         case CPU_PM_ENTER:
1379                 fpsimd_save_and_flush_cpu_state();
1380                 break;
1381         case CPU_PM_EXIT:
1382                 break;
1383         case CPU_PM_ENTER_FAILED:
1384         default:
1385                 return NOTIFY_DONE;
1386         }
1387         return NOTIFY_OK;
1388 }
1389 
1390 static struct notifier_block fpsimd_cpu_pm_notifier_block = {
1391         .notifier_call = fpsimd_cpu_pm_notifier,
1392 };
1393 
1394 static void __init fpsimd_pm_init(void)
1395 {
1396         cpu_pm_register_notifier(&fpsimd_cpu_pm_notifier_block);
1397 }
1398 
1399 #else
1400 static inline void fpsimd_pm_init(void) { }
1401 #endif /* CONFIG_CPU_PM */
1402 
1403 #ifdef CONFIG_HOTPLUG_CPU
1404 static int fpsimd_cpu_dead(unsigned int cpu)
1405 {
1406         per_cpu(fpsimd_last_state.st, cpu) = NULL;
1407         return 0;
1408 }
1409 
1410 static inline void fpsimd_hotplug_init(void)
1411 {
1412         cpuhp_setup_state_nocalls(CPUHP_ARM64_FPSIMD_DEAD, "arm64/fpsimd:dead",
1413                                   NULL, fpsimd_cpu_dead);
1414 }
1415 
1416 #else
1417 static inline void fpsimd_hotplug_init(void) { }
1418 #endif
1419 
1420 /*
1421  * FP/SIMD support code initialisation.
1422  */
1423 static int __init fpsimd_init(void)
1424 {
1425         if (cpu_have_named_feature(FP)) {
1426                 fpsimd_pm_init();
1427                 fpsimd_hotplug_init();
1428         } else {
1429                 pr_notice("Floating-point is not implemented\n");
1430         }
1431 
1432         if (!cpu_have_named_feature(ASIMD))
1433                 pr_notice("Advanced SIMD is not implemented\n");
1434 
1435         return sve_sysctl_init();
1436 }
1437 core_initcall(fpsimd_init);
/* [<][>][^][v][top][bottom][index][help] */
root/arch/arm64/kernel/fpsimd.c

DEFINITIONS
