root/arch/ia64/lib/copy_page.S

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  *
   4  * Optimized version of the standard copy_page() function
   5  *
   6  * Inputs:
   7  *      in0:    address of target page
   8  *      in1:    address of source page
   9  * Output:
  10  *      no return value
  11  *
  12  * Copyright (C) 1999, 2001 Hewlett-Packard Co
  13  *      Stephane Eranian <eranian@hpl.hp.com>
  14  *      David Mosberger <davidm@hpl.hp.com>
  15  *
  16  * 4/06/01 davidm       Tuned to make it perform well both for cached and uncached copies.
  17  */
  18 #include <asm/asmmacro.h>
  19 #include <asm/page.h>
  20 #include <asm/export.h>
  21 
  22 #define PIPE_DEPTH      3
  23 #define EPI             p[PIPE_DEPTH-1]
  24 
  25 #define lcount          r16
  26 #define saved_pr        r17
  27 #define saved_lc        r18
  28 #define saved_pfs       r19
  29 #define src1            r20
  30 #define src2            r21
  31 #define tgt1            r22
  32 #define tgt2            r23
  33 #define srcf            r24
  34 #define tgtf            r25
  35 #define tgt_last        r26
  36 
  37 #define Nrot            ((8*PIPE_DEPTH+7)&~7)
  38 
  39 GLOBAL_ENTRY(copy_page)
  40         .prologue
  41         .save ar.pfs, saved_pfs
  42         alloc saved_pfs=ar.pfs,3,Nrot-3,0,Nrot
  43 
  44         .rotr t1[PIPE_DEPTH], t2[PIPE_DEPTH], t3[PIPE_DEPTH], t4[PIPE_DEPTH], \
  45               t5[PIPE_DEPTH], t6[PIPE_DEPTH], t7[PIPE_DEPTH], t8[PIPE_DEPTH]
  46         .rotp p[PIPE_DEPTH]
  47 
  48         .save ar.lc, saved_lc
  49         mov saved_lc=ar.lc
  50         mov ar.ec=PIPE_DEPTH
  51 
  52         mov lcount=PAGE_SIZE/64-1
  53         .save pr, saved_pr
  54         mov saved_pr=pr
  55         mov pr.rot=1<<16
  56 
  57         .body
  58 
  59         mov src1=in1
  60         adds src2=8,in1
  61         mov tgt_last = PAGE_SIZE
  62         ;;
  63         adds tgt2=8,in0
  64         add srcf=512,in1
  65         mov ar.lc=lcount
  66         mov tgt1=in0
  67         add tgtf=512,in0
  68         add tgt_last = tgt_last, in0
  69         ;;
  70 1:
  71 (p[0])  ld8 t1[0]=[src1],16
  72 (EPI)   st8 [tgt1]=t1[PIPE_DEPTH-1],16
  73 (p[0])  ld8 t2[0]=[src2],16
  74 (EPI)   st8 [tgt2]=t2[PIPE_DEPTH-1],16
  75         cmp.ltu p6,p0 = tgtf, tgt_last
  76         ;;
  77 (p[0])  ld8 t3[0]=[src1],16
  78 (EPI)   st8 [tgt1]=t3[PIPE_DEPTH-1],16
  79 (p[0])  ld8 t4[0]=[src2],16
  80 (EPI)   st8 [tgt2]=t4[PIPE_DEPTH-1],16
  81         ;;
  82 (p[0])  ld8 t5[0]=[src1],16
  83 (EPI)   st8 [tgt1]=t5[PIPE_DEPTH-1],16
  84 (p[0])  ld8 t6[0]=[src2],16
  85 (EPI)   st8 [tgt2]=t6[PIPE_DEPTH-1],16
  86         ;;
  87 (p[0])  ld8 t7[0]=[src1],16
  88 (EPI)   st8 [tgt1]=t7[PIPE_DEPTH-1],16
  89 (p[0])  ld8 t8[0]=[src2],16
  90 (EPI)   st8 [tgt2]=t8[PIPE_DEPTH-1],16
  91 
  92 (p6)    lfetch [srcf], 64
  93 (p6)    lfetch [tgtf], 64
  94         br.ctop.sptk.few 1b
  95         ;;
  96         mov pr=saved_pr,0xffffffffffff0000      // restore predicates
  97         mov ar.pfs=saved_pfs
  98         mov ar.lc=saved_lc
  99         br.ret.sptk.many rp
 100 END(copy_page)
 101 EXPORT_SYMBOL(copy_page)