1########################################################################
2# Implement fast SHA-256 with AVX1 instructions. (x86_64)
3#
4# Copyright (C) 2013 Intel Corporation.
5#
6# Authors:
7#     James Guilford <james.guilford@intel.com>
8#     Kirk Yap <kirk.s.yap@intel.com>
9#     Tim Chen <tim.c.chen@linux.intel.com>
10#
11# This software is available to you under a choice of one of two
12# licenses.  You may choose to be licensed under the terms of the GNU
13# General Public License (GPL) Version 2, available from the file
14# COPYING in the main directory of this source tree, or the
15# OpenIB.org BSD license below:
16#
17#     Redistribution and use in source and binary forms, with or
18#     without modification, are permitted provided that the following
19#     conditions are met:
20#
21#      - Redistributions of source code must retain the above
22#        copyright notice, this list of conditions and the following
23#        disclaimer.
24#
25#      - Redistributions in binary form must reproduce the above
26#        copyright notice, this list of conditions and the following
27#        disclaimer in the documentation and/or other materials
28#        provided with the distribution.
29#
30# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
31# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
32# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
33# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
34# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
35# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
36# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
37# SOFTWARE.
38########################################################################
39#
40# This code is described in an Intel White-Paper:
41# "Fast SHA-256 Implementations on Intel Architecture Processors"
42#
43# To find it, surf to http://www.intel.com/p/en_US/embedded
44# and search for that title.
45#
46########################################################################
47# This code schedules 1 block at a time, with 4 lanes per block
48########################################################################
49
50#ifdef CONFIG_AS_AVX
51#include <linux/linkage.h>
52
53## assume buffers not aligned
54#define    VMOVDQ vmovdqu
55
56################################ Define Macros
57
58# addm [mem], reg
59# Add reg to mem using reg-mem add and store
60.macro addm p1 p2
61	add     \p1, \p2
62	mov     \p2, \p1
63.endm
64
65
66.macro MY_ROR p1 p2
67	shld    $(32-(\p1)), \p2, \p2
68.endm
69
70################################
71
72# COPY_XMM_AND_BSWAP xmm, [mem], byte_flip_mask
73# Load xmm with mem and byte swap each dword
74.macro COPY_XMM_AND_BSWAP p1 p2 p3
75	VMOVDQ \p2, \p1
76	vpshufb \p3, \p1, \p1
77.endm
78
79################################
80
81X0 = %xmm4
82X1 = %xmm5
83X2 = %xmm6
84X3 = %xmm7
85
86XTMP0 = %xmm0
87XTMP1 = %xmm1
88XTMP2 = %xmm2
89XTMP3 = %xmm3
90XTMP4 = %xmm8
91XFER = %xmm9
92XTMP5 = %xmm11
93
94SHUF_00BA = %xmm10      # shuffle xBxA -> 00BA
95SHUF_DC00 = %xmm12      # shuffle xDxC -> DC00
96BYTE_FLIP_MASK = %xmm13
97
98NUM_BLKS = %rdx   # 3rd arg
99INP = %rsi        # 2nd arg
100CTX = %rdi        # 1st arg
101
102SRND = %rsi       # clobbers INP
103c = %ecx
104d = %r8d
105e = %edx
106TBL = %rbp
107a = %eax
108b = %ebx
109
110f = %r9d
111g = %r10d
112h = %r11d
113
114y0 = %r13d
115y1 = %r14d
116y2 = %r15d
117
118
119_INP_END_SIZE = 8
120_INP_SIZE = 8
121_XFER_SIZE = 16
122_XMM_SAVE_SIZE = 0
123
124_INP_END = 0
125_INP            = _INP_END  + _INP_END_SIZE
126_XFER           = _INP      + _INP_SIZE
127_XMM_SAVE       = _XFER     + _XFER_SIZE
128STACK_SIZE      = _XMM_SAVE + _XMM_SAVE_SIZE
129
130# rotate_Xs
131# Rotate values of symbols X0...X3
132.macro rotate_Xs
133X_ = X0
134X0 = X1
135X1 = X2
136X2 = X3
137X3 = X_
138.endm
139
140# ROTATE_ARGS
141# Rotate values of symbols a...h
142.macro ROTATE_ARGS
143TMP_ = h
144h = g
145g = f
146f = e
147e = d
148d = c
149c = b
150b = a
151a = TMP_
152.endm
153
154.macro FOUR_ROUNDS_AND_SCHED
155	## compute s0 four at a time and s1 two at a time
156	## compute W[-16] + W[-7] 4 at a time
157
158	mov     e, y0			# y0 = e
159	MY_ROR  (25-11), y0             # y0 = e >> (25-11)
160	mov     a, y1                   # y1 = a
161	vpalignr $4, X2, X3, XTMP0      # XTMP0 = W[-7]
162	MY_ROR  (22-13), y1             # y1 = a >> (22-13)
163	xor     e, y0                   # y0 = e ^ (e >> (25-11))
164	mov     f, y2                   # y2 = f
165	MY_ROR  (11-6), y0              # y0 = (e >> (11-6)) ^ (e >> (25-6))
166	xor     a, y1                   # y1 = a ^ (a >> (22-13)
167	xor     g, y2                   # y2 = f^g
168	vpaddd  X0, XTMP0, XTMP0        # XTMP0 = W[-7] + W[-16]
169	xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
170	and     e, y2                   # y2 = (f^g)&e
171	MY_ROR  (13-2), y1              # y1 = (a >> (13-2)) ^ (a >> (22-2))
172	## compute s0
173	vpalignr $4, X0, X1, XTMP1      # XTMP1 = W[-15]
174	xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
175	MY_ROR  6, y0                   # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
176	xor     g, y2                   # y2 = CH = ((f^g)&e)^g
177	MY_ROR  2, y1                   # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
178	add     y0, y2                  # y2 = S1 + CH
179	add     _XFER(%rsp), y2         # y2 = k + w + S1 + CH
180	mov     a, y0                   # y0 = a
181	add     y2, h                   # h = h + S1 + CH + k + w
182	mov     a, y2                   # y2 = a
183	vpsrld  $7, XTMP1, XTMP2
184	or      c, y0                   # y0 = a|c
185	add     h, d                    # d = d + h + S1 + CH + k + w
186	and     c, y2                   # y2 = a&c
187	vpslld  $(32-7), XTMP1, XTMP3
188	and     b, y0                   # y0 = (a|c)&b
189	add     y1, h                   # h = h + S1 + CH + k + w + S0
190	vpor    XTMP2, XTMP3, XTMP3     # XTMP1 = W[-15] MY_ROR 7
191	or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
192	add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
193	ROTATE_ARGS
194	mov     e, y0                   # y0 = e
195	mov     a, y1                   # y1 = a
196	MY_ROR  (25-11), y0             # y0 = e >> (25-11)
197	xor     e, y0                   # y0 = e ^ (e >> (25-11))
198	mov     f, y2                   # y2 = f
199	MY_ROR  (22-13), y1             # y1 = a >> (22-13)
200	vpsrld  $18, XTMP1, XTMP2       #
201	xor     a, y1                   # y1 = a ^ (a >> (22-13)
202	MY_ROR  (11-6), y0              # y0 = (e >> (11-6)) ^ (e >> (25-6))
203	xor     g, y2                   # y2 = f^g
204	vpsrld  $3, XTMP1, XTMP4        # XTMP4 = W[-15] >> 3
205	MY_ROR  (13-2), y1              # y1 = (a >> (13-2)) ^ (a >> (22-2))
206	xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
207	and     e, y2                   # y2 = (f^g)&e
208	MY_ROR  6, y0                   # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
209	vpslld  $(32-18), XTMP1, XTMP1
210	xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
211	xor     g, y2                   # y2 = CH = ((f^g)&e)^g
212	vpxor   XTMP1, XTMP3, XTMP3     #
213	add     y0, y2                  # y2 = S1 + CH
214	add     (1*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH
215	MY_ROR  2, y1                   # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
216	vpxor   XTMP2, XTMP3, XTMP3     # XTMP1 = W[-15] MY_ROR 7 ^ W[-15] MY_ROR
217	mov     a, y0                   # y0 = a
218	add     y2, h                   # h = h + S1 + CH + k + w
219	mov     a, y2                   # y2 = a
220	vpxor   XTMP4, XTMP3, XTMP1     # XTMP1 = s0
221	or      c, y0                   # y0 = a|c
222	add     h, d                    # d = d + h + S1 + CH + k + w
223	and     c, y2                   # y2 = a&c
224	## compute low s1
225	vpshufd $0b11111010, X3, XTMP2  # XTMP2 = W[-2] {BBAA}
226	and     b, y0                   # y0 = (a|c)&b
227	add     y1, h                   # h = h + S1 + CH + k + w + S0
228	vpaddd  XTMP1, XTMP0, XTMP0     # XTMP0 = W[-16] + W[-7] + s0
229	or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
230	add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
231	ROTATE_ARGS
232	mov     e, y0                   # y0 = e
233	mov     a, y1                   # y1 = a
234	MY_ROR  (25-11), y0             # y0 = e >> (25-11)
235	xor     e, y0                   # y0 = e ^ (e >> (25-11))
236	MY_ROR  (22-13), y1             # y1 = a >> (22-13)
237	mov     f, y2                   # y2 = f
238	xor     a, y1                   # y1 = a ^ (a >> (22-13)
239	MY_ROR  (11-6), y0              # y0 = (e >> (11-6)) ^ (e >> (25-6))
240	vpsrld  $10, XTMP2, XTMP4       # XTMP4 = W[-2] >> 10 {BBAA}
241	xor     g, y2                   # y2 = f^g
242	vpsrlq  $19, XTMP2, XTMP3       # XTMP3 = W[-2] MY_ROR 19 {xBxA}
243	xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
244	and     e, y2                   # y2 = (f^g)&e
245	vpsrlq  $17, XTMP2, XTMP2       # XTMP2 = W[-2] MY_ROR 17 {xBxA}
246	MY_ROR  (13-2), y1              # y1 = (a >> (13-2)) ^ (a >> (22-2))
247	xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
248	xor     g, y2                   # y2 = CH = ((f^g)&e)^g
249	MY_ROR  6, y0                   # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
250	vpxor   XTMP3, XTMP2, XTMP2     #
251	add     y0, y2                  # y2 = S1 + CH
252	MY_ROR  2, y1                   # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
253	add     (2*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH
254	vpxor   XTMP2, XTMP4, XTMP4     # XTMP4 = s1 {xBxA}
255	mov     a, y0                   # y0 = a
256	add     y2, h                   # h = h + S1 + CH + k + w
257	mov     a, y2                   # y2 = a
258	vpshufb SHUF_00BA, XTMP4, XTMP4 # XTMP4 = s1 {00BA}
259	or      c, y0                   # y0 = a|c
260	add     h, d                    # d = d + h + S1 + CH + k + w
261	and     c, y2                   # y2 = a&c
262	vpaddd  XTMP4, XTMP0, XTMP0     # XTMP0 = {..., ..., W[1], W[0]}
263	and     b, y0                   # y0 = (a|c)&b
264	add     y1, h                   # h = h + S1 + CH + k + w + S0
265	## compute high s1
266	vpshufd $0b01010000, XTMP0, XTMP2 # XTMP2 = W[-2] {DDCC}
267	or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
268	add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
269	ROTATE_ARGS
270	mov     e, y0                   # y0 = e
271	MY_ROR  (25-11), y0             # y0 = e >> (25-11)
272	mov     a, y1                   # y1 = a
273	MY_ROR  (22-13), y1             # y1 = a >> (22-13)
274	xor     e, y0                   # y0 = e ^ (e >> (25-11))
275	mov     f, y2                   # y2 = f
276	MY_ROR  (11-6), y0              # y0 = (e >> (11-6)) ^ (e >> (25-6))
277	vpsrld  $10, XTMP2, XTMP5       # XTMP5 = W[-2] >> 10 {DDCC}
278	xor     a, y1                   # y1 = a ^ (a >> (22-13)
279	xor     g, y2                   # y2 = f^g
280	vpsrlq  $19, XTMP2, XTMP3       # XTMP3 = W[-2] MY_ROR 19 {xDxC}
281	xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
282	and     e, y2                   # y2 = (f^g)&e
283	MY_ROR  (13-2), y1              # y1 = (a >> (13-2)) ^ (a >> (22-2))
284	vpsrlq  $17, XTMP2, XTMP2       # XTMP2 = W[-2] MY_ROR 17 {xDxC}
285	xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
286	MY_ROR  6, y0                   # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
287	xor     g, y2                   # y2 = CH = ((f^g)&e)^g
288	vpxor   XTMP3, XTMP2, XTMP2
289	MY_ROR  2, y1                   # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
290	add     y0, y2                  # y2 = S1 + CH
291	add     (3*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH
292	vpxor   XTMP2, XTMP5, XTMP5     # XTMP5 = s1 {xDxC}
293	mov     a, y0                   # y0 = a
294	add     y2, h                   # h = h + S1 + CH + k + w
295	mov     a, y2                   # y2 = a
296	vpshufb SHUF_DC00, XTMP5, XTMP5 # XTMP5 = s1 {DC00}
297	or      c, y0                   # y0 = a|c
298	add     h, d                    # d = d + h + S1 + CH + k + w
299	and     c, y2                   # y2 = a&c
300	vpaddd  XTMP0, XTMP5, X0        # X0 = {W[3], W[2], W[1], W[0]}
301	and     b, y0                   # y0 = (a|c)&b
302	add     y1, h                   # h = h + S1 + CH + k + w + S0
303	or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
304	add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
305	ROTATE_ARGS
306	rotate_Xs
307.endm
308
309## input is [rsp + _XFER + %1 * 4]
310.macro DO_ROUND round
311	mov	e, y0			# y0 = e
312        MY_ROR  (25-11), y0             # y0 = e >> (25-11)
313        mov     a, y1                   # y1 = a
314        xor     e, y0                   # y0 = e ^ (e >> (25-11))
315        MY_ROR  (22-13), y1             # y1 = a >> (22-13)
316        mov     f, y2                   # y2 = f
317        xor     a, y1                   # y1 = a ^ (a >> (22-13)
318        MY_ROR  (11-6), y0              # y0 = (e >> (11-6)) ^ (e >> (25-6))
319        xor     g, y2                   # y2 = f^g
320        xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
321        MY_ROR  (13-2), y1              # y1 = (a >> (13-2)) ^ (a >> (22-2))
322        and     e, y2                   # y2 = (f^g)&e
323        xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
324        MY_ROR  6, y0                   # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
325        xor     g, y2                   # y2 = CH = ((f^g)&e)^g
326        add     y0, y2                  # y2 = S1 + CH
327        MY_ROR  2, y1                   # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
328        offset = \round * 4 + _XFER     #
329        add     offset(%rsp), y2	# y2 = k + w + S1 + CH
330        mov     a, y0			# y0 = a
331        add     y2, h                   # h = h + S1 + CH + k + w
332        mov     a, y2                   # y2 = a
333        or      c, y0                   # y0 = a|c
334        add     h, d                    # d = d + h + S1 + CH + k + w
335        and     c, y2                   # y2 = a&c
336        and     b, y0                   # y0 = (a|c)&b
337        add     y1, h                   # h = h + S1 + CH + k + w + S0
338        or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
339        add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
340        ROTATE_ARGS
341.endm
342
343########################################################################
344## void sha256_transform_avx(void *input_data, UINT32 digest[8], UINT64 num_blks)
345## arg 1 : pointer to digest
346## arg 2 : pointer to input data
347## arg 3 : Num blocks
348########################################################################
349.text
350ENTRY(sha256_transform_avx)
351.align 32
352	pushq   %rbx
353	pushq   %rbp
354	pushq   %r13
355	pushq   %r14
356	pushq   %r15
357	pushq   %r12
358
359	mov	%rsp, %r12
360	subq    $STACK_SIZE, %rsp	# allocate stack space
361	and	$~15, %rsp		# align stack pointer
362
363	shl     $6, NUM_BLKS		# convert to bytes
364	jz      done_hash
365	add     INP, NUM_BLKS		# pointer to end of data
366	mov     NUM_BLKS, _INP_END(%rsp)
367
368	## load initial digest
369	mov     4*0(CTX), a
370	mov     4*1(CTX), b
371	mov     4*2(CTX), c
372	mov     4*3(CTX), d
373	mov     4*4(CTX), e
374	mov     4*5(CTX), f
375	mov     4*6(CTX), g
376	mov     4*7(CTX), h
377
378	vmovdqa  PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK
379	vmovdqa  _SHUF_00BA(%rip), SHUF_00BA
380	vmovdqa  _SHUF_DC00(%rip), SHUF_DC00
381loop0:
382	lea     K256(%rip), TBL
383
384	## byte swap first 16 dwords
385	COPY_XMM_AND_BSWAP      X0, 0*16(INP), BYTE_FLIP_MASK
386	COPY_XMM_AND_BSWAP      X1, 1*16(INP), BYTE_FLIP_MASK
387	COPY_XMM_AND_BSWAP      X2, 2*16(INP), BYTE_FLIP_MASK
388	COPY_XMM_AND_BSWAP      X3, 3*16(INP), BYTE_FLIP_MASK
389
390	mov     INP, _INP(%rsp)
391
392	## schedule 48 input dwords, by doing 3 rounds of 16 each
393	mov     $3, SRND
394.align 16
395loop1:
396	vpaddd  (TBL), X0, XFER
397	vmovdqa XFER, _XFER(%rsp)
398	FOUR_ROUNDS_AND_SCHED
399
400	vpaddd  1*16(TBL), X0, XFER
401	vmovdqa XFER, _XFER(%rsp)
402	FOUR_ROUNDS_AND_SCHED
403
404	vpaddd  2*16(TBL), X0, XFER
405	vmovdqa XFER, _XFER(%rsp)
406	FOUR_ROUNDS_AND_SCHED
407
408	vpaddd  3*16(TBL), X0, XFER
409	vmovdqa XFER, _XFER(%rsp)
410	add	$4*16, TBL
411	FOUR_ROUNDS_AND_SCHED
412
413	sub     $1, SRND
414	jne     loop1
415
416	mov     $2, SRND
417loop2:
418	vpaddd  (TBL), X0, XFER
419	vmovdqa XFER, _XFER(%rsp)
420	DO_ROUND        0
421	DO_ROUND        1
422	DO_ROUND        2
423	DO_ROUND        3
424
425	vpaddd  1*16(TBL), X1, XFER
426	vmovdqa XFER, _XFER(%rsp)
427	add     $2*16, TBL
428	DO_ROUND        0
429	DO_ROUND        1
430	DO_ROUND        2
431	DO_ROUND        3
432
433	vmovdqa X2, X0
434	vmovdqa X3, X1
435
436	sub     $1, SRND
437	jne     loop2
438
439	addm    (4*0)(CTX),a
440	addm    (4*1)(CTX),b
441	addm    (4*2)(CTX),c
442	addm    (4*3)(CTX),d
443	addm    (4*4)(CTX),e
444	addm    (4*5)(CTX),f
445	addm    (4*6)(CTX),g
446	addm    (4*7)(CTX),h
447
448	mov     _INP(%rsp), INP
449	add     $64, INP
450	cmp     _INP_END(%rsp), INP
451	jne     loop0
452
453done_hash:
454
455	mov	%r12, %rsp
456
457	popq	%r12
458	popq    %r15
459	popq    %r14
460	popq    %r13
461	popq    %rbp
462	popq    %rbx
463	ret
464ENDPROC(sha256_transform_avx)
465
466.data
467.align 64
468K256:
469	.long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
470	.long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
471	.long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
472	.long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
473	.long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
474	.long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
475	.long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
476	.long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
477	.long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
478	.long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
479	.long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
480	.long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070
481	.long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
482	.long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
483	.long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
484	.long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
485
486PSHUFFLE_BYTE_FLIP_MASK:
487	.octa 0x0c0d0e0f08090a0b0405060700010203
488
489# shuffle xBxA -> 00BA
490_SHUF_00BA:
491	.octa 0xFFFFFFFFFFFFFFFF0b0a090803020100
492
493# shuffle xDxC -> DC00
494_SHUF_DC00:
495	.octa 0x0b0a090803020100FFFFFFFFFFFFFFFF
496#endif
497