669 lines
14 KiB
ArmAsm
669 lines
14 KiB
ArmAsm
// Copyright 2019 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
// Based on CRYPTOGAMS code with the following comment:
|
|
// # ====================================================================
|
|
// # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
|
|
// # project. The module is, however, dual licensed under OpenSSL and
|
|
// # CRYPTOGAMS licenses depending on where you obtain it. For further
|
|
// # details see http://www.openssl.org/~appro/cryptogams/.
|
|
// # ====================================================================
|
|
|
|
// Original code can be found at the link below:
|
|
// https://github.com/dot-asm/cryptogams/commit/a60f5b50ed908e91e5c39ca79126a4a876d5d8ff
|
|
|
|
// There are some differences between CRYPTOGAMS code and this one. The round
|
|
// loop for "_int" isn't the same as the original. Some adjustments were
|
|
// necessary because there are less vector registers available. For example, some
|
|
// X variables (r12, r13, r14, and r15) share the same register used by the
|
|
// counter. The original code uses ctr to name the counter. Here we use CNT
|
|
// because golang uses CTR as the counter register name.
|
|
|
|
// +build ppc64le,!gccgo,!appengine
|
|
|
|
#include "textflag.h"
|
|
|
|
#define OUT R3
|
|
#define INP R4
|
|
#define LEN R5
|
|
#define KEY R6
|
|
#define CNT R7
|
|
|
|
#define TEMP R8
|
|
|
|
#define X0 R11
|
|
#define X1 R12
|
|
#define X2 R14
|
|
#define X3 R15
|
|
#define X4 R16
|
|
#define X5 R17
|
|
#define X6 R18
|
|
#define X7 R19
|
|
#define X8 R20
|
|
#define X9 R21
|
|
#define X10 R22
|
|
#define X11 R23
|
|
#define X12 R24
|
|
#define X13 R25
|
|
#define X14 R26
|
|
#define X15 R27
|
|
|
|
#define CON0 X0
|
|
#define CON1 X1
|
|
#define CON2 X2
|
|
#define CON3 X3
|
|
|
|
#define KEY0 X4
|
|
#define KEY1 X5
|
|
#define KEY2 X6
|
|
#define KEY3 X7
|
|
#define KEY4 X8
|
|
#define KEY5 X9
|
|
#define KEY6 X10
|
|
#define KEY7 X11
|
|
|
|
#define CNT0 X12
|
|
#define CNT1 X13
|
|
#define CNT2 X14
|
|
#define CNT3 X15
|
|
|
|
#define TMP0 R9
|
|
#define TMP1 R10
|
|
#define TMP2 R28
|
|
#define TMP3 R29
|
|
|
|
#define CONSTS R8
|
|
|
|
#define A0 V0
|
|
#define B0 V1
|
|
#define C0 V2
|
|
#define D0 V3
|
|
#define A1 V4
|
|
#define B1 V5
|
|
#define C1 V6
|
|
#define D1 V7
|
|
#define A2 V8
|
|
#define B2 V9
|
|
#define C2 V10
|
|
#define D2 V11
|
|
#define T0 V12
|
|
#define T1 V13
|
|
#define T2 V14
|
|
|
|
#define K0 V15
|
|
#define K1 V16
|
|
#define K2 V17
|
|
#define K3 V18
|
|
#define K4 V19
|
|
#define K5 V20
|
|
|
|
#define FOUR V21
|
|
#define SIXTEEN V22
|
|
#define TWENTY4 V23
|
|
#define TWENTY V24
|
|
#define TWELVE V25
|
|
#define TWENTY5 V26
|
|
#define SEVEN V27
|
|
|
|
#define INPPERM V28
|
|
#define OUTPERM V29
|
|
#define OUTMASK V30
|
|
|
|
#define DD0 V31
|
|
#define DD1 SEVEN
|
|
#define DD2 T0
|
|
#define DD3 T1
|
|
#define DD4 T2
|
|
|
|
DATA ·consts+0x00(SB)/8, $0x3320646e61707865
|
|
DATA ·consts+0x08(SB)/8, $0x6b20657479622d32
|
|
DATA ·consts+0x10(SB)/8, $0x0000000000000001
|
|
DATA ·consts+0x18(SB)/8, $0x0000000000000000
|
|
DATA ·consts+0x20(SB)/8, $0x0000000000000004
|
|
DATA ·consts+0x28(SB)/8, $0x0000000000000000
|
|
DATA ·consts+0x30(SB)/8, $0x0a0b08090e0f0c0d
|
|
DATA ·consts+0x38(SB)/8, $0x0203000106070405
|
|
DATA ·consts+0x40(SB)/8, $0x090a0b080d0e0f0c
|
|
DATA ·consts+0x48(SB)/8, $0x0102030005060704
|
|
GLOBL ·consts(SB), RODATA, $80
|
|
|
|
//func chaCha20_ctr32_vmx(out, inp *byte, len int, key *[32]byte, counter *[16]byte)
|
|
TEXT ·chaCha20_ctr32_vmx(SB),NOSPLIT|NOFRAME,$0
|
|
// Load the arguments inside the registers
|
|
MOVD out+0(FP), OUT
|
|
MOVD inp+8(FP), INP
|
|
MOVD len+16(FP), LEN
|
|
MOVD key+24(FP), KEY
|
|
MOVD counter+32(FP), CNT
|
|
|
|
MOVD $·consts(SB), CONSTS // point to consts addr
|
|
|
|
MOVD $16, X0
|
|
MOVD $32, X1
|
|
MOVD $48, X2
|
|
MOVD $64, X3
|
|
MOVD $31, X4
|
|
MOVD $15, X5
|
|
|
|
// Load key
|
|
LVX (KEY)(R0), K1
|
|
LVSR (KEY)(R0), T0
|
|
LVX (KEY)(X0), K2
|
|
LVX (KEY)(X4), DD0
|
|
|
|
// Load counter
|
|
LVX (CNT)(R0), K3
|
|
LVSR (CNT)(R0), T1
|
|
LVX (CNT)(X5), DD1
|
|
|
|
// Load constants
|
|
LVX (CONSTS)(R0), K0
|
|
LVX (CONSTS)(X0), K5
|
|
LVX (CONSTS)(X1), FOUR
|
|
LVX (CONSTS)(X2), SIXTEEN
|
|
LVX (CONSTS)(X3), TWENTY4
|
|
|
|
// Align key and counter
|
|
VPERM K2, K1, T0, K1
|
|
VPERM DD0, K2, T0, K2
|
|
VPERM DD1, K3, T1, K3
|
|
|
|
// Load counter to GPR
|
|
MOVWZ 0(CNT), CNT0
|
|
MOVWZ 4(CNT), CNT1
|
|
MOVWZ 8(CNT), CNT2
|
|
MOVWZ 12(CNT), CNT3
|
|
|
|
// Adjust vectors for the initial state
|
|
VADDUWM K3, K5, K3
|
|
VADDUWM K3, K5, K4
|
|
VADDUWM K4, K5, K5
|
|
|
|
// Synthesized constants
|
|
VSPLTISW $-12, TWENTY
|
|
VSPLTISW $12, TWELVE
|
|
VSPLTISW $-7, TWENTY5
|
|
|
|
VXOR T0, T0, T0
|
|
VSPLTISW $-1, OUTMASK
|
|
LVSR (INP)(R0), INPPERM
|
|
LVSL (OUT)(R0), OUTPERM
|
|
VPERM OUTMASK, T0, OUTPERM, OUTMASK
|
|
|
|
loop_outer_vmx:
|
|
// Load constant
|
|
MOVD $0x61707865, CON0
|
|
MOVD $0x3320646e, CON1
|
|
MOVD $0x79622d32, CON2
|
|
MOVD $0x6b206574, CON3
|
|
|
|
VOR K0, K0, A0
|
|
VOR K0, K0, A1
|
|
VOR K0, K0, A2
|
|
VOR K1, K1, B0
|
|
|
|
MOVD $10, TEMP
|
|
|
|
// Load key to GPR
|
|
MOVWZ 0(KEY), X4
|
|
MOVWZ 4(KEY), X5
|
|
MOVWZ 8(KEY), X6
|
|
MOVWZ 12(KEY), X7
|
|
VOR K1, K1, B1
|
|
VOR K1, K1, B2
|
|
MOVWZ 16(KEY), X8
|
|
MOVWZ 0(CNT), X12
|
|
MOVWZ 20(KEY), X9
|
|
MOVWZ 4(CNT), X13
|
|
VOR K2, K2, C0
|
|
VOR K2, K2, C1
|
|
MOVWZ 24(KEY), X10
|
|
MOVWZ 8(CNT), X14
|
|
VOR K2, K2, C2
|
|
VOR K3, K3, D0
|
|
MOVWZ 28(KEY), X11
|
|
MOVWZ 12(CNT), X15
|
|
VOR K4, K4, D1
|
|
VOR K5, K5, D2
|
|
|
|
MOVD X4, TMP0
|
|
MOVD X5, TMP1
|
|
MOVD X6, TMP2
|
|
MOVD X7, TMP3
|
|
VSPLTISW $7, SEVEN
|
|
|
|
MOVD TEMP, CTR
|
|
|
|
loop_vmx:
|
|
// CRYPTOGAMS uses a macro to create a loop using perl. This isn't possible
|
|
// using assembly macros. Therefore, the macro expansion result was used
|
|
// in order to maintain the algorithm efficiency.
|
|
// This loop generates three keystream blocks using VMX instructions and,
|
|
// in parallel, one keystream block using scalar instructions.
|
|
ADD X4, X0, X0
|
|
ADD X5, X1, X1
|
|
VADDUWM A0, B0, A0
|
|
VADDUWM A1, B1, A1
|
|
ADD X6, X2, X2
|
|
ADD X7, X3, X3
|
|
VADDUWM A2, B2, A2
|
|
VXOR D0, A0, D0
|
|
XOR X0, X12, X12
|
|
XOR X1, X13, X13
|
|
VXOR D1, A1, D1
|
|
VXOR D2, A2, D2
|
|
XOR X2, X14, X14
|
|
XOR X3, X15, X15
|
|
VPERM D0, D0, SIXTEEN, D0
|
|
VPERM D1, D1, SIXTEEN, D1
|
|
ROTLW $16, X12, X12
|
|
ROTLW $16, X13, X13
|
|
VPERM D2, D2, SIXTEEN, D2
|
|
VADDUWM C0, D0, C0
|
|
ROTLW $16, X14, X14
|
|
ROTLW $16, X15, X15
|
|
VADDUWM C1, D1, C1
|
|
VADDUWM C2, D2, C2
|
|
ADD X12, X8, X8
|
|
ADD X13, X9, X9
|
|
VXOR B0, C0, T0
|
|
VXOR B1, C1, T1
|
|
ADD X14, X10, X10
|
|
ADD X15, X11, X11
|
|
VXOR B2, C2, T2
|
|
VRLW T0, TWELVE, B0
|
|
XOR X8, X4, X4
|
|
XOR X9, X5, X5
|
|
VRLW T1, TWELVE, B1
|
|
VRLW T2, TWELVE, B2
|
|
XOR X10, X6, X6
|
|
XOR X11, X7, X7
|
|
VADDUWM A0, B0, A0
|
|
VADDUWM A1, B1, A1
|
|
ROTLW $12, X4, X4
|
|
ROTLW $12, X5, X5
|
|
VADDUWM A2, B2, A2
|
|
VXOR D0, A0, D0
|
|
ROTLW $12, X6, X6
|
|
ROTLW $12, X7, X7
|
|
VXOR D1, A1, D1
|
|
VXOR D2, A2, D2
|
|
ADD X4, X0, X0
|
|
ADD X5, X1, X1
|
|
VPERM D0, D0, TWENTY4, D0
|
|
VPERM D1, D1, TWENTY4, D1
|
|
ADD X6, X2, X2
|
|
ADD X7, X3, X3
|
|
VPERM D2, D2, TWENTY4, D2
|
|
VADDUWM C0, D0, C0
|
|
XOR X0, X12, X12
|
|
XOR X1, X13, X13
|
|
VADDUWM C1, D1, C1
|
|
VADDUWM C2, D2, C2
|
|
XOR X2, X14, X14
|
|
XOR X3, X15, X15
|
|
VXOR B0, C0, T0
|
|
VXOR B1, C1, T1
|
|
ROTLW $8, X12, X12
|
|
ROTLW $8, X13, X13
|
|
VXOR B2, C2, T2
|
|
VRLW T0, SEVEN, B0
|
|
ROTLW $8, X14, X14
|
|
ROTLW $8, X15, X15
|
|
VRLW T1, SEVEN, B1
|
|
VRLW T2, SEVEN, B2
|
|
ADD X12, X8, X8
|
|
ADD X13, X9, X9
|
|
VSLDOI $8, C0, C0, C0
|
|
VSLDOI $8, C1, C1, C1
|
|
ADD X14, X10, X10
|
|
ADD X15, X11, X11
|
|
VSLDOI $8, C2, C2, C2
|
|
VSLDOI $12, B0, B0, B0
|
|
XOR X8, X4, X4
|
|
XOR X9, X5, X5
|
|
VSLDOI $12, B1, B1, B1
|
|
VSLDOI $12, B2, B2, B2
|
|
XOR X10, X6, X6
|
|
XOR X11, X7, X7
|
|
VSLDOI $4, D0, D0, D0
|
|
VSLDOI $4, D1, D1, D1
|
|
ROTLW $7, X4, X4
|
|
ROTLW $7, X5, X5
|
|
VSLDOI $4, D2, D2, D2
|
|
VADDUWM A0, B0, A0
|
|
ROTLW $7, X6, X6
|
|
ROTLW $7, X7, X7
|
|
VADDUWM A1, B1, A1
|
|
VADDUWM A2, B2, A2
|
|
ADD X5, X0, X0
|
|
ADD X6, X1, X1
|
|
VXOR D0, A0, D0
|
|
VXOR D1, A1, D1
|
|
ADD X7, X2, X2
|
|
ADD X4, X3, X3
|
|
VXOR D2, A2, D2
|
|
VPERM D0, D0, SIXTEEN, D0
|
|
XOR X0, X15, X15
|
|
XOR X1, X12, X12
|
|
VPERM D1, D1, SIXTEEN, D1
|
|
VPERM D2, D2, SIXTEEN, D2
|
|
XOR X2, X13, X13
|
|
XOR X3, X14, X14
|
|
VADDUWM C0, D0, C0
|
|
VADDUWM C1, D1, C1
|
|
ROTLW $16, X15, X15
|
|
ROTLW $16, X12, X12
|
|
VADDUWM C2, D2, C2
|
|
VXOR B0, C0, T0
|
|
ROTLW $16, X13, X13
|
|
ROTLW $16, X14, X14
|
|
VXOR B1, C1, T1
|
|
VXOR B2, C2, T2
|
|
ADD X15, X10, X10
|
|
ADD X12, X11, X11
|
|
VRLW T0, TWELVE, B0
|
|
VRLW T1, TWELVE, B1
|
|
ADD X13, X8, X8
|
|
ADD X14, X9, X9
|
|
VRLW T2, TWELVE, B2
|
|
VADDUWM A0, B0, A0
|
|
XOR X10, X5, X5
|
|
XOR X11, X6, X6
|
|
VADDUWM A1, B1, A1
|
|
VADDUWM A2, B2, A2
|
|
XOR X8, X7, X7
|
|
XOR X9, X4, X4
|
|
VXOR D0, A0, D0
|
|
VXOR D1, A1, D1
|
|
ROTLW $12, X5, X5
|
|
ROTLW $12, X6, X6
|
|
VXOR D2, A2, D2
|
|
VPERM D0, D0, TWENTY4, D0
|
|
ROTLW $12, X7, X7
|
|
ROTLW $12, X4, X4
|
|
VPERM D1, D1, TWENTY4, D1
|
|
VPERM D2, D2, TWENTY4, D2
|
|
ADD X5, X0, X0
|
|
ADD X6, X1, X1
|
|
VADDUWM C0, D0, C0
|
|
VADDUWM C1, D1, C1
|
|
ADD X7, X2, X2
|
|
ADD X4, X3, X3
|
|
VADDUWM C2, D2, C2
|
|
VXOR B0, C0, T0
|
|
XOR X0, X15, X15
|
|
XOR X1, X12, X12
|
|
VXOR B1, C1, T1
|
|
VXOR B2, C2, T2
|
|
XOR X2, X13, X13
|
|
XOR X3, X14, X14
|
|
VRLW T0, SEVEN, B0
|
|
VRLW T1, SEVEN, B1
|
|
ROTLW $8, X15, X15
|
|
ROTLW $8, X12, X12
|
|
VRLW T2, SEVEN, B2
|
|
VSLDOI $8, C0, C0, C0
|
|
ROTLW $8, X13, X13
|
|
ROTLW $8, X14, X14
|
|
VSLDOI $8, C1, C1, C1
|
|
VSLDOI $8, C2, C2, C2
|
|
ADD X15, X10, X10
|
|
ADD X12, X11, X11
|
|
VSLDOI $4, B0, B0, B0
|
|
VSLDOI $4, B1, B1, B1
|
|
ADD X13, X8, X8
|
|
ADD X14, X9, X9
|
|
VSLDOI $4, B2, B2, B2
|
|
VSLDOI $12, D0, D0, D0
|
|
XOR X10, X5, X5
|
|
XOR X11, X6, X6
|
|
VSLDOI $12, D1, D1, D1
|
|
VSLDOI $12, D2, D2, D2
|
|
XOR X8, X7, X7
|
|
XOR X9, X4, X4
|
|
ROTLW $7, X5, X5
|
|
ROTLW $7, X6, X6
|
|
ROTLW $7, X7, X7
|
|
ROTLW $7, X4, X4
|
|
BC 0x10, 0, loop_vmx
|
|
|
|
SUB $256, LEN, LEN
|
|
|
|
// Accumulate key block
|
|
ADD $0x61707865, X0, X0
|
|
ADD $0x3320646e, X1, X1
|
|
ADD $0x79622d32, X2, X2
|
|
ADD $0x6b206574, X3, X3
|
|
ADD TMP0, X4, X4
|
|
ADD TMP1, X5, X5
|
|
ADD TMP2, X6, X6
|
|
ADD TMP3, X7, X7
|
|
MOVWZ 16(KEY), TMP0
|
|
MOVWZ 20(KEY), TMP1
|
|
MOVWZ 24(KEY), TMP2
|
|
MOVWZ 28(KEY), TMP3
|
|
ADD TMP0, X8, X8
|
|
ADD TMP1, X9, X9
|
|
ADD TMP2, X10, X10
|
|
ADD TMP3, X11, X11
|
|
|
|
MOVWZ 12(CNT), TMP0
|
|
MOVWZ 8(CNT), TMP1
|
|
MOVWZ 4(CNT), TMP2
|
|
MOVWZ 0(CNT), TEMP
|
|
ADD TMP0, X15, X15
|
|
ADD TMP1, X14, X14
|
|
ADD TMP2, X13, X13
|
|
ADD TEMP, X12, X12
|
|
|
|
// Accumulate key block
|
|
VADDUWM A0, K0, A0
|
|
VADDUWM A1, K0, A1
|
|
VADDUWM A2, K0, A2
|
|
VADDUWM B0, K1, B0
|
|
VADDUWM B1, K1, B1
|
|
VADDUWM B2, K1, B2
|
|
VADDUWM C0, K2, C0
|
|
VADDUWM C1, K2, C1
|
|
VADDUWM C2, K2, C2
|
|
VADDUWM D0, K3, D0
|
|
VADDUWM D1, K4, D1
|
|
VADDUWM D2, K5, D2
|
|
|
|
// Increment counter
|
|
ADD $4, TEMP, TEMP
|
|
MOVW TEMP, 0(CNT)
|
|
|
|
VADDUWM K3, FOUR, K3
|
|
VADDUWM K4, FOUR, K4
|
|
VADDUWM K5, FOUR, K5
|
|
|
|
// XOR the input slice (INP) with the keystream, which is stored in GPRs (X0-X3).
|
|
|
|
// Load input (aligned or not)
|
|
MOVWZ 0(INP), TMP0
|
|
MOVWZ 4(INP), TMP1
|
|
MOVWZ 8(INP), TMP2
|
|
MOVWZ 12(INP), TMP3
|
|
|
|
// XOR with input
|
|
XOR TMP0, X0, X0
|
|
XOR TMP1, X1, X1
|
|
XOR TMP2, X2, X2
|
|
XOR TMP3, X3, X3
|
|
MOVWZ 16(INP), TMP0
|
|
MOVWZ 20(INP), TMP1
|
|
MOVWZ 24(INP), TMP2
|
|
MOVWZ 28(INP), TMP3
|
|
XOR TMP0, X4, X4
|
|
XOR TMP1, X5, X5
|
|
XOR TMP2, X6, X6
|
|
XOR TMP3, X7, X7
|
|
MOVWZ 32(INP), TMP0
|
|
MOVWZ 36(INP), TMP1
|
|
MOVWZ 40(INP), TMP2
|
|
MOVWZ 44(INP), TMP3
|
|
XOR TMP0, X8, X8
|
|
XOR TMP1, X9, X9
|
|
XOR TMP2, X10, X10
|
|
XOR TMP3, X11, X11
|
|
MOVWZ 48(INP), TMP0
|
|
MOVWZ 52(INP), TMP1
|
|
MOVWZ 56(INP), TMP2
|
|
MOVWZ 60(INP), TMP3
|
|
XOR TMP0, X12, X12
|
|
XOR TMP1, X13, X13
|
|
XOR TMP2, X14, X14
|
|
XOR TMP3, X15, X15
|
|
|
|
// Store output (aligned or not)
|
|
MOVW X0, 0(OUT)
|
|
MOVW X1, 4(OUT)
|
|
MOVW X2, 8(OUT)
|
|
MOVW X3, 12(OUT)
|
|
|
|
ADD $64, INP, INP // INP points to the end of the slice for the alignment code below
|
|
|
|
MOVW X4, 16(OUT)
|
|
MOVD $16, TMP0
|
|
MOVW X5, 20(OUT)
|
|
MOVD $32, TMP1
|
|
MOVW X6, 24(OUT)
|
|
MOVD $48, TMP2
|
|
MOVW X7, 28(OUT)
|
|
MOVD $64, TMP3
|
|
MOVW X8, 32(OUT)
|
|
MOVW X9, 36(OUT)
|
|
MOVW X10, 40(OUT)
|
|
MOVW X11, 44(OUT)
|
|
MOVW X12, 48(OUT)
|
|
MOVW X13, 52(OUT)
|
|
MOVW X14, 56(OUT)
|
|
MOVW X15, 60(OUT)
|
|
ADD $64, OUT, OUT
|
|
|
|
// Load input
|
|
LVX (INP)(R0), DD0
|
|
LVX (INP)(TMP0), DD1
|
|
LVX (INP)(TMP1), DD2
|
|
LVX (INP)(TMP2), DD3
|
|
LVX (INP)(TMP3), DD4
|
|
ADD $64, INP, INP
|
|
|
|
VPERM DD1, DD0, INPPERM, DD0 // Align input
|
|
VPERM DD2, DD1, INPPERM, DD1
|
|
VPERM DD3, DD2, INPPERM, DD2
|
|
VPERM DD4, DD3, INPPERM, DD3
|
|
VXOR A0, DD0, A0 // XOR with input
|
|
VXOR B0, DD1, B0
|
|
LVX (INP)(TMP0), DD1 // Keep loading input
|
|
VXOR C0, DD2, C0
|
|
LVX (INP)(TMP1), DD2
|
|
VXOR D0, DD3, D0
|
|
LVX (INP)(TMP2), DD3
|
|
LVX (INP)(TMP3), DD0
|
|
ADD $64, INP, INP
|
|
MOVD $63, TMP3 // 63 is not a typo
|
|
VPERM A0, A0, OUTPERM, A0
|
|
VPERM B0, B0, OUTPERM, B0
|
|
VPERM C0, C0, OUTPERM, C0
|
|
VPERM D0, D0, OUTPERM, D0
|
|
|
|
VPERM DD1, DD4, INPPERM, DD4 // Align input
|
|
VPERM DD2, DD1, INPPERM, DD1
|
|
VPERM DD3, DD2, INPPERM, DD2
|
|
VPERM DD0, DD3, INPPERM, DD3
|
|
VXOR A1, DD4, A1
|
|
VXOR B1, DD1, B1
|
|
LVX (INP)(TMP0), DD1 // Keep loading
|
|
VXOR C1, DD2, C1
|
|
LVX (INP)(TMP1), DD2
|
|
VXOR D1, DD3, D1
|
|
LVX (INP)(TMP2), DD3
|
|
|
|
// Note that the LVX address is always rounded down to the nearest 16-byte
|
|
// boundary, and that it always points to at most 15 bytes beyond the end of
|
|
// the slice, so we cannot cross a page boundary.
|
|
LVX (INP)(TMP3), DD4 // Redundant in aligned case.
|
|
ADD $64, INP, INP
|
|
VPERM A1, A1, OUTPERM, A1 // Pre-misalign output
|
|
VPERM B1, B1, OUTPERM, B1
|
|
VPERM C1, C1, OUTPERM, C1
|
|
VPERM D1, D1, OUTPERM, D1
|
|
|
|
VPERM DD1, DD0, INPPERM, DD0 // Align Input
|
|
VPERM DD2, DD1, INPPERM, DD1
|
|
VPERM DD3, DD2, INPPERM, DD2
|
|
VPERM DD4, DD3, INPPERM, DD3
|
|
VXOR A2, DD0, A2
|
|
VXOR B2, DD1, B2
|
|
VXOR C2, DD2, C2
|
|
VXOR D2, DD3, D2
|
|
VPERM A2, A2, OUTPERM, A2
|
|
VPERM B2, B2, OUTPERM, B2
|
|
VPERM C2, C2, OUTPERM, C2
|
|
VPERM D2, D2, OUTPERM, D2
|
|
|
|
ANDCC $15, OUT, X1 // Is out aligned?
|
|
MOVD OUT, X0
|
|
|
|
VSEL A0, B0, OUTMASK, DD0 // Collect pre-misaligned output
|
|
VSEL B0, C0, OUTMASK, DD1
|
|
VSEL C0, D0, OUTMASK, DD2
|
|
VSEL D0, A1, OUTMASK, DD3
|
|
VSEL A1, B1, OUTMASK, B0
|
|
VSEL B1, C1, OUTMASK, C0
|
|
VSEL C1, D1, OUTMASK, D0
|
|
VSEL D1, A2, OUTMASK, A1
|
|
VSEL A2, B2, OUTMASK, B1
|
|
VSEL B2, C2, OUTMASK, C1
|
|
VSEL C2, D2, OUTMASK, D1
|
|
|
|
STVX DD0, (OUT+TMP0)
|
|
STVX DD1, (OUT+TMP1)
|
|
STVX DD2, (OUT+TMP2)
|
|
ADD $64, OUT, OUT
|
|
STVX DD3, (OUT+R0)
|
|
STVX B0, (OUT+TMP0)
|
|
STVX C0, (OUT+TMP1)
|
|
STVX D0, (OUT+TMP2)
|
|
ADD $64, OUT, OUT
|
|
STVX A1, (OUT+R0)
|
|
STVX B1, (OUT+TMP0)
|
|
STVX C1, (OUT+TMP1)
|
|
STVX D1, (OUT+TMP2)
|
|
ADD $64, OUT, OUT
|
|
|
|
BEQ aligned_vmx
|
|
|
|
SUB X1, OUT, X2 // in misaligned case edges
|
|
MOVD $0, X3 // are written byte-by-byte
|
|
|
|
unaligned_tail_vmx:
|
|
STVEBX D2, (X2+X3)
|
|
ADD $1, X3, X3
|
|
CMPW X3, X1
|
|
BNE unaligned_tail_vmx
|
|
SUB X1, X0, X2
|
|
|
|
unaligned_head_vmx:
|
|
STVEBX A0, (X2+X1)
|
|
CMPW X1, $15
|
|
ADD $1, X1, X1
|
|
BNE unaligned_head_vmx
|
|
|
|
CMPU LEN, $255 // done with 256-byte block yet?
|
|
BGT loop_outer_vmx
|
|
|
|
JMP done_vmx
|
|
|
|
aligned_vmx:
|
|
STVX A0, (X0+R0)
|
|
CMPU LEN, $255 // done with 256-byte block yet?
|
|
BGT loop_outer_vmx
|
|
|
|
done_vmx:
|
|
RET
|