jochen
be92762969
The live tests reached one shared bus and assert, read and remove the mesh's own objects by their fixed names, so packages run in parallel deleted what each other read and the suite passed only one package at a time; a red suite read as noise. internal/testbus starts a server per test, linked in at the nats-server release go.mod pins, and a test holds that pin to the catalogue's bus image and to the facts snapshot's bus when there is one, so the tests never run a bus the mesh does not. The waiter test read a timing (the most connections held at one look) and now reads the state it means (the fewest held across the wait). make check runs the packages in parallel under the race detector, with a timeout.
225 lines
5.3 KiB
ArmAsm
225 lines
5.3 KiB
ArmAsm
// Copyright 2018 The Go Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style
|
|
// license that can be found in the LICENSE file.
|
|
|
|
//go:build gc && !purego
|
|
|
|
#include "go_asm.h"
|
|
#include "textflag.h"
|
|
|
|
// This is an implementation of the ChaCha20 encryption algorithm as
|
|
// specified in RFC 7539. It uses vector instructions to compute
|
|
// 4 keystream blocks in parallel (256 bytes) which are then XORed
|
|
// with the bytes in the input slice.
|
|
|
|
GLOBL ·constants<>(SB), RODATA|NOPTR, $32
|
|
// BSWAP: swap bytes in each 4-byte element
|
|
DATA ·constants<>+0x00(SB)/4, $0x03020100
|
|
DATA ·constants<>+0x04(SB)/4, $0x07060504
|
|
DATA ·constants<>+0x08(SB)/4, $0x0b0a0908
|
|
DATA ·constants<>+0x0c(SB)/4, $0x0f0e0d0c
|
|
// J0: [j0, j1, j2, j3]
|
|
DATA ·constants<>+0x10(SB)/4, $0x61707865
|
|
DATA ·constants<>+0x14(SB)/4, $0x3320646e
|
|
DATA ·constants<>+0x18(SB)/4, $0x79622d32
|
|
DATA ·constants<>+0x1c(SB)/4, $0x6b206574
|
|
|
|
#define BSWAP V5
|
|
#define J0 V6
|
|
#define KEY0 V7
|
|
#define KEY1 V8
|
|
#define NONCE V9
|
|
#define CTR V10
|
|
#define M0 V11
|
|
#define M1 V12
|
|
#define M2 V13
|
|
#define M3 V14
|
|
#define INC V15
|
|
#define X0 V16
|
|
#define X1 V17
|
|
#define X2 V18
|
|
#define X3 V19
|
|
#define X4 V20
|
|
#define X5 V21
|
|
#define X6 V22
|
|
#define X7 V23
|
|
#define X8 V24
|
|
#define X9 V25
|
|
#define X10 V26
|
|
#define X11 V27
|
|
#define X12 V28
|
|
#define X13 V29
|
|
#define X14 V30
|
|
#define X15 V31
|
|
|
|
#define NUM_ROUNDS 20
|
|
|
|
#define ROUND4(a0, a1, a2, a3, b0, b1, b2, b3, c0, c1, c2, c3, d0, d1, d2, d3) \
|
|
VAF a1, a0, a0 \
|
|
VAF b1, b0, b0 \
|
|
VAF c1, c0, c0 \
|
|
VAF d1, d0, d0 \
|
|
VX a0, a2, a2 \
|
|
VX b0, b2, b2 \
|
|
VX c0, c2, c2 \
|
|
VX d0, d2, d2 \
|
|
VERLLF $16, a2, a2 \
|
|
VERLLF $16, b2, b2 \
|
|
VERLLF $16, c2, c2 \
|
|
VERLLF $16, d2, d2 \
|
|
VAF a2, a3, a3 \
|
|
VAF b2, b3, b3 \
|
|
VAF c2, c3, c3 \
|
|
VAF d2, d3, d3 \
|
|
VX a3, a1, a1 \
|
|
VX b3, b1, b1 \
|
|
VX c3, c1, c1 \
|
|
VX d3, d1, d1 \
|
|
VERLLF $12, a1, a1 \
|
|
VERLLF $12, b1, b1 \
|
|
VERLLF $12, c1, c1 \
|
|
VERLLF $12, d1, d1 \
|
|
VAF a1, a0, a0 \
|
|
VAF b1, b0, b0 \
|
|
VAF c1, c0, c0 \
|
|
VAF d1, d0, d0 \
|
|
VX a0, a2, a2 \
|
|
VX b0, b2, b2 \
|
|
VX c0, c2, c2 \
|
|
VX d0, d2, d2 \
|
|
VERLLF $8, a2, a2 \
|
|
VERLLF $8, b2, b2 \
|
|
VERLLF $8, c2, c2 \
|
|
VERLLF $8, d2, d2 \
|
|
VAF a2, a3, a3 \
|
|
VAF b2, b3, b3 \
|
|
VAF c2, c3, c3 \
|
|
VAF d2, d3, d3 \
|
|
VX a3, a1, a1 \
|
|
VX b3, b1, b1 \
|
|
VX c3, c1, c1 \
|
|
VX d3, d1, d1 \
|
|
VERLLF $7, a1, a1 \
|
|
VERLLF $7, b1, b1 \
|
|
VERLLF $7, c1, c1 \
|
|
VERLLF $7, d1, d1
|
|
|
|
#define PERMUTE(mask, v0, v1, v2, v3) \
|
|
VPERM v0, v0, mask, v0 \
|
|
VPERM v1, v1, mask, v1 \
|
|
VPERM v2, v2, mask, v2 \
|
|
VPERM v3, v3, mask, v3
|
|
|
|
#define ADDV(x, v0, v1, v2, v3) \
|
|
VAF x, v0, v0 \
|
|
VAF x, v1, v1 \
|
|
VAF x, v2, v2 \
|
|
VAF x, v3, v3
|
|
|
|
#define XORV(off, dst, src, v0, v1, v2, v3) \
|
|
VLM off(src), M0, M3 \
|
|
PERMUTE(BSWAP, v0, v1, v2, v3) \
|
|
VX v0, M0, M0 \
|
|
VX v1, M1, M1 \
|
|
VX v2, M2, M2 \
|
|
VX v3, M3, M3 \
|
|
VSTM M0, M3, off(dst)
|
|
|
|
#define SHUFFLE(a, b, c, d, t, u, v, w) \
|
|
VMRHF a, c, t \ // t = {a[0], c[0], a[1], c[1]}
|
|
VMRHF b, d, u \ // u = {b[0], d[0], b[1], d[1]}
|
|
VMRLF a, c, v \ // v = {a[2], c[2], a[3], c[3]}
|
|
VMRLF b, d, w \ // w = {b[2], d[2], b[3], d[3]}
|
|
VMRHF t, u, a \ // a = {a[0], b[0], c[0], d[0]}
|
|
VMRLF t, u, b \ // b = {a[1], b[1], c[1], d[1]}
|
|
VMRHF v, w, c \ // c = {a[2], b[2], c[2], d[2]}
|
|
VMRLF v, w, d // d = {a[3], b[3], c[3], d[3]}
|
|
|
|
// func xorKeyStreamVX(dst, src []byte, key *[8]uint32, nonce *[3]uint32, counter *uint32)
|
|
TEXT ·xorKeyStreamVX(SB), NOSPLIT, $0
|
|
MOVD $·constants<>(SB), R1
|
|
MOVD dst+0(FP), R2 // R2=&dst[0]
|
|
LMG src+24(FP), R3, R4 // R3=&src[0] R4=len(src)
|
|
MOVD key+48(FP), R5 // R5=key
|
|
MOVD nonce+56(FP), R6 // R6=nonce
|
|
MOVD counter+64(FP), R7 // R7=counter
|
|
|
|
// load BSWAP and J0
|
|
VLM (R1), BSWAP, J0
|
|
|
|
// setup
|
|
MOVD $95, R0
|
|
VLM (R5), KEY0, KEY1
|
|
VLL R0, (R6), NONCE
|
|
VZERO M0
|
|
VLEIB $7, $32, M0
|
|
VSRLB M0, NONCE, NONCE
|
|
|
|
// initialize counter values
|
|
VLREPF (R7), CTR
|
|
VZERO INC
|
|
VLEIF $1, $1, INC
|
|
VLEIF $2, $2, INC
|
|
VLEIF $3, $3, INC
|
|
VAF INC, CTR, CTR
|
|
VREPIF $4, INC
|
|
|
|
chacha:
|
|
VREPF $0, J0, X0
|
|
VREPF $1, J0, X1
|
|
VREPF $2, J0, X2
|
|
VREPF $3, J0, X3
|
|
VREPF $0, KEY0, X4
|
|
VREPF $1, KEY0, X5
|
|
VREPF $2, KEY0, X6
|
|
VREPF $3, KEY0, X7
|
|
VREPF $0, KEY1, X8
|
|
VREPF $1, KEY1, X9
|
|
VREPF $2, KEY1, X10
|
|
VREPF $3, KEY1, X11
|
|
VLR CTR, X12
|
|
VREPF $1, NONCE, X13
|
|
VREPF $2, NONCE, X14
|
|
VREPF $3, NONCE, X15
|
|
|
|
MOVD $(NUM_ROUNDS/2), R1
|
|
|
|
loop:
|
|
ROUND4(X0, X4, X12, X8, X1, X5, X13, X9, X2, X6, X14, X10, X3, X7, X15, X11)
|
|
ROUND4(X0, X5, X15, X10, X1, X6, X12, X11, X2, X7, X13, X8, X3, X4, X14, X9)
|
|
|
|
ADD $-1, R1
|
|
BNE loop
|
|
|
|
// decrement length
|
|
ADD $-256, R4
|
|
|
|
// rearrange vectors
|
|
SHUFFLE(X0, X1, X2, X3, M0, M1, M2, M3)
|
|
ADDV(J0, X0, X1, X2, X3)
|
|
SHUFFLE(X4, X5, X6, X7, M0, M1, M2, M3)
|
|
ADDV(KEY0, X4, X5, X6, X7)
|
|
SHUFFLE(X8, X9, X10, X11, M0, M1, M2, M3)
|
|
ADDV(KEY1, X8, X9, X10, X11)
|
|
VAF CTR, X12, X12
|
|
SHUFFLE(X12, X13, X14, X15, M0, M1, M2, M3)
|
|
ADDV(NONCE, X12, X13, X14, X15)
|
|
|
|
// increment counters
|
|
VAF INC, CTR, CTR
|
|
|
|
// xor keystream with plaintext
|
|
XORV(0*64, R2, R3, X0, X4, X8, X12)
|
|
XORV(1*64, R2, R3, X1, X5, X9, X13)
|
|
XORV(2*64, R2, R3, X2, X6, X10, X14)
|
|
XORV(3*64, R2, R3, X3, X7, X11, X15)
|
|
|
|
// increment pointers
|
|
MOVD $256(R2), R2
|
|
MOVD $256(R3), R3
|
|
|
|
CMPBNE R4, $0, chacha
|
|
|
|
VSTEF $0, CTR, (R7)
|
|
RET
|