mirror of
https://github.com/oxen-io/lokinet.git
synced 2024-11-03 23:15:52 +00:00
176 lines
6.1 KiB
C
176 lines
6.1 KiB
C
|
|
||
|
|
#define VEC4_ROT(A, IMM) \
|
||
|
|
_mm_or_si128(_mm_slli_epi32(A, IMM), _mm_srli_epi32(A, (32 - IMM)))
|
||
|
|
|
||
|
|
/* same, but replace 2 of the shift/shift/or "rotation" by byte shuffles (8 &
|
||
|
|
* 16) (better) */
|
||
|
|
#define VEC4_QUARTERROUND_SHUFFLE(A, B, C, D) \
|
||
|
|
x_##A = _mm_add_epi32(x_##A, x_##B); \
|
||
|
|
t_##A = _mm_xor_si128(x_##D, x_##A); \
|
||
|
|
x_##D = _mm_shuffle_epi8(t_##A, rot16); \
|
||
|
|
x_##C = _mm_add_epi32(x_##C, x_##D); \
|
||
|
|
t_##C = _mm_xor_si128(x_##B, x_##C); \
|
||
|
|
x_##B = VEC4_ROT(t_##C, 12); \
|
||
|
|
x_##A = _mm_add_epi32(x_##A, x_##B); \
|
||
|
|
t_##A = _mm_xor_si128(x_##D, x_##A); \
|
||
|
|
x_##D = _mm_shuffle_epi8(t_##A, rot8); \
|
||
|
|
x_##C = _mm_add_epi32(x_##C, x_##D); \
|
||
|
|
t_##C = _mm_xor_si128(x_##B, x_##C); \
|
||
|
|
x_##B = VEC4_ROT(t_##C, 7)
|
||
|
|
|
||
|
|
#define VEC4_QUARTERROUND(A, B, C, D) VEC4_QUARTERROUND_SHUFFLE(A, B, C, D)
|
||
|
|
|
||
|
|
if (bytes >= 256) {
|
||
|
|
/* constant for shuffling bytes (replacing multiple-of-8 rotates) */
|
||
|
|
__m128i rot16 =
|
||
|
|
_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2);
|
||
|
|
__m128i rot8 =
|
||
|
|
_mm_set_epi8(14, 13, 12, 15, 10, 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3);
|
||
|
|
|
||
|
|
__m128i x_0 = _mm_set1_epi32(x[0]);
|
||
|
|
__m128i x_1 = _mm_set1_epi32(x[1]);
|
||
|
|
__m128i x_2 = _mm_set1_epi32(x[2]);
|
||
|
|
__m128i x_3 = _mm_set1_epi32(x[3]);
|
||
|
|
__m128i x_4 = _mm_set1_epi32(x[4]);
|
||
|
|
__m128i x_5 = _mm_set1_epi32(x[5]);
|
||
|
|
__m128i x_6 = _mm_set1_epi32(x[6]);
|
||
|
|
__m128i x_7 = _mm_set1_epi32(x[7]);
|
||
|
|
__m128i x_8 = _mm_set1_epi32(x[8]);
|
||
|
|
__m128i x_9 = _mm_set1_epi32(x[9]);
|
||
|
|
__m128i x_10 = _mm_set1_epi32(x[10]);
|
||
|
|
__m128i x_11 = _mm_set1_epi32(x[11]);
|
||
|
|
__m128i x_12;
|
||
|
|
__m128i x_13;
|
||
|
|
__m128i x_14 = _mm_set1_epi32(x[14]);
|
||
|
|
__m128i x_15 = _mm_set1_epi32(x[15]);
|
||
|
|
__m128i orig0 = x_0;
|
||
|
|
__m128i orig1 = x_1;
|
||
|
|
__m128i orig2 = x_2;
|
||
|
|
__m128i orig3 = x_3;
|
||
|
|
__m128i orig4 = x_4;
|
||
|
|
__m128i orig5 = x_5;
|
||
|
|
__m128i orig6 = x_6;
|
||
|
|
__m128i orig7 = x_7;
|
||
|
|
__m128i orig8 = x_8;
|
||
|
|
__m128i orig9 = x_9;
|
||
|
|
__m128i orig10 = x_10;
|
||
|
|
__m128i orig11 = x_11;
|
||
|
|
__m128i orig12;
|
||
|
|
__m128i orig13;
|
||
|
|
__m128i orig14 = x_14;
|
||
|
|
__m128i orig15 = x_15;
|
||
|
|
__m128i t_0, t_1, t_2, t_3, t_4, t_5, t_6, t_7, t_8, t_9, t_10, t_11, t_12,
|
||
|
|
t_13, t_14, t_15;
|
||
|
|
|
||
|
|
uint32_t in12, in13;
|
||
|
|
int i;
|
||
|
|
|
||
|
|
while (bytes >= 256) {
|
||
|
|
const __m128i addv12 = _mm_set_epi64x(1, 0);
|
||
|
|
const __m128i addv13 = _mm_set_epi64x(3, 2);
|
||
|
|
__m128i t12, t13;
|
||
|
|
uint64_t in1213;
|
||
|
|
|
||
|
|
x_0 = orig0;
|
||
|
|
x_1 = orig1;
|
||
|
|
x_2 = orig2;
|
||
|
|
x_3 = orig3;
|
||
|
|
x_4 = orig4;
|
||
|
|
x_5 = orig5;
|
||
|
|
x_6 = orig6;
|
||
|
|
x_7 = orig7;
|
||
|
|
x_8 = orig8;
|
||
|
|
x_9 = orig9;
|
||
|
|
x_10 = orig10;
|
||
|
|
x_11 = orig11;
|
||
|
|
x_14 = orig14;
|
||
|
|
x_15 = orig15;
|
||
|
|
|
||
|
|
in12 = x[12];
|
||
|
|
in13 = x[13];
|
||
|
|
in1213 = ((uint64_t) in12) | (((uint64_t) in13) << 32);
|
||
|
|
t12 = _mm_set1_epi64x(in1213);
|
||
|
|
t13 = _mm_set1_epi64x(in1213);
|
||
|
|
|
||
|
|
x_12 = _mm_add_epi64(addv12, t12);
|
||
|
|
x_13 = _mm_add_epi64(addv13, t13);
|
||
|
|
|
||
|
|
t12 = _mm_unpacklo_epi32(x_12, x_13);
|
||
|
|
t13 = _mm_unpackhi_epi32(x_12, x_13);
|
||
|
|
|
||
|
|
x_12 = _mm_unpacklo_epi32(t12, t13);
|
||
|
|
x_13 = _mm_unpackhi_epi32(t12, t13);
|
||
|
|
|
||
|
|
orig12 = x_12;
|
||
|
|
orig13 = x_13;
|
||
|
|
|
||
|
|
in1213 += 4;
|
||
|
|
|
||
|
|
x[12] = in1213 & 0xFFFFFFFF;
|
||
|
|
x[13] = (in1213 >> 32) & 0xFFFFFFFF;
|
||
|
|
|
||
|
|
for (i = 0; i < ROUNDS; i += 2) {
|
||
|
|
VEC4_QUARTERROUND(0, 4, 8, 12);
|
||
|
|
VEC4_QUARTERROUND(1, 5, 9, 13);
|
||
|
|
VEC4_QUARTERROUND(2, 6, 10, 14);
|
||
|
|
VEC4_QUARTERROUND(3, 7, 11, 15);
|
||
|
|
VEC4_QUARTERROUND(0, 5, 10, 15);
|
||
|
|
VEC4_QUARTERROUND(1, 6, 11, 12);
|
||
|
|
VEC4_QUARTERROUND(2, 7, 8, 13);
|
||
|
|
VEC4_QUARTERROUND(3, 4, 9, 14);
|
||
|
|
}
|
||
|
|
|
||
|
|
#define ONEQUAD_TRANSPOSE(A, B, C, D) \
|
||
|
|
{ \
|
||
|
|
__m128i t0, t1, t2, t3; \
|
||
|
|
\
|
||
|
|
x_##A = _mm_add_epi32(x_##A, orig##A); \
|
||
|
|
x_##B = _mm_add_epi32(x_##B, orig##B); \
|
||
|
|
x_##C = _mm_add_epi32(x_##C, orig##C); \
|
||
|
|
x_##D = _mm_add_epi32(x_##D, orig##D); \
|
||
|
|
t_##A = _mm_unpacklo_epi32(x_##A, x_##B); \
|
||
|
|
t_##B = _mm_unpacklo_epi32(x_##C, x_##D); \
|
||
|
|
t_##C = _mm_unpackhi_epi32(x_##A, x_##B); \
|
||
|
|
t_##D = _mm_unpackhi_epi32(x_##C, x_##D); \
|
||
|
|
x_##A = _mm_unpacklo_epi64(t_##A, t_##B); \
|
||
|
|
x_##B = _mm_unpackhi_epi64(t_##A, t_##B); \
|
||
|
|
x_##C = _mm_unpacklo_epi64(t_##C, t_##D); \
|
||
|
|
x_##D = _mm_unpackhi_epi64(t_##C, t_##D); \
|
||
|
|
\
|
||
|
|
t0 = _mm_xor_si128(x_##A, _mm_loadu_si128((__m128i*) (m + 0))); \
|
||
|
|
_mm_storeu_si128((__m128i*) (c + 0), t0); \
|
||
|
|
t1 = _mm_xor_si128(x_##B, _mm_loadu_si128((__m128i*) (m + 64))); \
|
||
|
|
_mm_storeu_si128((__m128i*) (c + 64), t1); \
|
||
|
|
t2 = _mm_xor_si128(x_##C, _mm_loadu_si128((__m128i*) (m + 128))); \
|
||
|
|
_mm_storeu_si128((__m128i*) (c + 128), t2); \
|
||
|
|
t3 = _mm_xor_si128(x_##D, _mm_loadu_si128((__m128i*) (m + 192))); \
|
||
|
|
_mm_storeu_si128((__m128i*) (c + 192), t3); \
|
||
|
|
}
|
||
|
|
|
||
|
|
#define ONEQUAD(A, B, C, D) ONEQUAD_TRANSPOSE(A, B, C, D)
|
||
|
|
|
||
|
|
ONEQUAD(0, 1, 2, 3);
|
||
|
|
m += 16;
|
||
|
|
c += 16;
|
||
|
|
ONEQUAD(4, 5, 6, 7);
|
||
|
|
m += 16;
|
||
|
|
c += 16;
|
||
|
|
ONEQUAD(8, 9, 10, 11);
|
||
|
|
m += 16;
|
||
|
|
c += 16;
|
||
|
|
ONEQUAD(12, 13, 14, 15);
|
||
|
|
m -= 48;
|
||
|
|
c -= 48;
|
||
|
|
|
||
|
|
#undef ONEQUAD
|
||
|
|
#undef ONEQUAD_TRANSPOSE
|
||
|
|
|
||
|
|
bytes -= 256;
|
||
|
|
c += 256;
|
||
|
|
m += 256;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
#undef VEC4_ROT
|
||
|
|
#undef VEC4_QUARTERROUND
|
||
|
|
#undef VEC4_QUARTERROUND_SHUFFLE
|