static const __m128i ZERO =
_MM_SETR_EPI32(0u, 0u, 0u, 0u);
static const __m128i INDEXES =
_MM_SETR_EPI8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
static const __m128i ONES = _MM_SETR_EPI32(0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
_Alignas(32) static unsigned char MASK_SOURCE[32] =
{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF};
static __m128i mask_string1(__m128i input, uint32_t *const plen) {
const __m128i zeros = _mm_cmpeq_epi8(input, ZERO);
if (_mm_testz_si128(zeros, zeros)) {
*plen = 16;
return input;
} else {
const uint32_t length = _tzcnt_u32(_mm_movemask_epi8(zeros));
*plen = length;
return
length < 15 ?
_mm_and_si128(input, _mm_loadu_si128((__m128i_u *) (MASK_SOURCE + (16 - length)))) :
input;
}
}
static __m128i mask_string2(__m128i input, uint32_t *const plen) {
__m128i zeros = _mm_cmpeq_epi8(input, ZERO);
if (_mm_testz_si128(zeros, zeros)) {
*plen = 16;
return input;
} else {
const uint32_t length = _tzcnt_u32(_mm_movemask_epi8(zeros));
*plen = length;
if (length < 15) {
zeros = _mm_or_si128(zeros, _mm_slli_si128(zeros, 1));
zeros = _mm_or_si128(zeros, _mm_slli_si128(zeros, 2));
zeros = _mm_or_si128(zeros, _mm_slli_si128(zeros, 4));
zeros = _mm_or_si128(zeros, _mm_slli_si128(zeros, 8));
// Now apply that mask
return _mm_andnot_si128(zeros, input);
} else {
return input;
}
}
}
static __m128i mask_string3(__m128i input, uint32_t *const plen) {
const __m128i zeros = _mm_cmpeq_epi8(input, ZERO);
if (_mm_testz_si128(zeros, zeros)) {
*plen = 16;
return input;
} else {
const uint32_t length = _tzcnt_u32(_mm_movemask_epi8(zeros));
*plen = length;
return
length < 15 ?
_mm_andnot_si128(_mm_cmpgt_epi8(INDEXES, _mm_set1_epi8(length)), input) :
input;
}
}
__m128i set_zeros_3(__m128i v, uint32_t *plen) {
// cmp zeros
__m128i eq_zero = _mm_cmpeq_epi8(ZERO, v);
if (_mm_testz_si128(eq_zero, eq_zero)) {
*plen = 16;
return v;
} else {
*plen = _tzcnt_u32(_mm_movemask_epi8(eq_zero));
#ifdef COND
if (_mm_testz_si128(eq_zero, eq_zero)) {
return;
}
#endif
__m128i eq_zero64 = _mm_cmpeq_epi64(eq_zero, ZERO);
__m128i mask64_1 = _mm_unpacklo_epi64(ONES, eq_zero64);
// add(-1) / sub(1)
__m128i partial_mask = _mm_add_epi64(eq_zero, ONES);
#if defined __AVX512F__ && defined __AVX512VL__
__m128i result =
_mm_ternarylogic_epi64(partial_mask, mask64_1, v, (1 << 7));
#else
__m128i mask = _mm_and_si128(mask64_1, partial_mask);
__m128i result = _mm_and_si128(mask, v);
#endif
return result;
}
}