Revert "Optimize lowbd fwd_txfm modules" This reverts commit aa7195d608e0fb29c42334d2d924e6a14aa4edba. Reason for revert: Mismatch in encoding result between AVX2 and non-AVX2 architectures. STATS_CHANGED BUG=aomedia:2309 Change-Id: I6d77e341ed529302aa67bd1fa56c089f4bc868b4
diff --git a/aom_dsp/x86/txfm_common_avx2.h b/aom_dsp/x86/txfm_common_avx2.h index cc729ba..8a40508 100644 --- a/aom_dsp/x86/txfm_common_avx2.h +++ b/aom_dsp/x86/txfm_common_avx2.h
@@ -167,35 +167,7 @@ out[6 + 8] = _mm256_permute2x128_si256(c[4 + 2], c[5 + 2], 0x31); out[7 + 8] = _mm256_permute2x128_si256(c[12 + 2], c[13 + 2], 0x31); } -static INLINE void transpose_16bit_16x8_avx2(const __m256i *const in, - __m256i *const out) { - const __m256i a0 = _mm256_unpacklo_epi16(in[0], in[1]); - const __m256i a1 = _mm256_unpacklo_epi16(in[2], in[3]); - const __m256i a2 = _mm256_unpacklo_epi16(in[4], in[5]); - const __m256i a3 = _mm256_unpacklo_epi16(in[6], in[7]); - const __m256i a4 = _mm256_unpackhi_epi16(in[0], in[1]); - const __m256i a5 = _mm256_unpackhi_epi16(in[2], in[3]); - const __m256i a6 = _mm256_unpackhi_epi16(in[4], in[5]); - const __m256i a7 = _mm256_unpackhi_epi16(in[6], in[7]); - const __m256i b0 = _mm256_unpacklo_epi32(a0, a1); - const __m256i b1 = _mm256_unpacklo_epi32(a2, a3); - const __m256i b2 = _mm256_unpacklo_epi32(a4, a5); - const __m256i b3 = _mm256_unpacklo_epi32(a6, a7); - const __m256i b4 = _mm256_unpackhi_epi32(a0, a1); - const __m256i b5 = _mm256_unpackhi_epi32(a2, a3); - const __m256i b6 = _mm256_unpackhi_epi32(a4, a5); - const __m256i b7 = _mm256_unpackhi_epi32(a6, a7); - - out[0] = _mm256_unpacklo_epi64(b0, b1); - out[1] = _mm256_unpackhi_epi64(b0, b1); - out[2] = _mm256_unpacklo_epi64(b4, b5); - out[3] = _mm256_unpackhi_epi64(b4, b5); - out[4] = _mm256_unpacklo_epi64(b2, b3); - out[5] = _mm256_unpackhi_epi64(b2, b3); - out[6] = _mm256_unpacklo_epi64(b6, b7); - out[7] = _mm256_unpackhi_epi64(b6, b7); -} static INLINE void flip_buf_avx2(__m256i *in, __m256i *out, int size) { for (int i = 0; i < size; ++i) { out[size - i - 1] = in[i]; @@ -263,61 +235,7 @@ } } } -static INLINE __m256i scale_round_avx2(const __m256i a, const int scale) { - const __m256i scale_rounding = - pair_set_w16_epi16(scale, 1 << (NewSqrt2Bits - 1)); - const __m256i b = _mm256_madd_epi16(a, scale_rounding); - return _mm256_srai_epi32(b, NewSqrt2Bits); -} -static INLINE void store_rect_16bit_to_32bit_w8_avx2(const __m256i a, - int32_t *const b) { - const __m256i one = _mm256_set1_epi16(1); - const __m256i a_lo = _mm256_unpacklo_epi16(a, one); - const __m256i a_hi = _mm256_unpackhi_epi16(a, one); - const __m256i b_lo = scale_round_avx2(a_lo, NewSqrt2); - const __m256i b_hi = scale_round_avx2(a_hi, NewSqrt2); - const __m256i temp = _mm256_permute2f128_si256(b_lo, b_hi, 0x31); - _mm_store_si128((__m128i *)b, _mm256_castsi256_si128(b_lo)); - _mm_store_si128((__m128i *)(b + 4), _mm256_castsi256_si128(b_hi)); - _mm256_store_si256((__m256i *)(b + 64), temp); -} -static INLINE void store_rect_buffer_16bit_to_32bit_w8_avx2( - const __m256i *const in, int32_t *const out, const int stride, - const int out_size) { - for (int i = 0; i < out_size; ++i) { - store_rect_16bit_to_32bit_w8_avx2(in[i], out + i * stride); - } -} -static INLINE void pack_reg(const __m128i *in1, const __m128i *in2, - __m256i *out) { - out[0] = _mm256_insertf128_si256(_mm256_castsi128_si256(in1[0]), in2[0], 0x1); - out[1] = _mm256_insertf128_si256(_mm256_castsi128_si256(in1[1]), in2[1], 0x1); - out[2] = _mm256_insertf128_si256(_mm256_castsi128_si256(in1[2]), in2[2], 0x1); - out[3] = _mm256_insertf128_si256(_mm256_castsi128_si256(in1[3]), in2[3], 0x1); - out[4] = _mm256_insertf128_si256(_mm256_castsi128_si256(in1[4]), in2[4], 0x1); - out[5] = _mm256_insertf128_si256(_mm256_castsi128_si256(in1[5]), in2[5], 0x1); - out[6] = _mm256_insertf128_si256(_mm256_castsi128_si256(in1[6]), in2[6], 0x1); - out[7] = _mm256_insertf128_si256(_mm256_castsi128_si256(in1[7]), in2[7], 0x1); -} -static INLINE void extract_reg(const __m256i *in, __m128i *out1) { - out1[0] = _mm256_castsi256_si128(in[0]); - out1[1] = _mm256_castsi256_si128(in[1]); - out1[2] = _mm256_castsi256_si128(in[2]); - out1[3] = _mm256_castsi256_si128(in[3]); - out1[4] = _mm256_castsi256_si128(in[4]); - out1[5] = _mm256_castsi256_si128(in[5]); - out1[6] = _mm256_castsi256_si128(in[6]); - out1[7] = _mm256_castsi256_si128(in[7]); - out1[8] = _mm256_extractf128_si256(in[0], 0x01); - out1[9] = _mm256_extractf128_si256(in[1], 0x01); - out1[10] = _mm256_extractf128_si256(in[2], 0x01); - out1[11] = _mm256_extractf128_si256(in[3], 0x01); - out1[12] = _mm256_extractf128_si256(in[4], 0x01); - out1[13] = _mm256_extractf128_si256(in[5], 0x01); - out1[14] = _mm256_extractf128_si256(in[6], 0x01); - out1[15] = _mm256_extractf128_si256(in[7], 0x01); -} #ifdef __cplusplus } #endif
diff --git a/av1/encoder/x86/av1_fwd_txfm2d_avx2.c b/av1/encoder/x86/av1_fwd_txfm2d_avx2.c index 3f0f2c3..13982cc 100644 --- a/av1/encoder/x86/av1_fwd_txfm2d_avx2.c +++ b/av1/encoder/x86/av1_fwd_txfm2d_avx2.c
@@ -1407,6 +1407,13 @@ output[14] = x1[15]; output[15] = x1[0]; } + +static INLINE __m256i scale_round_avx2(const __m256i a, const int scale) { + const __m256i scale__r = pair_set_w16_epi16(scale, 1 << (NewSqrt2Bits - 1)); + const __m256i b = _mm256_madd_epi16(a, scale__r); + return _mm256_srai_epi32(b, NewSqrt2Bits); +} + static INLINE void fidentity16x16_new_avx2(const __m256i *input, __m256i *output, int8_t cos_bit) { (void)cos_bit; @@ -1990,781 +1997,6 @@ } } -void btf_16_avx2(__m256i w0, __m256i w1, __m256i in0, __m256i in1, - __m128i *out0, __m128i *out1, __m128i *out2, __m128i *out3, - __m256i __rounding, int8_t cos_bit) { - __m256i t0 = _mm256_unpacklo_epi16(in0, in1); - __m256i t1 = _mm256_unpackhi_epi16(in0, in1); - __m256i u0 = _mm256_madd_epi16(t0, w0); - __m256i u1 = _mm256_madd_epi16(t1, w0); - __m256i v0 = _mm256_madd_epi16(t0, w1); - __m256i v1 = _mm256_madd_epi16(t1, w1); - - __m256i a0 = _mm256_add_epi32(u0, __rounding); - __m256i a1 = _mm256_add_epi32(u1, __rounding); - __m256i b0 = _mm256_add_epi32(v0, __rounding); - __m256i b1 = _mm256_add_epi32(v1, __rounding); - - __m256i c0 = _mm256_srai_epi32(a0, cos_bit); - __m256i c1 = _mm256_srai_epi32(a1, cos_bit); - __m256i d0 = _mm256_srai_epi32(b0, cos_bit); - __m256i d1 = _mm256_srai_epi32(b1, cos_bit); - - __m256i temp0 = _mm256_packs_epi32(c0, c1); - __m256i temp1 = _mm256_packs_epi32(d0, d1); - - *out0 = _mm256_castsi256_si128(temp0); - *out1 = _mm256_castsi256_si128(temp1); - *out2 = _mm256_extractf128_si256(temp0, 0x01); - *out3 = _mm256_extractf128_si256(temp1, 0x01); -} -static INLINE void fdct8x8_new_avx2(const __m256i *input, __m256i *output, - int8_t cos_bit) { - const int32_t *cospi = cospi_arr(cos_bit); - const __m256i __rounding = _mm256_set1_epi32(1 << (cos_bit - 1)); - - __m256i cospi_m32_p32 = pair_set_w16_epi16(-cospi[32], cospi[32]); - __m256i cospi_p32_p32 = pair_set_w16_epi16(cospi[32], cospi[32]); - __m256i cospi_p32_m32 = pair_set_w16_epi16(cospi[32], -cospi[32]); - __m256i cospi_p48_p16 = pair_set_w16_epi16(cospi[48], cospi[16]); - __m256i cospi_m16_p48 = pair_set_w16_epi16(-cospi[16], cospi[48]); - __m256i cospi_p56_p08 = pair_set_w16_epi16(cospi[56], cospi[8]); - __m256i cospi_m08_p56 = pair_set_w16_epi16(-cospi[8], cospi[56]); - __m256i cospi_p24_p40 = pair_set_w16_epi16(cospi[24], cospi[40]); - __m256i cospi_m40_p24 = pair_set_w16_epi16(-cospi[40], cospi[24]); - - // stage 1 - __m256i x1[8]; - x1[0] = _mm256_adds_epi16(input[0], input[7]); - x1[7] = _mm256_subs_epi16(input[0], input[7]); - x1[1] = _mm256_adds_epi16(input[1], input[6]); - x1[6] = _mm256_subs_epi16(input[1], input[6]); - x1[2] = _mm256_adds_epi16(input[2], input[5]); - x1[5] = _mm256_subs_epi16(input[2], input[5]); - x1[3] = _mm256_adds_epi16(input[3], input[4]); - x1[4] = _mm256_subs_epi16(input[3], input[4]); - - // stage 2 - __m256i x2[8]; - x2[0] = _mm256_adds_epi16(x1[0], x1[3]); - x2[3] = _mm256_subs_epi16(x1[0], x1[3]); - x2[1] = _mm256_adds_epi16(x1[1], x1[2]); - x2[2] = _mm256_subs_epi16(x1[1], x1[2]); - x2[4] = x1[4]; - btf_16_w16_avx2(cospi_m32_p32, cospi_p32_p32, &x1[5], &x1[6], __rounding, - cos_bit); - x2[5] = x1[5]; - x2[6] = x1[6]; - x2[7] = x1[7]; - - // stage 3 - __m256i x3[8]; - btf_16_w16_avx2(cospi_p32_p32, cospi_p32_m32, &x2[0], &x2[1], __rounding, - cos_bit); - x3[0] = x2[0]; - x3[1] = x2[1]; - btf_16_w16_avx2(cospi_p48_p16, cospi_m16_p48, &x2[2], &x2[3], __rounding, - cos_bit); - x3[2] = x2[2]; - x3[3] = x2[3]; - x3[4] = _mm256_adds_epi16(x2[4], x2[5]); - x3[5] = _mm256_subs_epi16(x2[4], x2[5]); - x3[6] = _mm256_subs_epi16(x2[7], x2[6]); - x3[7] = _mm256_adds_epi16(x2[7], x2[6]); - - // stage 4 - __m256i x4[8]; - x4[0] = x3[0]; - x4[1] = x3[1]; - x4[2] = x3[2]; - x4[3] = x3[3]; - btf_16_w16_avx2(cospi_p56_p08, cospi_m08_p56, &x3[4], &x3[7], __rounding, - cos_bit); - x4[4] = x3[4]; - x4[7] = x3[7]; - btf_16_w16_avx2(cospi_p24_p40, cospi_m40_p24, &x3[5], &x3[6], __rounding, - cos_bit); - x4[5] = x3[5]; - x4[6] = x3[6]; - // stage 5 - output[0] = x4[0]; - output[1] = x4[4]; - output[2] = x4[2]; - output[3] = x4[6]; - output[4] = x4[1]; - output[5] = x4[5]; - output[6] = x4[3]; - output[7] = x4[7]; -} -static INLINE void fadst8x8_new_avx2(const __m256i *input, __m256i *output, - int8_t cos_bit) { - const int32_t *cospi = cospi_arr(cos_bit); - const __m256i __zero = _mm256_setzero_si256(); - const __m256i __rounding = _mm256_set1_epi32(1 << (cos_bit - 1)); - - __m256i cospi_p32_p32 = pair_set_w16_epi16(cospi[32], cospi[32]); - __m256i cospi_p32_m32 = pair_set_w16_epi16(cospi[32], -cospi[32]); - __m256i cospi_p16_p48 = pair_set_w16_epi16(cospi[16], cospi[48]); - __m256i cospi_p48_m16 = pair_set_w16_epi16(cospi[48], -cospi[16]); - __m256i cospi_m48_p16 = pair_set_w16_epi16(-cospi[48], cospi[16]); - __m256i cospi_p04_p60 = pair_set_w16_epi16(cospi[4], cospi[60]); - __m256i cospi_p60_m04 = pair_set_w16_epi16(cospi[60], -cospi[4]); - __m256i cospi_p20_p44 = pair_set_w16_epi16(cospi[20], cospi[44]); - __m256i cospi_p44_m20 = pair_set_w16_epi16(cospi[44], -cospi[20]); - __m256i cospi_p36_p28 = pair_set_w16_epi16(cospi[36], cospi[28]); - __m256i cospi_p28_m36 = pair_set_w16_epi16(cospi[28], -cospi[36]); - __m256i cospi_p52_p12 = pair_set_w16_epi16(cospi[52], cospi[12]); - __m256i cospi_p12_m52 = pair_set_w16_epi16(cospi[12], -cospi[52]); - - // stage 1 - __m256i x1[8]; - x1[0] = input[0]; - x1[1] = _mm256_subs_epi16(__zero, input[7]); - x1[2] = _mm256_subs_epi16(__zero, input[3]); - x1[3] = input[4]; - x1[4] = _mm256_subs_epi16(__zero, input[1]); - x1[5] = input[6]; - x1[6] = input[2]; - x1[7] = _mm256_subs_epi16(__zero, input[5]); - - // stage 2 - __m256i x2[8]; - x2[0] = x1[0]; - x2[1] = x1[1]; - btf_16_w16_avx2(cospi_p32_p32, cospi_p32_m32, &x1[2], &x1[3], __rounding, - cos_bit); - x2[2] = x1[2]; - x2[3] = x1[3]; - x2[4] = x1[4]; - x2[5] = x1[5]; - btf_16_w16_avx2(cospi_p32_p32, cospi_p32_m32, &x1[6], &x1[7], __rounding, - cos_bit); - x2[6] = x1[6]; - x2[7] = x1[7]; - - // stage 3 - __m256i x3[8]; - x3[0] = _mm256_adds_epi16(x2[0], x2[2]); - x3[2] = _mm256_subs_epi16(x2[0], x2[2]); - x3[1] = _mm256_adds_epi16(x2[1], x2[3]); - x3[3] = _mm256_subs_epi16(x2[1], x2[3]); - x3[4] = _mm256_adds_epi16(x2[4], x2[6]); - x3[6] = _mm256_subs_epi16(x2[4], x2[6]); - x3[5] = _mm256_adds_epi16(x2[5], x2[7]); - x3[7] = _mm256_subs_epi16(x2[5], x2[7]); - - // stage 4 - __m256i x4[8]; - x4[0] = x3[0]; - x4[1] = x3[1]; - x4[2] = x3[2]; - x4[3] = x3[3]; - btf_16_w16_avx2(cospi_p16_p48, cospi_p48_m16, &x3[4], &x3[5], __rounding, - cos_bit); - x4[4] = x3[4]; - x4[5] = x3[5]; - btf_16_w16_avx2(cospi_m48_p16, cospi_p16_p48, &x3[6], &x3[7], __rounding, - cos_bit); - x4[6] = x3[6]; - x4[7] = x3[7]; - - // stage 5 - __m256i x5[8]; - x5[0] = _mm256_adds_epi16(x4[0], x4[4]); - x5[4] = _mm256_subs_epi16(x4[0], x4[4]); - x5[1] = _mm256_adds_epi16(x4[1], x4[5]); - x5[5] = _mm256_subs_epi16(x4[1], x4[5]); - x5[2] = _mm256_adds_epi16(x4[2], x4[6]); - x5[6] = _mm256_subs_epi16(x4[2], x4[6]); - x5[3] = _mm256_adds_epi16(x4[3], x4[7]); - x5[7] = _mm256_subs_epi16(x4[3], x4[7]); - - // stage 6 - __m256i x6[8]; - btf_16_w16_avx2(cospi_p04_p60, cospi_p60_m04, &x5[0], &x5[1], __rounding, - cos_bit); - x6[0] = x5[0]; - x6[1] = x5[1]; - btf_16_w16_avx2(cospi_p20_p44, cospi_p44_m20, &x5[2], &x5[3], __rounding, - cos_bit); - x6[2] = x5[2]; - x6[3] = x5[3]; - btf_16_w16_avx2(cospi_p36_p28, cospi_p28_m36, &x5[4], &x5[5], __rounding, - cos_bit); - x6[4] = x5[4]; - x6[5] = x5[5]; - btf_16_w16_avx2(cospi_p52_p12, cospi_p12_m52, &x5[6], &x5[7], __rounding, - cos_bit); - x6[6] = x5[6]; - x6[7] = x5[7]; - - // stage 7 - output[0] = x6[1]; - output[1] = x6[6]; - output[2] = x6[3]; - output[3] = x6[4]; - output[4] = x6[5]; - output[5] = x6[2]; - output[6] = x6[7]; - output[7] = x6[0]; -} -static INLINE void fidentity8x8_new_avx2(const __m256i *input, __m256i *output, - int8_t cos_bit) { - (void)cos_bit; - - output[0] = _mm256_adds_epi16(input[0], input[0]); - output[1] = _mm256_adds_epi16(input[1], input[1]); - output[2] = _mm256_adds_epi16(input[2], input[2]); - output[3] = _mm256_adds_epi16(input[3], input[3]); - output[4] = _mm256_adds_epi16(input[4], input[4]); - output[5] = _mm256_adds_epi16(input[5], input[5]); - output[6] = _mm256_adds_epi16(input[6], input[6]); - output[7] = _mm256_adds_epi16(input[7], input[7]); -} -static INLINE void fdct8x16_new_avx2(const __m128i *input, __m128i *output, - int8_t cos_bit) { - const int32_t *cospi = cospi_arr(cos_bit); - const __m256i __rounding_256 = _mm256_set1_epi32(1 << (cos_bit - 1)); - const __m128i __rounding = _mm_set1_epi32(1 << (cos_bit - 1)); - __m128i temp0, temp1, temp2, temp3; - __m256i in0, in1; - __m128i cospi_m32_p32 = pair_set_epi16(-cospi[32], cospi[32]); - __m128i cospi_p32_p32 = pair_set_epi16(cospi[32], cospi[32]); - __m128i cospi_p32_m32 = pair_set_epi16(cospi[32], -cospi[32]); - __m128i cospi_p48_p16 = pair_set_epi16(cospi[48], cospi[16]); - __m128i cospi_m16_p48 = pair_set_epi16(-cospi[16], cospi[48]); - __m128i cospi_m48_m16 = pair_set_epi16(-cospi[48], -cospi[16]); - __m128i cospi_p56_p08 = pair_set_epi16(cospi[56], cospi[8]); - __m128i cospi_m08_p56 = pair_set_epi16(-cospi[8], cospi[56]); - __m128i cospi_p24_p40 = pair_set_epi16(cospi[24], cospi[40]); - __m128i cospi_m40_p24 = pair_set_epi16(-cospi[40], cospi[24]); - __m128i cospi_p60_p04 = pair_set_epi16(cospi[60], cospi[4]); - __m128i cospi_m04_p60 = pair_set_epi16(-cospi[4], cospi[60]); - __m128i cospi_p28_p36 = pair_set_epi16(cospi[28], cospi[36]); - __m128i cospi_m36_p28 = pair_set_epi16(-cospi[36], cospi[28]); - __m128i cospi_p44_p20 = pair_set_epi16(cospi[44], cospi[20]); - __m128i cospi_m20_p44 = pair_set_epi16(-cospi[20], cospi[44]); - __m128i cospi_p12_p52 = pair_set_epi16(cospi[12], cospi[52]); - __m128i cospi_m52_p12 = pair_set_epi16(-cospi[52], cospi[12]); - - __m256i cospi_arr[12]; - - cospi_arr[0] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_m32_p32), - cospi_m32_p32, 0x1); - cospi_arr[1] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_p32_p32), - cospi_p32_p32, 0x1); - cospi_arr[2] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_p32_p32), - cospi_p48_p16, 0x1); - cospi_arr[3] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_p32_m32), - cospi_m16_p48, 0x1); - cospi_arr[4] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_m16_p48), - cospi_m48_m16, 0x1); - cospi_arr[5] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_p48_p16), - cospi_m16_p48, 0x1); - cospi_arr[6] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_p56_p08), - cospi_p24_p40, 0x1); - cospi_arr[7] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_m08_p56), - cospi_m40_p24, 0x1); - cospi_arr[8] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_p60_p04), - cospi_p28_p36, 0x1); - cospi_arr[9] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_m04_p60), - cospi_m36_p28, 0x1); - cospi_arr[10] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_p44_p20), - cospi_p12_p52, 0x1); - cospi_arr[11] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_m20_p44), - cospi_m52_p12, 0x1); - - __m256i x[8]; - x[0] = - _mm256_insertf128_si256(_mm256_castsi128_si256(input[0]), input[1], 0x1); - x[1] = _mm256_insertf128_si256(_mm256_castsi128_si256(input[15]), input[14], - 0x1); - x[2] = - _mm256_insertf128_si256(_mm256_castsi128_si256(input[2]), input[3], 0x1); - x[3] = _mm256_insertf128_si256(_mm256_castsi128_si256(input[13]), input[12], - 0x1); - x[4] = - _mm256_insertf128_si256(_mm256_castsi128_si256(input[5]), input[4], 0x1); - x[5] = _mm256_insertf128_si256(_mm256_castsi128_si256(input[10]), input[11], - 0x1); - x[6] = - _mm256_insertf128_si256(_mm256_castsi128_si256(input[7]), input[6], 0x1); - x[7] = - _mm256_insertf128_si256(_mm256_castsi128_si256(input[8]), input[9], 0x1); - - // stage 1 - __m256i x1[16]; - x1[0] = _mm256_adds_epi16(x[0], x[1]); - x1[7] = _mm256_subs_epi16(x[0], x[1]); - x1[1] = _mm256_adds_epi16(x[2], x[3]); - x1[6] = _mm256_subs_epi16(x[2], x[3]); - x1[2] = _mm256_adds_epi16(x[4], x[5]); - x1[5] = _mm256_subs_epi16(x[4], x[5]); - x1[3] = _mm256_adds_epi16(x[6], x[7]); - x1[4] = _mm256_subs_epi16(x[6], x[7]); - - // stage 2 - __m256i x2[8]; - x2[0] = _mm256_adds_epi16(x1[0], x1[3]); - x2[7] = _mm256_subs_epi16(x1[0], x1[3]); - x2[1] = _mm256_adds_epi16(x1[1], x1[2]); - x2[6] = _mm256_subs_epi16(x1[1], x1[2]); - x2[2] = x1[4]; - x2[3] = x1[7]; - btf_16_avx2(cospi_arr[0], cospi_arr[1], x1[5], x1[6], &temp0, &temp1, &temp2, - &temp3, __rounding_256, cos_bit); - x2[4] = _mm256_insertf128_si256(_mm256_castsi128_si256(temp2), temp0, 0x1); - x2[5] = _mm256_insertf128_si256(_mm256_castsi128_si256(temp3), temp1, 0x1); - - // stage 3 - __m256i x3[8]; - x2[1] = _mm256_permute4x64_epi64(x2[1], 0x4e); - x3[0] = _mm256_adds_epi16(x2[0], x2[1]); - x3[1] = _mm256_subs_epi16(x2[0], x2[1]); - x3[2] = _mm256_blend_epi32(x2[7], x2[6], 0xf0); - btf_16_sse2(cospi_m32_p32, cospi_p32_p32, _mm256_castsi256_si128(x2[6]), - _mm256_extractf128_si256(x2[7], 0x01), temp0, temp1); - x3[7] = _mm256_insertf128_si256(_mm256_castsi128_si256(temp1), temp0, 0x1); - x3[3] = _mm256_adds_epi16(x2[2], x2[4]); - x3[4] = _mm256_subs_epi16(x2[2], x2[4]); - x3[5] = _mm256_adds_epi16(x2[3], x2[5]); - x3[6] = _mm256_subs_epi16(x2[3], x2[5]); - - // stage 4 - __m256i x4[8]; - in0 = _mm256_blend_epi32(x3[0], x3[1], 0xf0); - in1 = _mm256_permute2f128_si256(x3[0], x3[1], 0x21); - btf_16_avx2(cospi_arr[2], cospi_arr[3], in0, in1, &output[0], &output[8], - &output[4], &output[12], __rounding_256, cos_bit); - x4[2] = _mm256_adds_epi16(x3[2], x3[7]); - x4[3] = _mm256_subs_epi16(x3[2], x3[7]); - x4[4] = _mm256_permute2f128_si256(x3[3], x3[4], 0x20); - x4[5] = _mm256_permute2f128_si256(x3[6], x3[5], 0x20); - in0 = _mm256_permute2f128_si256(x3[3], x3[4], 0x31); - in1 = _mm256_permute2f128_si256(x3[5], x3[6], 0x31); - btf_16_avx2(cospi_arr[4], cospi_arr[5], in0, in1, &temp0, &temp1, &temp2, - &temp3, __rounding_256, cos_bit); - x4[6] = _mm256_insertf128_si256(_mm256_castsi128_si256(temp0), temp2, 0x1); - x4[7] = _mm256_insertf128_si256(_mm256_castsi128_si256(temp3), temp1, 0x1); - - // stage 5 - __m256i x5[8]; - in0 = _mm256_permute2f128_si256(x4[2], x4[3], 0x31); - in1 = _mm256_permute2f128_si256(x4[2], x4[3], 0x20); - btf_16_avx2(cospi_arr[6], cospi_arr[7], in0, in1, &output[2], &output[14], - &output[10], &output[6], __rounding_256, cos_bit); - x5[4] = _mm256_adds_epi16(x4[4], x4[6]); - x5[5] = _mm256_subs_epi16(x4[4], x4[6]); - x5[6] = _mm256_adds_epi16(x4[5], x4[7]); - x5[7] = _mm256_subs_epi16(x4[5], x4[7]); - - // stage 6 - in0 = _mm256_permute2f128_si256(x5[4], x5[5], 0x20); - in1 = _mm256_permute2f128_si256(x5[6], x5[7], 0x31); - btf_16_avx2(cospi_arr[8], cospi_arr[9], in0, in1, &output[1], &output[15], - &output[9], &output[7], __rounding_256, cos_bit); - in0 = _mm256_permute2f128_si256(x5[5], x5[4], 0x31); - in1 = _mm256_permute2f128_si256(x5[7], x5[6], 0x20); - btf_16_avx2(cospi_arr[10], cospi_arr[11], in0, in1, &output[5], &output[11], - &output[13], &output[3], __rounding_256, cos_bit); -} -static INLINE void fadst8x16_new_avx2(const __m128i *input, __m128i *output, - int8_t cos_bit) { - const int32_t *cospi = cospi_arr(cos_bit); - const __m256i __zero = _mm256_setzero_si256(); - const __m256i __rounding_256 = _mm256_set1_epi32(1 << (cos_bit - 1)); - __m256i in0, in1; - __m128i temp0, temp1, temp2, temp3; - - __m128i cospi_p32_p32 = pair_set_epi16(cospi[32], cospi[32]); - __m128i cospi_p32_m32 = pair_set_epi16(cospi[32], -cospi[32]); - __m128i cospi_p16_p48 = pair_set_epi16(cospi[16], cospi[48]); - __m128i cospi_p48_m16 = pair_set_epi16(cospi[48], -cospi[16]); - __m128i cospi_m48_p16 = pair_set_epi16(-cospi[48], cospi[16]); - __m128i cospi_p08_p56 = pair_set_epi16(cospi[8], cospi[56]); - __m128i cospi_p56_m08 = pair_set_epi16(cospi[56], -cospi[8]); - __m128i cospi_p40_p24 = pair_set_epi16(cospi[40], cospi[24]); - __m128i cospi_p24_m40 = pair_set_epi16(cospi[24], -cospi[40]); - __m128i cospi_m56_p08 = pair_set_epi16(-cospi[56], cospi[8]); - __m128i cospi_m24_p40 = pair_set_epi16(-cospi[24], cospi[40]); - __m128i cospi_p02_p62 = pair_set_epi16(cospi[2], cospi[62]); - __m128i cospi_p62_m02 = pair_set_epi16(cospi[62], -cospi[2]); - __m128i cospi_p10_p54 = pair_set_epi16(cospi[10], cospi[54]); - __m128i cospi_p54_m10 = pair_set_epi16(cospi[54], -cospi[10]); - __m128i cospi_p18_p46 = pair_set_epi16(cospi[18], cospi[46]); - __m128i cospi_p46_m18 = pair_set_epi16(cospi[46], -cospi[18]); - __m128i cospi_p26_p38 = pair_set_epi16(cospi[26], cospi[38]); - __m128i cospi_p38_m26 = pair_set_epi16(cospi[38], -cospi[26]); - __m128i cospi_p34_p30 = pair_set_epi16(cospi[34], cospi[30]); - __m128i cospi_p30_m34 = pair_set_epi16(cospi[30], -cospi[34]); - __m128i cospi_p42_p22 = pair_set_epi16(cospi[42], cospi[22]); - __m128i cospi_p22_m42 = pair_set_epi16(cospi[22], -cospi[42]); - __m128i cospi_p50_p14 = pair_set_epi16(cospi[50], cospi[14]); - __m128i cospi_p14_m50 = pair_set_epi16(cospi[14], -cospi[50]); - __m128i cospi_p58_p06 = pair_set_epi16(cospi[58], cospi[6]); - __m128i cospi_p06_m58 = pair_set_epi16(cospi[6], -cospi[58]); - - __m256i cospi_arr[20]; - - cospi_arr[0] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_p32_p32), - cospi_p32_p32, 0x1); - cospi_arr[1] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_p32_m32), - cospi_p32_m32, 0x1); - cospi_arr[2] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_p32_p32), - cospi_p32_p32, 0x1); - cospi_arr[3] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_p32_m32), - cospi_p32_m32, 0x1); - cospi_arr[4] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_p16_p48), - cospi_m48_p16, 0x1); - cospi_arr[5] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_p48_m16), - cospi_p16_p48, 0x1); - cospi_arr[6] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_p16_p48), - cospi_m48_p16, 0x1); - cospi_arr[7] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_p48_m16), - cospi_p16_p48, 0x1); - cospi_arr[8] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_p08_p56), - cospi_p40_p24, 0x1); - cospi_arr[9] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_p56_m08), - cospi_p24_m40, 0x1); - cospi_arr[10] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_m56_p08), - cospi_m24_p40, 0x1); - cospi_arr[11] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_p08_p56), - cospi_p40_p24, 0x1); - cospi_arr[12] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_p02_p62), - cospi_p10_p54, 0x1); - cospi_arr[13] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_p62_m02), - cospi_p54_m10, 0x1); - cospi_arr[14] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_p18_p46), - cospi_p26_p38, 0x1); - cospi_arr[15] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_p46_m18), - cospi_p38_m26, 0x1); - cospi_arr[16] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_p34_p30), - cospi_p42_p22, 0x1); - cospi_arr[17] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_p30_m34), - cospi_p22_m42, 0x1); - cospi_arr[18] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_p50_p14), - cospi_p58_p06, 0x1); - cospi_arr[19] = _mm256_insertf128_si256(_mm256_castsi128_si256(cospi_p14_m50), - cospi_p06_m58, 0x1); - - __m256i x[8]; - x[0] = - _mm256_insertf128_si256(_mm256_castsi128_si256(input[0]), input[4], 0x1); - x[1] = - _mm256_insertf128_si256(_mm256_castsi128_si256(input[2]), input[6], 0x1); - x[2] = - _mm256_insertf128_si256(_mm256_castsi128_si256(input[8]), input[12], 0x1); - x[3] = _mm256_insertf128_si256(_mm256_castsi128_si256(input[10]), input[14], - 0x1); - x[4] = - _mm256_insertf128_si256(_mm256_castsi128_si256(input[1]), input[9], 0x1); - x[5] = - _mm256_insertf128_si256(_mm256_castsi128_si256(input[3]), input[11], 0x1); - x[6] = - _mm256_insertf128_si256(_mm256_castsi128_si256(input[5]), input[13], 0x1); - x[7] = - _mm256_insertf128_si256(_mm256_castsi128_si256(input[7]), input[15], 0x1); - - // stage 1 - __m256i x1[8]; - x1[0] = x[0]; - x1[1] = _mm256_subs_epi16(__zero, x[7]); - x1[2] = x[2]; - x1[3] = _mm256_subs_epi16(__zero, x[5]); - x1[4] = _mm256_subs_epi16(__zero, x[4]); - x1[5] = x[3]; - x1[6] = _mm256_subs_epi16(__zero, x[6]); - x1[7] = x[1]; - - // stage 2 - __m256i x2[8]; - x2[0] = _mm256_blend_epi32(x1[0], x1[1], 0xf0); - x2[3] = _mm256_blend_epi32(x1[3], x1[2], 0xf0); - x2[4] = _mm256_blend_epi32(x1[4], x1[5], 0xf0); - x2[7] = _mm256_blend_epi32(x1[7], x1[6], 0xf0); - in0 = _mm256_blend_epi32(x1[1], x1[0], 0xf0); - in1 = _mm256_blend_epi32(x1[2], x1[3], 0xf0); - btf_16_avx2(cospi_arr[0], cospi_arr[1], in0, in1, &temp0, &temp1, &temp2, - &temp3, __rounding_256, cos_bit); - x2[1] = _mm256_insertf128_si256(_mm256_castsi128_si256(temp0), temp1, 0x1); - x2[2] = _mm256_insertf128_si256(_mm256_castsi128_si256(temp2), temp3, 0x1); - in0 = _mm256_permute2f128_si256(x1[7], x1[6], 0x21); - in1 = _mm256_permute2f128_si256(x1[4], x1[5], 0x21); - btf_16_avx2(cospi_arr[2], cospi_arr[3], in0, in1, &temp0, &temp1, &temp2, - &temp3, __rounding_256, cos_bit); - x2[5] = _mm256_insertf128_si256(_mm256_castsi128_si256(temp0), temp1, 0x1); - x2[6] = _mm256_insertf128_si256(_mm256_castsi128_si256(temp2), temp3, 0x1); - - // stage 3 - __m256i x3[8]; - x3[0] = _mm256_adds_epi16(x2[0], x2[1]); - x3[1] = _mm256_subs_epi16(x2[0], x2[1]); - x3[2] = _mm256_adds_epi16(x2[3], x2[2]); - x3[3] = _mm256_subs_epi16(x2[3], x2[2]); - x3[4] = _mm256_adds_epi16(x2[4], x2[5]); - x3[5] = _mm256_subs_epi16(x2[4], x2[5]); - x3[6] = _mm256_adds_epi16(x2[7], x2[6]); - x3[7] = _mm256_subs_epi16(x2[7], x2[6]); - - // stage 4 - __m256i x4[8]; - x4[0] = x3[0]; - x4[1] = x3[1]; - x4[4] = x3[4]; - x4[5] = x3[5]; - in0 = _mm256_permute2f128_si256(x3[2], x3[3], 0x20); - in1 = _mm256_permute2f128_si256(x3[2], x3[3], 0x31); - btf_16_avx2(cospi_arr[4], cospi_arr[5], in0, in1, &temp0, &temp1, &temp2, - &temp3, __rounding_256, cos_bit); - x4[2] = _mm256_insertf128_si256(_mm256_castsi128_si256(temp0), temp1, 0x1); - x4[3] = _mm256_insertf128_si256(_mm256_castsi128_si256(temp2), temp3, 0x1); - in0 = _mm256_permute2f128_si256(x3[6], x3[7], 0x20); - in1 = _mm256_permute2f128_si256(x3[6], x3[7], 0x31); - btf_16_avx2(cospi_arr[6], cospi_arr[7], in0, in1, &temp0, &temp1, &temp2, - &temp3, __rounding_256, cos_bit); - x4[6] = _mm256_insertf128_si256(_mm256_castsi128_si256(temp0), temp1, 0x1); - x4[7] = _mm256_insertf128_si256(_mm256_castsi128_si256(temp2), temp3, 0x1); - - // stage 5 - __m256i x5[8]; - x5[0] = _mm256_adds_epi16(x4[0], x4[2]); - x5[1] = _mm256_subs_epi16(x4[0], x4[2]); - x5[2] = _mm256_adds_epi16(x4[1], x4[3]); - x5[3] = _mm256_subs_epi16(x4[1], x4[3]); - x5[4] = _mm256_adds_epi16(x4[4], x4[6]); - x5[5] = _mm256_subs_epi16(x4[4], x4[6]); - x5[6] = _mm256_adds_epi16(x4[5], x4[7]); - x5[7] = _mm256_subs_epi16(x4[5], x4[7]); - - // stage 6 - __m256i x6[8]; - x6[0] = x5[0]; - x6[1] = x5[2]; - x6[2] = x5[1]; - x6[3] = x5[3]; - in0 = _mm256_permute2f128_si256(x5[4], x5[6], 0x20); - in1 = _mm256_permute2f128_si256(x5[4], x5[6], 0x31); - btf_16_avx2(cospi_arr[8], cospi_arr[9], in0, in1, &temp0, &temp1, &temp2, - &temp3, __rounding_256, cos_bit); - x6[4] = _mm256_insertf128_si256(_mm256_castsi128_si256(temp0), temp1, 0x1); - x6[5] = _mm256_insertf128_si256(_mm256_castsi128_si256(temp2), temp3, 0x1); - in0 = _mm256_permute2f128_si256(x5[5], x5[7], 0x20); - in1 = _mm256_permute2f128_si256(x5[5], x5[7], 0x31); - btf_16_avx2(cospi_arr[10], cospi_arr[11], in0, in1, &temp0, &temp1, &temp2, - &temp3, __rounding_256, cos_bit); - x6[6] = _mm256_insertf128_si256(_mm256_castsi128_si256(temp0), temp1, 0x1); - x6[7] = _mm256_insertf128_si256(_mm256_castsi128_si256(temp2), temp3, 0x1); - - // stage 7 - __m256i x7[8]; - x7[0] = _mm256_adds_epi16(x6[0], x6[4]); - x7[1] = _mm256_subs_epi16(x6[0], x6[4]); - x7[2] = _mm256_adds_epi16(x6[1], x6[5]); - x7[3] = _mm256_subs_epi16(x6[1], x6[5]); - x7[4] = _mm256_adds_epi16(x6[2], x6[6]); - x7[5] = _mm256_subs_epi16(x6[2], x6[6]); - x7[6] = _mm256_adds_epi16(x6[3], x6[7]); - x7[7] = _mm256_subs_epi16(x6[3], x6[7]); - - // stage 8 - in0 = _mm256_permute2f128_si256(x7[0], x7[2], 0x20); - in1 = _mm256_permute2f128_si256(x7[0], x7[2], 0x31); - btf_16_avx2(cospi_arr[12], cospi_arr[13], in0, in1, &output[15], &output[0], - &output[13], &output[2], __rounding_256, cos_bit); - in0 = _mm256_permute2f128_si256(x7[4], x7[6], 0x20); - in1 = _mm256_permute2f128_si256(x7[4], x7[6], 0x31); - btf_16_avx2(cospi_arr[14], cospi_arr[15], in0, in1, &output[11], &output[4], - &output[9], &output[6], __rounding_256, cos_bit); - in0 = _mm256_permute2f128_si256(x7[1], x7[3], 0x20); - in1 = _mm256_permute2f128_si256(x7[1], x7[3], 0x31); - btf_16_avx2(cospi_arr[16], cospi_arr[17], in0, in1, &output[7], &output[8], - &output[5], &output[10], __rounding_256, cos_bit); - in0 = _mm256_permute2f128_si256(x7[5], x7[7], 0x20); - in1 = _mm256_permute2f128_si256(x7[5], x7[7], 0x31); - btf_16_avx2(cospi_arr[18], cospi_arr[19], in0, in1, &output[3], &output[12], - &output[1], &output[14], __rounding_256, cos_bit); -} -static INLINE void fidentity8x16_new_avx2(const __m128i *input, __m128i *output, - int8_t cos_bit) { - (void)cos_bit; - const __m256i one = _mm256_set1_epi16(1); - __m256i temp; - for (int i = 0; i < 16; i += 2) { - temp = _mm256_insertf128_si256(_mm256_castsi128_si256(input[i]), - input[i + 1], 0x1); - const __m256i a_lo = _mm256_unpacklo_epi16(temp, one); - const __m256i a_hi = _mm256_unpackhi_epi16(temp, one); - const __m256i b_lo = scale_round_avx2(a_lo, 2 * NewSqrt2); - const __m256i b_hi = scale_round_avx2(a_hi, 2 * NewSqrt2); - temp = _mm256_packs_epi32(b_lo, b_hi); - output[i] = _mm256_castsi256_si128(temp); - output[i + 1] = _mm256_extractf128_si256(temp, 0x1); - } -} -static const transform_1d_avx2 row_txfm8x16_arr[TX_TYPES] = { - fdct8x8_new_avx2, // DCT_DCT - fdct8x8_new_avx2, // ADST_DCT - fadst8x8_new_avx2, // DCT_ADST - fadst8x8_new_avx2, // ADST_ADST - fdct8x8_new_avx2, // FLIPADST_DCT - fadst8x8_new_avx2, // DCT_FLIPADST - fadst8x8_new_avx2, // FLIPADST_FLIPADST - fadst8x8_new_avx2, // ADST_FLIPADST - fadst8x8_new_avx2, // FLIPADST_ADST - fidentity8x8_new_avx2, // IDTX - fidentity8x8_new_avx2, // V_DCT - fdct8x8_new_avx2, // H_DCT - fidentity8x8_new_avx2, // V_ADST - fadst8x8_new_avx2, // H_ADST - fidentity8x8_new_avx2, // V_FLIPADST - fadst8x8_new_avx2 // H_FLIPADST -}; - -static const transform_1d_sse2 col_txfm8x16_arr[TX_TYPES] = { - fdct8x16_new_avx2, // DCT_DCT - fadst8x16_new_avx2, // ADST_DCT - fdct8x16_new_avx2, // DCT_ADST - fadst8x16_new_avx2, // ADST_ADST - fadst8x16_new_avx2, // FLIPADST_DCT - fdct8x16_new_avx2, // DCT_FLIPADST - fadst8x16_new_avx2, // FLIPADST_FLIPADST - fadst8x16_new_avx2, // ADST_FLIPADST - fadst8x16_new_avx2, // FLIPADST_ADST - fidentity8x16_new_avx2, // IDTX - fdct8x16_new_avx2, // V_DCT - fidentity8x16_new_avx2, // H_DCT - fadst8x16_new_avx2, // V_ADST - fidentity8x16_new_avx2, // H_ADST - fadst8x16_new_avx2, // V_FLIPADST - fidentity8x16_new_avx2 // H_FLIPADST -}; -static const transform_1d_avx2 col_txfm16x8_arr[TX_TYPES] = { - fdct8x8_new_avx2, // DCT_DCT - fadst8x8_new_avx2, // ADST_DCT - fdct8x8_new_avx2, // DCT_ADST - fadst8x8_new_avx2, // ADST_ADST - fadst8x8_new_avx2, // FLIPADST_DCT - fdct8x8_new_avx2, // DCT_FLIPADST - fadst8x8_new_avx2, // FLIPADST_FLIPADST - fadst8x8_new_avx2, // ADST_FLIPADST - fadst8x8_new_avx2, // FLIPADST_ADST - fidentity8x8_new_avx2, // IDTX - fdct8x8_new_avx2, // V_DCT - fidentity8x8_new_avx2, // H_DCT - fadst8x8_new_avx2, // V_ADST - fidentity8x8_new_avx2, // H_ADST - fadst8x8_new_avx2, // V_FLIPADST - fidentity8x8_new_avx2, // H_FLIPADST -}; - -static const transform_1d_sse2 row_txfm16x8_arr[TX_TYPES] = { - fdct8x16_new_avx2, // DCT_DCT - fdct8x16_new_avx2, // ADST_DCT - fadst8x16_new_avx2, // DCT_ADST - fadst8x16_new_avx2, // ADST_ADST - fdct8x16_new_avx2, // FLIPADST_DCT - fadst8x16_new_avx2, // DCT_FLIPADST - fadst8x16_new_avx2, // FLIPADST_FLIPADST - fadst8x16_new_avx2, // ADST_FLIPADST - fadst8x16_new_avx2, // FLIPADST_ADST - fidentity8x16_new_avx2, // IDTX - fidentity8x16_new_avx2, // V_DCT - fdct8x16_new_avx2, // H_DCT - fidentity8x16_new_avx2, // V_ADST - fadst8x16_new_avx2, // H_ADST - fidentity8x16_new_avx2, // V_FLIPADST - fadst8x16_new_avx2 // H_FLIPADST -}; -void lowbd_fwd_txfm2d_8x16_avx2(const int16_t *input, int32_t *output, - int stride, TX_TYPE tx_type, int bd) { - (void)bd; - __m128i buf0[16], buf1[16]; - __m256i buf2[8]; - const int8_t *shift = fwd_txfm_shift_ls[TX_8X16]; - const int txw_idx = get_txw_idx(TX_8X16); - const int txh_idx = get_txh_idx(TX_8X16); - const int cos_bit_col = fwd_cos_bit_col[txw_idx][txh_idx]; - const int cos_bit_row = fwd_cos_bit_row[txw_idx][txh_idx]; - const int width = 8; - const int height = 16; - const transform_1d_sse2 col_txfm = col_txfm8x16_arr[tx_type]; - const transform_1d_avx2 row_txfm = row_txfm8x16_arr[tx_type]; - int ud_flip, lr_flip; - - get_flip_cfg(tx_type, &ud_flip, &lr_flip); - if (ud_flip) { - load_buffer_16bit_to_16bit_flip(input, stride, buf0, height); - } else { - load_buffer_16bit_to_16bit(input, stride, buf0, height); - } - round_shift_16bit(buf0, height, shift[0]); - col_txfm(buf0, buf0, cos_bit_col); - round_shift_16bit(buf0, height, shift[1]); - transpose_16bit_8x8(buf0, buf1); - transpose_16bit_8x8(buf0 + 8, buf1 + 8); - - __m128i *bufl, *bufu; - if (lr_flip) { - bufl = buf0; - bufu = buf0 + 8; - flip_buf_sse2(buf1 + width * 0, bufl, width); - flip_buf_sse2(buf1 + width * 1, bufu, width); - } else { - bufl = buf1 + width * 0; - bufu = buf1 + width * 1; - } - pack_reg(bufl, bufu, buf2); - row_txfm(buf2, buf2, cos_bit_row); - round_shift_16bit_w16_avx2(buf2, width, shift[2]); - transpose_16bit_16x8_avx2(buf2, buf2); - store_rect_buffer_16bit_to_32bit_w8_avx2(buf2, output, width, 8); -} -void lowbd_fwd_txfm2d_16x8_avx2(const int16_t *input, int32_t *output, - int stride, TX_TYPE tx_type, int bd) { - (void)bd; - __m128i buf0[16], buf1[16]; - __m256i buf2[8]; - const int8_t *shift = fwd_txfm_shift_ls[TX_16X8]; - const int txw_idx = get_txw_idx(TX_16X8); - const int txh_idx = get_txh_idx(TX_16X8); - const int cos_bit_col = fwd_cos_bit_col[txw_idx][txh_idx]; - const int cos_bit_row = fwd_cos_bit_row[txw_idx][txh_idx]; - const int width = 16; - const int height = 8; - const transform_1d_avx2 col_txfm = col_txfm16x8_arr[tx_type]; - const transform_1d_sse2 row_txfm = row_txfm16x8_arr[tx_type]; - __m128i *buf; - int ud_flip, lr_flip; - - get_flip_cfg(tx_type, &ud_flip, &lr_flip); - - if (ud_flip) { - load_buffer_16bit_to_16bit_flip(input + 8 * 0, stride, buf0, height); - load_buffer_16bit_to_16bit_flip(input + 8 * 1, stride, &buf0[8], height); - } else { - load_buffer_16bit_to_16bit(input + 8 * 0, stride, buf0, height); - load_buffer_16bit_to_16bit(input + 8 * 1, stride, &buf0[8], height); - } - pack_reg(buf0, &buf0[8], buf2); - round_shift_16bit_w16_avx2(buf2, height, shift[0]); - col_txfm(buf2, buf2, cos_bit_col); - round_shift_16bit_w16_avx2(buf2, height, shift[1]); - transpose_16bit_16x8_avx2(buf2, buf2); - extract_reg(buf2, buf1); - - if (lr_flip) { - buf = buf0; - flip_buf_sse2(buf1, buf, width); - } else { - buf = buf1; - } - row_txfm(buf, buf, cos_bit_row); - round_shift_16bit(buf, width, shift[2]); - transpose_16bit_8x8(buf, buf); - store_rect_buffer_16bit_to_32bit_w8(buf, output, width, height); - transpose_16bit_8x8(buf + 8, buf + 8); - store_rect_buffer_16bit_to_32bit_w8(buf + 8, output + 8, width, height); -} static FwdTxfm2dFunc fwd_txfm2d_func_ls[TX_SIZES_ALL] = { av1_lowbd_fwd_txfm2d_4x4_sse2, // 4x4 transform av1_lowbd_fwd_txfm2d_8x8_sse2, // 8x8 transform @@ -2773,8 +2005,8 @@ lowbd_fwd_txfm2d_64x64_avx2, // 64x64 transform av1_lowbd_fwd_txfm2d_4x8_sse2, // 4x8 transform av1_lowbd_fwd_txfm2d_8x4_sse2, // 8x4 transform - lowbd_fwd_txfm2d_8x16_avx2, // 8x16 transform - lowbd_fwd_txfm2d_16x8_avx2, // 16x8 transform + av1_lowbd_fwd_txfm2d_8x16_sse2, // 8x16 transform + av1_lowbd_fwd_txfm2d_16x8_sse2, // 16x8 transform lowbd_fwd_txfm2d_16x32_avx2, // 16x32 transform lowbd_fwd_txfm2d_32x16_avx2, // 32x16 transform lowbd_fwd_txfm2d_32x64_avx2, // 32x64 transform
diff --git a/test/av1_fwd_txfm2d_test.cc b/test/av1_fwd_txfm2d_test.cc index b40d389..d55085f 100644 --- a/test/av1_fwd_txfm2d_test.cc +++ b/test/av1_fwd_txfm2d_test.cc
@@ -287,67 +287,6 @@ } } } -void AV1FwdTxfm2dSpeedTest(TX_SIZE tx_size, lowbd_fwd_txfm_func target_func) { - TxfmParam param; - memset(¶m, 0, sizeof(param)); - const int rows = tx_size_high[tx_size]; - const int cols = tx_size_wide[tx_size]; - const int num_loops = 1000000 / (rows * cols); - - for (int i = 0; i < 2; ++i) { - const int bd = 8; - for (int tx_type = 0; tx_type < TX_TYPES; ++tx_type) { - if (libaom_test::IsTxSizeTypeValid( - tx_size, static_cast<TX_TYPE>(tx_type)) == false) { - continue; - } - - FwdTxfm2dFunc ref_func = libaom_test::fwd_txfm_func_ls[tx_size]; - if (ref_func != NULL) { - DECLARE_ALIGNED(32, int16_t, input[64 * 64]) = { 0 }; - DECLARE_ALIGNED(32, int32_t, output[64 * 64]); - DECLARE_ALIGNED(32, int32_t, ref_output[64 * 64]); - int input_stride = 64; - ACMRandom rnd(ACMRandom::DeterministicSeed()); - - for (int r = 0; r < rows; ++r) { - for (int c = 0; c < cols; ++c) { - input[r * input_stride + c] = rnd.Rand16() % (1 << bd); - } - } - - param.tx_type = (TX_TYPE)tx_type; - param.tx_size = (TX_SIZE)tx_size; - param.tx_set_type = EXT_TX_SET_ALL16; - param.bd = bd; - - aom_usec_timer ref_timer, test_timer; - - aom_usec_timer_start(&ref_timer); - for (int i = 0; i < num_loops; ++i) { - ref_func(input, ref_output, input_stride, (TX_TYPE)tx_type, bd); - } - aom_usec_timer_mark(&ref_timer); - const int elapsed_time_c = - static_cast<int>(aom_usec_timer_elapsed(&ref_timer)); - - aom_usec_timer_start(&test_timer); - for (int i = 0; i < num_loops; ++i) { - target_func(input, output, input_stride, ¶m); - } - aom_usec_timer_mark(&test_timer); - const int elapsed_time_simd = - static_cast<int>(aom_usec_timer_elapsed(&test_timer)); - - printf( - "txfm_size[%d] \t txfm_type[%d] \t c_time=%d \t simd_time=%d \t " - "gain=%d \n", - tx_size, tx_type, elapsed_time_c, elapsed_time_simd, - (elapsed_time_c / elapsed_time_simd)); - } - } - } -} typedef ::testing::tuple<TX_SIZE, lowbd_fwd_txfm_func> LbdFwdTxfm2dParam; @@ -356,9 +295,7 @@ TEST_P(AV1FwdTxfm2dTest, match) { AV1FwdTxfm2dMatchTest(GET_PARAM(0), GET_PARAM(1)); } -TEST_P(AV1FwdTxfm2dTest, DISABLED_Speed) { - AV1FwdTxfm2dSpeedTest(GET_PARAM(0), GET_PARAM(1)); -} + using ::testing::Combine; using ::testing::Values; using ::testing::ValuesIn;