Obey do_average flag when doing convolve_round Doing this means that we don't have to memset temporary buffers to zero in reconinter.c, which was taking ~5% of cycles in a short encoding test (using perf to attach to a running encode). Change-Id: Ibb6e31920000b876c6ee99f454d89c8a97e9fb31
diff --git a/av1/common/convolve.c b/av1/common/convolve.c index bc76a3c..40234ad 100644 --- a/av1/common/convolve.c +++ b/av1/common/convolve.c
@@ -361,7 +361,10 @@ sum += y_filter[k] * src_vert[(y - fo_vert + k) * im_stride + x]; } CONV_BUF_TYPE res = ROUND_POWER_OF_TWO(sum, conv_params->round_1); - dst[y * dst_stride + x] += res; + if (conv_params->do_average) + dst[y * dst_stride + x] += res; + else + dst[y * dst_stride + x] = res; } } } @@ -416,7 +419,10 @@ sum += y_filter[k] * src_y[(k - fo_vert) * im_stride]; } CONV_BUF_TYPE res = ROUND_POWER_OF_TWO(sum, conv_params->round_1); - dst[y * dst_stride + x] += res; + if (conv_params->do_average) + dst[y * dst_stride + x] += res; + else + dst[y * dst_stride + x] = res; } src_vert++; } @@ -475,7 +481,10 @@ CONV_BUF_TYPE res = ROUND_POWER_OF_TWO(sum, conv_params->round_1) - ((1 << (offset_bits - conv_params->round_1)) + (1 << (offset_bits - conv_params->round_1 - 1))); - dst[y * dst_stride + x] += res; + if (conv_params->do_average) + dst[y * dst_stride + x] += res; + else + dst[y * dst_stride + x] = res; } } } @@ -536,7 +545,10 @@ CONV_BUF_TYPE res = ROUND_POWER_OF_TWO(sum, conv_params->round_1) - ((1 << (offset_bits - conv_params->round_1)) + (1 << (offset_bits - conv_params->round_1 - 1))); - dst[y * dst_stride + x] += res; + if (conv_params->do_average) + dst[y * dst_stride + x] += res; + else + dst[y * dst_stride + x] = res; } src_vert++; } @@ -669,7 +681,10 @@ sum += y_filter[k] * src_vert[(y - fo_vert + k) * im_stride + x]; } CONV_BUF_TYPE res = ROUND_POWER_OF_TWO(sum, conv_params->round_1); - dst[y * dst_stride + x] += res; + if (conv_params->do_average) + dst[y * dst_stride + x] += res; + else + dst[y * dst_stride + x] = res; } } } @@ -725,7 +740,10 @@ sum += y_filter[k] * src_y[(k - fo_vert) * im_stride]; } CONV_BUF_TYPE res = ROUND_POWER_OF_TWO(sum, conv_params->round_1); - dst[y * dst_stride + x] += res; + if (conv_params->do_average) + dst[y * dst_stride + x] += res; + else + dst[y * dst_stride + x] = res; } src_vert++; } @@ -778,7 +796,10 @@ CONV_BUF_TYPE res = ROUND_POWER_OF_TWO(sum, conv_params->round_1) - ((1 << (offset_bits - conv_params->round_1)) + (1 << (offset_bits - conv_params->round_1 - 1))); - dst[y * dst_stride + x] += res; + if (conv_params->do_average) + dst[y * dst_stride + x] += res; + else + dst[y * dst_stride + x] = res; } } } @@ -838,7 +859,10 @@ CONV_BUF_TYPE res = ROUND_POWER_OF_TWO(sum, conv_params->round_1) - ((1 << (offset_bits - conv_params->round_1)) + (1 << (offset_bits - conv_params->round_1 - 1))); - dst[y * dst_stride + x] += res; + if (conv_params->do_average) + dst[y * dst_stride + x] += res; + else + dst[y * dst_stride + x] = res; } src_vert++; }
diff --git a/av1/common/reconinter.c b/av1/common/reconinter.c index f9ea2f7..7cdaff2 100644 --- a/av1/common/reconinter.c +++ b/av1/common/reconinter.c
@@ -975,7 +975,6 @@ #if CONFIG_HIGHBITDEPTH #if CONFIG_CONVOLVE_ROUND DECLARE_ALIGNED(16, CONV_BUF_TYPE, tmp_dst2[MAX_SB_SQUARE]); - memset(tmp_dst2, 0, sizeof(tmp_dst2)); int tmp_dst2_stride = MAX_SB_SIZE; CONV_BUF_TYPE *org_dst = conv_params->dst; int org_dst_stride = conv_params->dst_stride; @@ -1310,7 +1309,6 @@ is_compound = has_second_ref(this_mbmi); #if CONFIG_CONVOLVE_ROUND DECLARE_ALIGNED(16, int32_t, tmp_dst[8 * 8]); - av1_zero(tmp_dst); int tmp_dst_stride = 8; assert(w <= 8 && h <= 8); #endif // CONFIG_CONVOLVE_ROUND @@ -1489,7 +1487,6 @@ SubpelParams subpel_params[2]; #if CONFIG_CONVOLVE_ROUND DECLARE_ALIGNED(16, int32_t, tmp_dst[MAX_SB_SIZE * MAX_SB_SIZE]); - av1_zero(tmp_dst); #endif // CONFIG_CONVOLVE_ROUND #if CONFIG_EXT_INTER && CONFIG_COMPOUND_SINGLEREF
diff --git a/av1/common/warped_motion.c b/av1/common/warped_motion.c index 1ab5d3a..4b93a60 100644 --- a/av1/common/warped_motion.c +++ b/av1/common/warped_motion.c
@@ -1050,7 +1050,10 @@ (1 << (offset_bits_horiz + FILTER_BITS - conv_params->round_0 - conv_params->round_1)) - (1 << (offset_bits_vert - conv_params->round_1)); - *p += sum; + if (conv_params->do_average) + *p += sum; + else + *p = sum; } else { #else { @@ -1395,7 +1398,10 @@ (1 << (offset_bits_horiz + FILTER_BITS - conv_params->round_0 - conv_params->round_1)) - (1 << (offset_bits_vert - conv_params->round_1)); - *p += sum; + if (conv_params->do_average) + *p += sum; + else + *p = sum; } else { #else {
diff --git a/av1/common/x86/convolve_2d_sse2.c b/av1/common/x86/convolve_2d_sse2.c index 46c2674..e4d352c 100644 --- a/av1/common/x86/convolve_2d_sse2.c +++ b/av1/common/x86/convolve_2d_sse2.c
@@ -31,6 +31,7 @@ int i, j; const int fo_vert = filter_params_y->taps / 2 - 1; const int fo_horiz = filter_params_x->taps / 2 - 1; + const int do_average = conv_params->do_average; const uint8_t *const src_ptr = src - fo_vert * src_stride - fo_horiz; const __m128i zero = _mm_setzero_si128(); @@ -181,9 +182,15 @@ // Accumulate values into the destination buffer __m128i *const p = (__m128i *)&dst[i * dst_stride + j]; - _mm_storeu_si128(p, _mm_add_epi32(_mm_loadu_si128(p), res_lo_round)); - _mm_storeu_si128(p + 1, - _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi_round)); + if (do_average) { + _mm_storeu_si128(p + 0, + _mm_add_epi32(_mm_loadu_si128(p + 0), res_lo_round)); + _mm_storeu_si128(p + 1, + _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi_round)); + } else { + _mm_storeu_si128(p + 0, res_lo_round); + _mm_storeu_si128(p + 1, res_hi_round); + } } } } @@ -204,6 +211,7 @@ int i, j; const int fo_vert = filter_params_y->taps / 2 - 1; const int fo_horiz = filter_params_x->taps / 2 - 1; + const int do_average = conv_params->do_average; const uint8_t *const src_ptr = src - fo_vert * src_stride - fo_horiz; const __m128i zero = _mm_setzero_si128(); @@ -357,9 +365,15 @@ // Accumulate values into the destination buffer __m128i *const p = (__m128i *)&dst[i * dst_stride + j]; - _mm_storeu_si128(p, _mm_add_epi32(_mm_loadu_si128(p), res_lo_round)); - _mm_storeu_si128(p + 1, - _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi_round)); + if (do_average) { + _mm_storeu_si128(p + 0, + _mm_add_epi32(_mm_loadu_si128(p + 0), res_lo_round)); + _mm_storeu_si128(p + 1, + _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi_round)); + } else { + _mm_storeu_si128(p + 0, res_lo_round); + _mm_storeu_si128(p + 1, res_hi_round); + } } } }
diff --git a/av1/common/x86/highbd_convolve_2d_ssse3.c b/av1/common/x86/highbd_convolve_2d_ssse3.c index ff4a0a0..195f0f5 100644 --- a/av1/common/x86/highbd_convolve_2d_ssse3.c +++ b/av1/common/x86/highbd_convolve_2d_ssse3.c
@@ -32,6 +32,7 @@ int i, j; const int fo_vert = filter_params_y->taps / 2 - 1; const int fo_horiz = filter_params_x->taps / 2 - 1; + const int do_average = conv_params->do_average; const uint16_t *const src_ptr = src - fo_vert * src_stride - fo_horiz; /* Horizontal filter */ @@ -185,9 +186,15 @@ // Accumulate values into the destination buffer __m128i *const p = (__m128i *)&dst[i * dst_stride + j]; - _mm_storeu_si128(p, _mm_add_epi32(_mm_loadu_si128(p), res_lo_round)); - _mm_storeu_si128(p + 1, - _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi_round)); + if (do_average) { + _mm_storeu_si128(p + 0, + _mm_add_epi32(_mm_loadu_si128(p + 0), res_lo_round)); + _mm_storeu_si128(p + 1, + _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi_round)); + } else { + _mm_storeu_si128(p + 0, res_lo_round); + _mm_storeu_si128(p + 1, res_hi_round); + } } } } @@ -204,6 +211,7 @@ int im_h = h + filter_params_y->taps - 1; int im_stride = MAX_SB_SIZE; int i, j; + const int do_average = conv_params->do_average; const int fo_vert = filter_params_y->taps / 2 - 1; const int fo_horiz = filter_params_x->taps / 2 - 1; const uint16_t *const src_ptr = src - fo_vert * src_stride - fo_horiz; @@ -362,9 +370,15 @@ // Accumulate values into the destination buffer __m128i *const p = (__m128i *)&dst[i * dst_stride + j]; - _mm_storeu_si128(p, _mm_add_epi32(_mm_loadu_si128(p), res_lo_round)); - _mm_storeu_si128(p + 1, - _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi_round)); + if (do_average) { + _mm_storeu_si128(p + 0, + _mm_add_epi32(_mm_loadu_si128(p + 0), res_lo_round)); + _mm_storeu_si128(p + 1, + _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi_round)); + } else { + _mm_storeu_si128(p + 0, res_lo_round); + _mm_storeu_si128(p + 1, res_hi_round); + } } } }
diff --git a/av1/common/x86/highbd_warp_plane_ssse3.c b/av1/common/x86/highbd_warp_plane_ssse3.c index 7865bc7..ca3b635 100644 --- a/av1/common/x86/highbd_warp_plane_ssse3.c +++ b/av1/common/x86/highbd_warp_plane_ssse3.c
@@ -320,21 +320,20 @@ __m128i *const p = (__m128i *)&conv_params ->dst[(i + k + 4) * conv_params->dst_stride + j]; - const __m128i orig_lo = _mm_loadu_si128(p); const __m128i round_const = _mm_set1_epi32( -(1 << (bd + 2 * FILTER_BITS - conv_params->round_0 - 1)) + ((1 << (conv_params->round_1)) >> 1)); res_lo = _mm_add_epi32(res_lo, round_const); - res_lo = _mm_add_epi32( - orig_lo, - _mm_srl_epi16(res_lo, _mm_cvtsi32_si128(conv_params->round_1))); + res_lo = + _mm_srl_epi16(res_lo, _mm_cvtsi32_si128(conv_params->round_1)); + if (comp_avg) res_lo = _mm_add_epi32(_mm_loadu_si128(p), res_lo); _mm_storeu_si128(p, res_lo); if (p_width > 4) { - const __m128i orig_hi = _mm_loadu_si128(p + 1); res_hi = _mm_add_epi32(res_hi, round_const); - res_hi = _mm_add_epi32( - orig_hi, - _mm_srl_epi16(res_hi, _mm_cvtsi32_si128(conv_params->round_1))); + res_hi = + _mm_srl_epi16(res_hi, _mm_cvtsi32_si128(conv_params->round_1)); + if (comp_avg) + res_hi = _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi); _mm_storeu_si128(p + 1, res_hi); } } else {
diff --git a/av1/common/x86/warp_plane_sse2.c b/av1/common/x86/warp_plane_sse2.c index 22958eb..a19cb6e 100644 --- a/av1/common/x86/warp_plane_sse2.c +++ b/av1/common/x86/warp_plane_sse2.c
@@ -316,21 +316,20 @@ __m128i *const p = (__m128i *)&conv_params ->dst[(i + k + 4) * conv_params->dst_stride + j]; - const __m128i orig_lo = _mm_loadu_si128(p); const __m128i round_const = _mm_set1_epi32( -(1 << (bd + 2 * FILTER_BITS - conv_params->round_0 - 1)) + ((1 << (conv_params->round_1)) >> 1)); res_lo = _mm_add_epi32(res_lo, round_const); - res_lo = _mm_add_epi32( - orig_lo, - _mm_srl_epi16(res_lo, _mm_cvtsi32_si128(conv_params->round_1))); + res_lo = + _mm_srl_epi16(res_lo, _mm_cvtsi32_si128(conv_params->round_1)); + if (comp_avg) res_lo = _mm_add_epi32(_mm_loadu_si128(p), res_lo); _mm_storeu_si128(p, res_lo); if (p_width > 4) { - const __m128i orig_hi = _mm_loadu_si128(p + 1); res_hi = _mm_add_epi32(res_hi, round_const); - res_hi = _mm_add_epi32( - orig_hi, - _mm_srl_epi16(res_hi, _mm_cvtsi32_si128(conv_params->round_1))); + res_hi = + _mm_srl_epi16(res_hi, _mm_cvtsi32_si128(conv_params->round_1)); + if (comp_avg) + res_hi = _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi); _mm_storeu_si128(p + 1, res_hi); } } else {
diff --git a/av1/common/x86/warp_plane_ssse3.c b/av1/common/x86/warp_plane_ssse3.c index 57bcc7b..475ed48 100644 --- a/av1/common/x86/warp_plane_ssse3.c +++ b/av1/common/x86/warp_plane_ssse3.c
@@ -492,21 +492,20 @@ __m128i *const p = (__m128i *)&conv_params ->dst[(i + k + 4) * conv_params->dst_stride + j]; - const __m128i orig_lo = _mm_loadu_si128(p); const __m128i round_const = _mm_set1_epi32( -(1 << (bd + 2 * FILTER_BITS - conv_params->round_0 - 1)) + ((1 << (conv_params->round_1)) >> 1)); res_lo = _mm_add_epi32(res_lo, round_const); - res_lo = _mm_add_epi32( - orig_lo, - _mm_srl_epi16(res_lo, _mm_cvtsi32_si128(conv_params->round_1))); + res_lo = + _mm_srl_epi16(res_lo, _mm_cvtsi32_si128(conv_params->round_1)); + if (comp_avg) res_lo = _mm_add_epi32(_mm_loadu_si128(p), res_lo); _mm_storeu_si128(p, res_lo); if (p_width > 4) { - const __m128i orig_hi = _mm_loadu_si128(p + 1); res_hi = _mm_add_epi32(res_hi, round_const); - res_hi = _mm_add_epi32( - orig_hi, - _mm_srl_epi16(res_hi, _mm_cvtsi32_si128(conv_params->round_1))); + res_hi = + _mm_srl_epi16(res_hi, _mm_cvtsi32_si128(conv_params->round_1)); + if (comp_avg) + res_hi = _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi); _mm_storeu_si128(p + 1, res_hi); } } else {
diff --git a/test/av1_convolve_2d_test_util.cc b/test/av1_convolve_2d_test_util.cc index 8c1a49b..3b61f6b 100644 --- a/test/av1_convolve_2d_test_util.cc +++ b/test/av1_convolve_2d_test_util.cc
@@ -56,10 +56,11 @@ av1_get_interp_filter_params((InterpFilter)hfilter); InterpFilterParams filter_params_y = av1_get_interp_filter_params((InterpFilter)vfilter); + const int do_average = rnd_.Rand8() & 1; ConvolveParams conv_params1 = - get_conv_params_no_round(0, 0, 0, output, MAX_SB_SIZE); + get_conv_params_no_round(0, do_average, 0, output, MAX_SB_SIZE); ConvolveParams conv_params2 = - get_conv_params_no_round(0, 0, 0, output2, MAX_SB_SIZE); + get_conv_params_no_round(0, do_average, 0, output2, MAX_SB_SIZE); for (subx = 0; subx < 16; ++subx) for (suby = 0; suby < 16; ++suby) {