Convolve copy function for jnt_comp Added a copy function (c version and sse2 version) for full-pixel motion vectors for jnt_comp experiment following existing av1_convolve_2d_copy function. Change-Id: I20fd2219799f9c1451f591574fbe97364f40e0f0
diff --git a/av1/common/av1_rtcd_defs.pl b/av1/common/av1_rtcd_defs.pl index cbf79b8..5ac9460 100755 --- a/av1/common/av1_rtcd_defs.pl +++ b/av1/common/av1_rtcd_defs.pl
@@ -599,6 +599,11 @@ if (aom_config("CONFIG_JNT_COMP") eq "yes") { add_proto qw/void av1_jnt_convolve_2d/, "const uint8_t *src, int src_stride, CONV_BUF_TYPE *dst, int dst_stride, int w, int h, InterpFilterParams *filter_params_x, InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params"; specialize qw/av1_jnt_convolve_2d sse4_1/; + + if (aom_config("CONFIG_COMPOUND_ROUND") ne "yes") { + add_proto qw/void av1_jnt_convolve_2d_copy/, "const uint8_t *src, int src_stride, CONV_BUF_TYPE *dst, int dst_stride, int w, int h, InterpFilterParams *filter_params_x, InterpFilterParams *filter_params_y, const int subpel_x_q4, const int subpel_y_q4, ConvolveParams *conv_params"; + specialize qw/av1_jnt_convolve_2d_copy sse4_1/; + } } if (aom_config("CONFIG_HIGHBITDEPTH") eq "yes") {
diff --git a/av1/common/convolve.c b/av1/common/convolve.c index cb11305..13b92fcb 100644 --- a/av1/common/convolve.c +++ b/av1/common/convolve.c
@@ -756,6 +756,42 @@ } } } + +void av1_jnt_convolve_2d_copy_c(const uint8_t *src, int src_stride, + CONV_BUF_TYPE *dst, int dst_stride, int w, + int h, InterpFilterParams *filter_params_x, + InterpFilterParams *filter_params_y, + const int subpel_x_q4, const int subpel_y_q4, + ConvolveParams *conv_params) { + const int bits = + FILTER_BITS * 2 - conv_params->round_1 - conv_params->round_0; + + (void)filter_params_x; + (void)filter_params_y; + (void)subpel_x_q4; + (void)subpel_y_q4; + + for (int y = 0; y < h; ++y) { + for (int x = 0; x < w; ++x) { + CONV_BUF_TYPE res = (1 << bits) * src[y * src_stride + x]; + if (conv_params->use_jnt_comp_avg) { + if (conv_params->do_average) { + dst[y * dst_stride + x] += res * conv_params->bck_offset; + + dst[y * dst_stride + x] = ROUND_POWER_OF_TWO(dst[y * dst_stride + x], + DIST_PRECISION_BITS - 1); + } else { + dst[y * dst_stride + x] = res * conv_params->fwd_offset; + } + } else { + if (conv_params->do_average) + dst[y * dst_stride + x] += res; + else + dst[y * dst_stride + x] = res; + } + } + } +} #endif // CONFIG_JNT_COMP void av1_convolve_2d_scale_c(const uint8_t *src, int src_stride, @@ -903,17 +939,44 @@ tr_dst_stride, h, w); } else { #if CONFIG_JNT_COMP - if (scaled) + if (scaled) { av1_convolve_2d_scale(src, src_stride, conv_params->dst, conv_params->dst_stride, w, h, &filter_params_x, &filter_params_y, subpel_x_q4, x_step_q4, subpel_y_q4, y_step_q4, conv_params); - else + } else { +#if CONFIG_COMPOUND_ROUND av1_jnt_convolve_2d(src, src_stride, conv_params->dst, conv_params->dst_stride, w, h, &filter_params_x, &filter_params_y, subpel_x_q4, subpel_y_q4, conv_params); #else + if (subpel_x_q4 == 0 && subpel_y_q4 == 0) { + av1_jnt_convolve_2d_copy(src, src_stride, conv_params->dst, + conv_params->dst_stride, w, h, + &filter_params_x, &filter_params_y, + subpel_x_q4, subpel_y_q4, conv_params); + } else if (subpel_x_q4 == 0) { + // place holder + av1_jnt_convolve_2d(src, src_stride, conv_params->dst, + conv_params->dst_stride, w, h, &filter_params_x, + &filter_params_y, subpel_x_q4, subpel_y_q4, + conv_params); + } else if (subpel_y_q4 == 0) { + // place holder + av1_jnt_convolve_2d(src, src_stride, conv_params->dst, + conv_params->dst_stride, w, h, &filter_params_x, + &filter_params_y, subpel_x_q4, subpel_y_q4, + conv_params); + } else { + av1_jnt_convolve_2d(src, src_stride, conv_params->dst, + conv_params->dst_stride, w, h, &filter_params_x, + &filter_params_y, subpel_x_q4, subpel_y_q4, + conv_params); + } +#endif // CONFIG_COMPOUND_ROUND + } +#else if (scaled) { av1_convolve_2d_scale(src, src_stride, conv_params->dst, conv_params->dst_stride, w, h, &filter_params_x,
diff --git a/av1/common/x86/convolve_2d_sse4.c b/av1/common/x86/convolve_2d_sse4.c index 71c32e7..893036b 100644 --- a/av1/common/x86/convolve_2d_sse4.c +++ b/av1/common/x86/convolve_2d_sse4.c
@@ -450,5 +450,194 @@ } } } + +void av1_jnt_convolve_2d_copy_sse4_1(const uint8_t *src, int src_stride, + CONV_BUF_TYPE *dst, int dst_stride, int w, + int h, InterpFilterParams *filter_params_x, + InterpFilterParams *filter_params_y, + const int subpel_x_q4, + const int subpel_y_q4, + ConvolveParams *conv_params) { + const int bits = + FILTER_BITS * 2 - conv_params->round_1 - conv_params->round_0; + const int do_average = conv_params->do_average; + const __m128i zero = _mm_setzero_si128(); + const __m128i left_shift = _mm_cvtsi32_si128(bits); + int i, j; + (void)filter_params_x; + (void)filter_params_y; + (void)subpel_x_q4; + (void)subpel_y_q4; + + const int w0 = conv_params->fwd_offset; + const int w1 = conv_params->bck_offset; + const __m128i wt0 = _mm_set1_epi32(w0); + const __m128i wt1 = _mm_set1_epi32(w1); + const int jnt_round_const = 1 << (DIST_PRECISION_BITS - 2); + const __m128i jnt_r = _mm_set1_epi32(jnt_round_const); + + if (!(w % 16)) { + for (i = 0; i < h; ++i) { + for (j = 0; j < w; j += 16) { + const __m128i d8 = _mm_loadu_si128((__m128i *)&src[j]); + const __m128i d16_0 = _mm_unpacklo_epi8(d8, zero); + const __m128i d16_1 = _mm_unpackhi_epi8(d8, zero); + __m128i d32_0 = _mm_unpacklo_epi16(d16_0, zero); + __m128i d32_1 = _mm_unpackhi_epi16(d16_0, zero); + __m128i d32_2 = _mm_unpacklo_epi16(d16_1, zero); + __m128i d32_3 = _mm_unpackhi_epi16(d16_1, zero); + + d32_0 = _mm_sll_epi32(d32_0, left_shift); + d32_1 = _mm_sll_epi32(d32_1, left_shift); + d32_2 = _mm_sll_epi32(d32_2, left_shift); + d32_3 = _mm_sll_epi32(d32_3, left_shift); + + __m128i *const p = (__m128i *)&dst[j]; + + if (conv_params->use_jnt_comp_avg) { + if (do_average) { + __m128i weighted_res = _mm_mullo_epi32(d32_0, wt1); + __m128i sum = _mm_add_epi32(_mm_loadu_si128(p + 0), weighted_res); + d32_0 = _mm_srai_epi32(_mm_add_epi32(sum, jnt_r), + DIST_PRECISION_BITS - 1); + + weighted_res = _mm_mullo_epi32(d32_1, wt1); + sum = _mm_add_epi32(_mm_loadu_si128(p + 1), weighted_res); + d32_1 = _mm_srai_epi32(_mm_add_epi32(sum, jnt_r), + DIST_PRECISION_BITS - 1); + + weighted_res = _mm_mullo_epi32(d32_2, wt1); + sum = _mm_add_epi32(_mm_loadu_si128(p + 2), weighted_res); + d32_2 = _mm_srai_epi32(_mm_add_epi32(sum, jnt_r), + DIST_PRECISION_BITS - 1); + + weighted_res = _mm_mullo_epi32(d32_3, wt1); + sum = _mm_add_epi32(_mm_loadu_si128(p + 3), weighted_res); + d32_3 = _mm_srai_epi32(_mm_add_epi32(sum, jnt_r), + DIST_PRECISION_BITS - 1); + } else { + d32_0 = _mm_mullo_epi32(d32_0, wt0); + d32_1 = _mm_mullo_epi32(d32_1, wt0); + d32_2 = _mm_mullo_epi32(d32_2, wt0); + d32_3 = _mm_mullo_epi32(d32_3, wt0); + } + } else { + if (do_average) { + d32_0 = _mm_add_epi32(_mm_loadu_si128(p + 0), d32_0); + d32_1 = _mm_add_epi32(_mm_loadu_si128(p + 1), d32_1); + d32_2 = _mm_add_epi32(_mm_loadu_si128(p + 2), d32_2); + d32_3 = _mm_add_epi32(_mm_loadu_si128(p + 3), d32_3); + } + } + + _mm_storeu_si128(p + 0, d32_0); + _mm_storeu_si128(p + 1, d32_1); + _mm_storeu_si128(p + 2, d32_2); + _mm_storeu_si128(p + 3, d32_3); + } + src += src_stride; + dst += dst_stride; + } + } else if (!(w % 8)) { + for (i = 0; i < h; ++i) { + for (j = 0; j < w; j += 8) { + const __m128i d8 = _mm_loadl_epi64((__m128i *)&src[j]); + const __m128i d16_0 = _mm_unpacklo_epi8(d8, zero); + __m128i d32_0 = _mm_unpacklo_epi16(d16_0, zero); + __m128i d32_1 = _mm_unpackhi_epi16(d16_0, zero); + + d32_0 = _mm_sll_epi32(d32_0, left_shift); + d32_1 = _mm_sll_epi32(d32_1, left_shift); + + __m128i *const p = (__m128i *)&dst[j]; + if (conv_params->use_jnt_comp_avg) { + if (do_average) { + __m128i weighted_res = _mm_mullo_epi32(d32_0, wt1); + __m128i sum = _mm_add_epi32(_mm_loadu_si128(p + 0), weighted_res); + d32_0 = _mm_srai_epi32(_mm_add_epi32(sum, jnt_r), + DIST_PRECISION_BITS - 1); + + weighted_res = _mm_mullo_epi32(d32_1, wt1); + sum = _mm_add_epi32(_mm_loadu_si128(p + 1), weighted_res); + d32_1 = _mm_srai_epi32(_mm_add_epi32(sum, jnt_r), + DIST_PRECISION_BITS - 1); + } else { + d32_0 = _mm_mullo_epi32(d32_0, wt0); + d32_1 = _mm_mullo_epi32(d32_1, wt0); + } + } else { + if (do_average) { + d32_0 = _mm_add_epi32(_mm_loadu_si128(p + 0), d32_0); + d32_1 = _mm_add_epi32(_mm_loadu_si128(p + 1), d32_1); + } + } + + _mm_storeu_si128(p + 0, d32_0); + _mm_storeu_si128(p + 1, d32_1); + } + src += src_stride; + dst += dst_stride; + } + } else if (!(w % 4)) { + for (i = 0; i < h; ++i) { + for (j = 0; j < w; j += 4) { + const __m128i d8 = _mm_loadl_epi64((__m128i *)&src[j]); + const __m128i d16_0 = _mm_unpacklo_epi8(d8, zero); + __m128i d32_0 = _mm_unpacklo_epi16(d16_0, zero); + + d32_0 = _mm_sll_epi32(d32_0, left_shift); + + __m128i *const p = (__m128i *)&dst[j]; + if (conv_params->use_jnt_comp_avg) { + if (do_average) { + __m128i weighted_res = _mm_mullo_epi32(d32_0, wt1); + __m128i sum = _mm_add_epi32(_mm_loadu_si128(p + 0), weighted_res); + d32_0 = _mm_srai_epi32(_mm_add_epi32(sum, jnt_r), + DIST_PRECISION_BITS - 1); + } else { + d32_0 = _mm_mullo_epi32(d32_0, wt0); + } + } else { + if (do_average) { + d32_0 = _mm_add_epi32(_mm_loadu_si128(p + 0), d32_0); + } + } + + _mm_storeu_si128(p, d32_0); + } + src += src_stride; + dst += dst_stride; + } + } else { + for (i = 0; i < h; ++i) { + for (j = 0; j < w; j += 2) { + const __m128i d8 = _mm_cvtsi32_si128(*(const int *)&src[j]); + const __m128i d16_0 = _mm_unpacklo_epi8(d8, zero); + __m128i d32_0 = _mm_unpacklo_epi16(d16_0, zero); + + d32_0 = _mm_sll_epi32(d32_0, left_shift); + __m128i *const p = (__m128i *)&dst[j]; + if (conv_params->use_jnt_comp_avg) { + if (do_average) { + __m128i weighted_res = _mm_mullo_epi32(d32_0, wt1); + __m128i sum = _mm_add_epi32(_mm_loadl_epi64(p), weighted_res); + d32_0 = _mm_srai_epi32(_mm_add_epi32(sum, jnt_r), + DIST_PRECISION_BITS - 1); + } else { + d32_0 = _mm_mullo_epi32(d32_0, wt0); + } + } else { + if (do_average) { + d32_0 = _mm_add_epi32(_mm_loadl_epi64(p), d32_0); + } + } + + _mm_storel_epi64(p, d32_0); + } + src += src_stride; + dst += dst_stride; + } + } +} #endif // CONFIG_COMPOUND_ROUND #endif // CONFIG_JNT_COMP