Round compound prediction into 16 bits
Apply shifts for compound prediction immediately when sum
or weighted sum (for jnt_comp) is computed. Such that all
intermediate results can be fit into 16 bits.
Note: now the buffer is still 32 bits. We need new simd functions
for 16 bits and finally reduce buffer to 16 bits.
Change-Id: Ia46a4736d69aa028623dfb9f036a6ce527e5cd9f
diff --git a/aom_dsp/x86/convolve_avx2.h b/aom_dsp/x86/convolve_avx2.h
index ec5868e..3ca424c 100644
--- a/aom_dsp/x86/convolve_avx2.h
+++ b/aom_dsp/x86/convolve_avx2.h
@@ -122,11 +122,12 @@
static INLINE void add_store_aligned(CONV_BUF_TYPE *const dst,
const __m256i *const res,
- const __m256i *const avg_mask) {
+ const __m256i *const avg_mask, int shift) {
__m256i d;
d = _mm256_load_si256((__m256i *)dst);
d = _mm256_and_si256(d, *avg_mask);
d = _mm256_add_epi32(d, *res);
+ if (shift) d = _mm256_srai_epi32(d, 1);
_mm256_store_si256((__m256i *)dst, d);
}
diff --git a/av1/common/convolve.c b/av1/common/convolve.c
index fef179c..ba9b96f 100644
--- a/av1/common/convolve.c
+++ b/av1/common/convolve.c
@@ -424,10 +424,13 @@
CONV_BUF_TYPE res = ROUND_POWER_OF_TWO(sum, conv_params->round_1) -
((1 << (offset_bits - conv_params->round_1)) +
(1 << (offset_bits - conv_params->round_1 - 1)));
- if (conv_params->do_average)
- dst[y * dst_stride + x] += res;
- else
+ if (conv_params->do_average) {
+ int32_t tmp = dst[y * dst_stride + x];
+ tmp += res;
+ dst[y * dst_stride + x] = tmp >> 1;
+ } else {
dst[y * dst_stride + x] = res;
+ }
}
}
}
@@ -460,10 +463,13 @@
}
res *= (1 << bits);
res = ROUND_POWER_OF_TWO(res, conv_params->round_1);
- if (conv_params->do_average)
- dst[y * dst_stride + x] += res;
- else
+ if (conv_params->do_average) {
+ int32_t tmp = dst[y * dst_stride + x];
+ tmp += res;
+ dst[y * dst_stride + x] = tmp >> 1;
+ } else {
dst[y * dst_stride + x] = res;
+ }
}
}
}
@@ -495,10 +501,13 @@
res += x_filter[k] * src[y * src_stride + x - fo_horiz + k];
}
res = (1 << bits) * ROUND_POWER_OF_TWO(res, conv_params->round_0);
- if (conv_params->do_average)
- dst[y * dst_stride + x] += res;
- else
+ if (conv_params->do_average) {
+ int32_t tmp = dst[y * dst_stride + x];
+ tmp += res;
+ dst[y * dst_stride + x] = tmp >> 1;
+ } else {
dst[y * dst_stride + x] = res;
+ }
}
}
}
@@ -524,10 +533,13 @@
for (int y = 0; y < h; ++y) {
for (int x = 0; x < w; ++x) {
CONV_BUF_TYPE res = src[y * src_stride + x] << bits;
- if (conv_params->do_average)
- dst[y * dst_stride + x] += res;
- else
+ if (conv_params->do_average) {
+ int32_t tmp = dst[y * dst_stride + x];
+ tmp += res;
+ dst[y * dst_stride + x] = tmp >> 1;
+ } else {
dst[y * dst_stride + x] = res;
+ }
}
}
}
@@ -714,15 +726,20 @@
(1 << (offset_bits - conv_params->round_1 - 1)));
if (conv_params->use_jnt_comp_avg) {
if (conv_params->do_average) {
- dst[y * dst_stride + x] += res * conv_params->bck_offset;
+ int32_t tmp = dst[y * dst_stride + x];
+ tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset;
+ dst[y * dst_stride + x] = tmp >> DIST_PRECISION_BITS;
} else {
- dst[y * dst_stride + x] = res * conv_params->fwd_offset;
+ dst[y * dst_stride + x] = res;
}
} else {
- if (conv_params->do_average)
- dst[y * dst_stride + x] += res;
- else
+ if (conv_params->do_average) {
+ int32_t tmp = dst[y * dst_stride + x];
+ tmp += res;
+ dst[y * dst_stride + x] = tmp >> 1;
+ } else {
dst[y * dst_stride + x] = res;
+ }
}
}
}
@@ -756,15 +773,20 @@
res = ROUND_POWER_OF_TWO(res, conv_params->round_1);
if (conv_params->use_jnt_comp_avg) {
if (conv_params->do_average) {
- dst[y * dst_stride + x] += res * conv_params->bck_offset;
+ int32_t tmp = dst[y * dst_stride + x];
+ tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset;
+ dst[y * dst_stride + x] = tmp >> DIST_PRECISION_BITS;
} else {
- dst[y * dst_stride + x] = res * conv_params->fwd_offset;
+ dst[y * dst_stride + x] = res;
}
} else {
- if (conv_params->do_average)
- dst[y * dst_stride + x] += res;
- else
+ if (conv_params->do_average) {
+ int32_t tmp = dst[y * dst_stride + x];
+ tmp += res;
+ dst[y * dst_stride + x] = tmp >> 1;
+ } else {
dst[y * dst_stride + x] = res;
+ }
}
}
}
@@ -797,15 +819,20 @@
res = (1 << bits) * ROUND_POWER_OF_TWO(res, conv_params->round_0);
if (conv_params->use_jnt_comp_avg) {
if (conv_params->do_average) {
- dst[y * dst_stride + x] += res * conv_params->bck_offset;
+ int32_t tmp = dst[y * dst_stride + x];
+ tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset;
+ dst[y * dst_stride + x] = tmp >> DIST_PRECISION_BITS;
} else {
- dst[y * dst_stride + x] = res * conv_params->fwd_offset;
+ dst[y * dst_stride + x] = res;
}
} else {
- if (conv_params->do_average)
- dst[y * dst_stride + x] += res;
- else
+ if (conv_params->do_average) {
+ int32_t tmp = dst[y * dst_stride + x];
+ tmp += res;
+ dst[y * dst_stride + x] = tmp >> 1;
+ } else {
dst[y * dst_stride + x] = res;
+ }
}
}
}
@@ -834,15 +861,20 @@
CONV_BUF_TYPE res = src[y * src_stride + x] << bits;
if (conv_params->use_jnt_comp_avg) {
if (conv_params->do_average) {
- dst[y * dst_stride + x] += res * conv_params->bck_offset;
+ int32_t tmp = dst[y * dst_stride + x];
+ tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset;
+ dst[y * dst_stride + x] = tmp >> DIST_PRECISION_BITS;
} else {
- dst[y * dst_stride + x] = res * conv_params->fwd_offset;
+ dst[y * dst_stride + x] = res;
}
} else {
- if (conv_params->do_average)
- dst[y * dst_stride + x] += res;
- else
+ if (conv_params->do_average) {
+ int32_t tmp = dst[y * dst_stride + x];
+ tmp += res;
+ dst[y * dst_stride + x] = tmp >> 1;
+ } else {
dst[y * dst_stride + x] = res;
+ }
}
}
}
@@ -907,21 +939,29 @@
#if CONFIG_JNT_COMP
if (conv_params->use_jnt_comp_avg) {
if (conv_params->do_average) {
- dst[y * dst_stride + x] += res * conv_params->bck_offset;
+ int32_t tmp = dst[y * dst_stride + x];
+ tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset;
+ dst[y * dst_stride + x] = tmp >> DIST_PRECISION_BITS;
} else {
- dst[y * dst_stride + x] = res * conv_params->fwd_offset;
+ dst[y * dst_stride + x] = res;
}
} else {
- if (conv_params->do_average)
- dst[y * dst_stride + x] += res;
- else
+ if (conv_params->do_average) {
+ int32_t tmp = dst[y * dst_stride + x];
+ tmp += res;
+ dst[y * dst_stride + x] = tmp >> 1;
+ } else {
dst[y * dst_stride + x] = res;
+ }
}
#else
- if (conv_params->do_average)
- dst[y * dst_stride + x] += res;
- else
+ if (conv_params->do_average) {
+ int32_t tmp = dst[y * dst_stride + x];
+ tmp += res;
+ dst[y * dst_stride + x] = tmp >> 1;
+ } else {
dst[y * dst_stride + x] = res;
+ }
#endif // CONFIG_JNT_COMP
}
src_vert++;
@@ -1044,10 +1084,13 @@
CONV_BUF_TYPE res = ROUND_POWER_OF_TWO(sum, conv_params->round_1) -
((1 << (offset_bits - conv_params->round_1)) +
(1 << (offset_bits - conv_params->round_1 - 1)));
- if (conv_params->do_average)
- dst[y * dst_stride + x] += res;
- else
+ if (conv_params->do_average) {
+ int32_t tmp = dst[y * dst_stride + x];
+ tmp += res;
+ dst[y * dst_stride + x] = tmp >> 1;
+ } else {
dst[y * dst_stride + x] = res;
+ }
}
}
}
@@ -1339,15 +1382,20 @@
(1 << (offset_bits - conv_params->round_1 - 1)));
if (conv_params->use_jnt_comp_avg) {
if (conv_params->do_average) {
- dst[y * dst_stride + x] += res * conv_params->bck_offset;
+ int32_t tmp = dst[y * dst_stride + x];
+ tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset;
+ dst[y * dst_stride + x] = tmp >> DIST_PRECISION_BITS;
} else {
- dst[y * dst_stride + x] = res * conv_params->fwd_offset;
+ dst[y * dst_stride + x] = res;
}
} else {
- if (conv_params->do_average)
- dst[y * dst_stride + x] += res;
- else
+ if (conv_params->do_average) {
+ int32_t tmp = dst[y * dst_stride + x];
+ tmp += res;
+ dst[y * dst_stride + x] = tmp >> 1;
+ } else {
dst[y * dst_stride + x] = res;
+ }
}
}
}
@@ -1533,21 +1581,29 @@
#if CONFIG_JNT_COMP
if (conv_params->use_jnt_comp_avg) {
if (conv_params->do_average) {
- dst[y * dst_stride + x] += res * conv_params->bck_offset;
+ int32_t tmp = dst[y * dst_stride + x];
+ tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset;
+ dst[y * dst_stride + x] = tmp >> DIST_PRECISION_BITS;
} else {
- dst[y * dst_stride + x] = res * conv_params->fwd_offset;
+ dst[y * dst_stride + x] = res;
}
} else {
- if (conv_params->do_average)
- dst[y * dst_stride + x] += res;
- else
+ if (conv_params->do_average) {
+ int32_t tmp = dst[y * dst_stride + x];
+ tmp += res;
+ dst[y * dst_stride + x] = tmp >> 1;
+ } else {
dst[y * dst_stride + x] = res;
+ }
}
#else
- if (conv_params->do_average)
- dst[y * dst_stride + x] += res;
- else
+ if (conv_params->do_average) {
+ int32_t tmp = dst[y * dst_stride + x];
+ tmp += res;
+ dst[y * dst_stride + x] = tmp >> 1;
+ } else {
dst[y * dst_stride + x] = res;
+ }
#endif // CONFIG_JNT_COMP
}
src_vert++;
diff --git a/av1/common/reconinter.c b/av1/common/reconinter.c
index 18e316a..d975aa8 100644
--- a/av1/common/reconinter.c
+++ b/av1/common/reconinter.c
@@ -30,17 +30,9 @@
// prediction.
static INLINE int get_compound_post_rounding_bits(
- const MB_MODE_INFO *const mbmi, const ConvolveParams *conv_params) {
+ const ConvolveParams *conv_params) {
assert(conv_params->is_compound);
- int round_bits =
- 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
- if (is_masked_compound_type(mbmi->interinter_compound_type))
- return round_bits;
- round_bits += conv_params->is_compound;
-#if CONFIG_JNT_COMP
- if (conv_params->use_jnt_comp_avg) round_bits += DIST_PRECISION_BITS - 1;
-#endif // CONFIG_JNT_COMP
- return round_bits;
+ return 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1;
}
static INLINE int allow_warp(const MODE_INFO *const mi,
@@ -1105,8 +1097,7 @@
if (conv_params.is_compound) {
assert(conv_params.dst != NULL);
- int round_bits =
- get_compound_post_rounding_bits(&mi->mbmi, &conv_params);
+ int round_bits = get_compound_post_rounding_bits(&conv_params);
if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH)
av1_highbd_convolve_rounding(tmp_dst, tmp_dst_stride, dst,
dst_buf->stride, b4_w, b4_h,
@@ -1242,7 +1233,7 @@
// TODO(angiebird): This part needs optimization
if (conv_params.is_compound) {
assert(conv_params.dst != NULL);
- int round_bits = get_compound_post_rounding_bits(&mi->mbmi, &conv_params);
+ int round_bits = get_compound_post_rounding_bits(&conv_params);
if (xd->cur_buf->flags & YV12_FLAG_HIGHBITDEPTH)
av1_highbd_convolve_rounding(tmp_dst, MAX_SB_SIZE, dst, dst_buf->stride,
w, h, round_bits, xd->bd);
diff --git a/av1/common/warped_motion.c b/av1/common/warped_motion.c
index fc4d8da..fa6f7ec 100644
--- a/av1/common/warped_motion.c
+++ b/av1/common/warped_motion.c
@@ -508,21 +508,30 @@
#if CONFIG_JNT_COMP
if (conv_params->use_jnt_comp_avg) {
if (conv_params->do_average) {
- *p += sum * conv_params->bck_offset;
+ int32_t tmp32 = *p;
+ tmp32 = tmp32 * conv_params->fwd_offset +
+ sum * conv_params->bck_offset;
+ *p = tmp32 >> DIST_PRECISION_BITS;
} else {
- *p = sum * conv_params->fwd_offset;
+ *p = sum;
}
} else {
- if (conv_params->do_average)
- *p += sum;
- else
+ if (conv_params->do_average) {
+ int32_t tmp32 = *p;
+ tmp32 += sum;
+ *p = tmp32 >> 1;
+ } else {
*p = sum;
+ }
}
#else
- if (conv_params->do_average)
- *p += sum;
- else
+ if (conv_params->do_average) {
+ int32_t tmp32 = *p;
+ tmp32 += sum;
+ *p = tmp32 >> 1;
+ } else {
*p = sum;
+ }
#endif // CONFIG_JNT_COMP
} else {
uint16_t *p =
@@ -802,21 +811,30 @@
#if CONFIG_JNT_COMP
if (conv_params->use_jnt_comp_avg) {
if (conv_params->do_average) {
- *p += sum * conv_params->bck_offset;
+ int32_t tmp32 = *p;
+ tmp32 = tmp32 * conv_params->fwd_offset +
+ sum * conv_params->bck_offset;
+ *p = tmp32 >> DIST_PRECISION_BITS;
} else {
- *p = sum * conv_params->fwd_offset;
+ *p = sum;
}
} else {
- if (conv_params->do_average)
- *p += sum;
- else
+ if (conv_params->do_average) {
+ int32_t tmp32 = *p;
+ tmp32 += sum;
+ *p = tmp32 >> 1;
+ } else {
*p = sum;
+ }
}
#else
- if (conv_params->do_average)
- *p += sum;
- else
+ if (conv_params->do_average) {
+ int32_t tmp32 = *p;
+ tmp32 += sum;
+ *p = tmp32 >> 1;
+ } else {
*p = sum;
+ }
#endif // CONFIG_JNT_COMP
} else {
uint8_t *p =
diff --git a/av1/common/x86/av1_convolve_scale_sse4.c b/av1/common/x86/av1_convolve_scale_sse4.c
index e53717d..973dbe3 100644
--- a/av1/common/x86/av1_convolve_scale_sse4.c
+++ b/av1/common/x86/av1_convolve_scale_sse4.c
@@ -314,20 +314,26 @@
__m128i result;
if (conv_params->use_jnt_comp_avg) {
if (conv_params->do_average) {
- result = _mm_add_epi32(_mm_loadu_si128((__m128i *)dst_x),
- _mm_mullo_epi32(subbed, bck_offset));
+ __m128i tmp = _mm_loadu_si128((__m128i *)dst_x);
+ tmp = _mm_add_epi32(_mm_mullo_epi32(tmp, fwd_offset),
+ _mm_mullo_epi32(subbed, bck_offset));
+ result = _mm_srai_epi32(tmp, DIST_PRECISION_BITS);
} else {
- result = _mm_mullo_epi32(subbed, fwd_offset);
+ result = subbed;
}
} else {
- result = (conv_params->do_average)
- ? _mm_add_epi32(subbed, _mm_loadu_si128((__m128i *)dst_x))
- : subbed;
+ result =
+ (conv_params->do_average)
+ ? _mm_srai_epi32(
+ _mm_add_epi32(subbed, _mm_loadu_si128((__m128i *)dst_x)),
+ 1)
+ : subbed;
}
#else
const __m128i result =
(conv_params->do_average)
- ? _mm_add_epi32(subbed, _mm_loadu_si128((__m128i *)dst_x))
+ ? _mm_srai_epi32(
+ _mm_add_epi32(subbed, _mm_loadu_si128((__m128i *)dst_x)), 1)
: subbed;
#endif // CONFIG_JNT_COMP
@@ -341,16 +347,21 @@
#if CONFIG_JNT_COMP
if (conv_params->use_jnt_comp_avg) {
if (conv_params->do_average) {
- dst[y * dst_stride + x] += res * conv_params->bck_offset;
+ int32_t tmp = dst[y * dst_stride + x];
+ tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset;
+ dst[y * dst_stride + x] = tmp >> DIST_PRECISION_BITS;
} else {
- dst[y * dst_stride + x] = res * conv_params->fwd_offset;
+ dst[y * dst_stride + x] = res;
}
} else {
#endif // CONFIG_JNT_COMP
- if (conv_params->do_average)
- dst[y * dst_stride + x] += res;
- else
+ if (conv_params->do_average) {
+ int32_t tmp = dst[y * dst_stride + x];
+ tmp += res;
+ dst[y * dst_stride + x] = tmp >> 1;
+ } else {
dst[y * dst_stride + x] = res;
+ }
#if CONFIG_JNT_COMP
}
#endif // CONFIG_JNT_COMP
@@ -426,20 +437,26 @@
__m128i result;
if (conv_params->use_jnt_comp_avg) {
if (conv_params->do_average) {
- result = _mm_add_epi32(_mm_loadu_si128((__m128i *)dst_x),
- _mm_mullo_epi32(subbed, bck_offset));
+ __m128i tmp = _mm_loadu_si128((__m128i *)dst_x);
+ tmp = _mm_add_epi32(_mm_mullo_epi32(tmp, fwd_offset),
+ _mm_mullo_epi32(subbed, bck_offset));
+ result = _mm_srai_epi32(tmp, DIST_PRECISION_BITS);
} else {
- result = _mm_mullo_epi32(subbed, fwd_offset);
+ result = subbed;
}
} else {
- result = (conv_params->do_average)
- ? _mm_add_epi32(subbed, _mm_loadu_si128((__m128i *)dst_x))
- : subbed;
+ result =
+ (conv_params->do_average)
+ ? _mm_srai_epi32(
+ _mm_add_epi32(subbed, _mm_loadu_si128((__m128i *)dst_x)),
+ 1)
+ : subbed;
}
#else
const __m128i result =
(conv_params->do_average)
- ? _mm_add_epi32(subbed, _mm_loadu_si128((__m128i *)dst_x))
+ ? _mm_srai_epi32(
+ _mm_add_epi32(subbed, _mm_loadu_si128((__m128i *)dst_x)), 1)
: subbed;
#endif // CONFIG_JNT_COMP
@@ -453,16 +470,21 @@
#if CONFIG_JNT_COMP
if (conv_params->use_jnt_comp_avg) {
if (conv_params->do_average) {
- dst[y * dst_stride + x] += res * conv_params->bck_offset;
+ int32_t tmp = dst[y * dst_stride + x];
+ tmp = tmp * conv_params->fwd_offset + res * conv_params->bck_offset;
+ dst[y * dst_stride + x] = tmp >> DIST_PRECISION_BITS;
} else {
- dst[y * dst_stride + x] = res * conv_params->fwd_offset;
+ dst[y * dst_stride + x] = res;
}
} else {
#endif // CONFIG_JNT_COMP
- if (conv_params->do_average)
- dst[y * dst_stride + x] += res;
- else
+ if (conv_params->do_average) {
+ int32_t tmp = dst[y * dst_stride + x];
+ tmp += res;
+ dst[y * dst_stride + x] = tmp >> 1;
+ } else {
dst[y * dst_stride + x] = res;
+ }
#if CONFIG_JNT_COMP
}
#endif // CONFIG_JNT_COMP
diff --git a/av1/common/x86/convolve_2d_avx2.c b/av1/common/x86/convolve_2d_avx2.c
index fafe344..6407c3a 100644
--- a/av1/common/x86/convolve_2d_avx2.c
+++ b/av1/common/x86/convolve_2d_avx2.c
@@ -126,9 +126,10 @@
const __m256i res_bx =
_mm256_permute2x128_si256(res_a_round, res_b_round, 0x31);
- add_store_aligned(&dst[i * dst_stride + j], &res_ax, &avg_mask);
+ add_store_aligned(&dst[i * dst_stride + j], &res_ax, &avg_mask,
+ conv_params->do_average);
add_store_aligned(&dst[i * dst_stride + j + dst_stride], &res_bx,
- &avg_mask);
+ &avg_mask, conv_params->do_average);
} else {
const __m128i res_ax = _mm256_extracti128_si256(res_a_round, 0);
const __m128i res_bx = _mm256_extracti128_si256(res_a_round, 1);
@@ -140,6 +141,10 @@
r1 = _mm_and_si128(r1, _mm256_extracti128_si256(avg_mask, 0));
r0 = _mm_add_epi32(r0, res_ax);
r1 = _mm_add_epi32(r1, res_bx);
+ if (conv_params->do_average) {
+ r0 = _mm_srai_epi32(r0, 1);
+ r1 = _mm_srai_epi32(r1, 1);
+ }
_mm_store_si128((__m128i *)&dst[i * dst_stride + j], r0);
_mm_store_si128((__m128i *)&dst[i * dst_stride + j + dst_stride], r1);
}
diff --git a/av1/common/x86/convolve_2d_sse2.c b/av1/common/x86/convolve_2d_sse2.c
index 96a6042..941d195 100644
--- a/av1/common/x86/convolve_2d_sse2.c
+++ b/av1/common/x86/convolve_2d_sse2.c
@@ -193,10 +193,14 @@
// Accumulate values into the destination buffer
__m128i *const p = (__m128i *)&dst[i * dst_stride + j];
if (do_average) {
- _mm_storeu_si128(p + 0,
- _mm_add_epi32(_mm_loadu_si128(p + 0), res_lo_round));
- _mm_storeu_si128(p + 1,
- _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi_round));
+ _mm_storeu_si128(
+ p + 0,
+ _mm_srai_epi32(
+ _mm_add_epi32(_mm_loadu_si128(p + 0), res_lo_round), 1));
+ _mm_storeu_si128(
+ p + 1,
+ _mm_srai_epi32(
+ _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi_round), 1));
} else {
_mm_storeu_si128(p + 0, res_lo_round);
_mm_storeu_si128(p + 1, res_hi_round);
@@ -444,10 +448,18 @@
__m128i *const p = (__m128i *)&dst[j];
if (do_average) {
- _mm_storeu_si128(p + 0, _mm_add_epi32(_mm_loadu_si128(p + 0), d32_0));
- _mm_storeu_si128(p + 1, _mm_add_epi32(_mm_loadu_si128(p + 1), d32_1));
- _mm_storeu_si128(p + 2, _mm_add_epi32(_mm_loadu_si128(p + 2), d32_2));
- _mm_storeu_si128(p + 3, _mm_add_epi32(_mm_loadu_si128(p + 3), d32_3));
+ _mm_storeu_si128(
+ p + 0,
+ _mm_srai_epi32(_mm_add_epi32(_mm_loadu_si128(p + 0), d32_0), 1));
+ _mm_storeu_si128(
+ p + 1,
+ _mm_srai_epi32(_mm_add_epi32(_mm_loadu_si128(p + 1), d32_1), 1));
+ _mm_storeu_si128(
+ p + 2,
+ _mm_srai_epi32(_mm_add_epi32(_mm_loadu_si128(p + 2), d32_2), 1));
+ _mm_storeu_si128(
+ p + 3,
+ _mm_srai_epi32(_mm_add_epi32(_mm_loadu_si128(p + 3), d32_3), 1));
} else {
_mm_storeu_si128(p + 0, d32_0);
_mm_storeu_si128(p + 1, d32_1);
@@ -471,8 +483,12 @@
__m128i *const p = (__m128i *)&dst[j];
if (do_average) {
- _mm_storeu_si128(p + 0, _mm_add_epi32(_mm_loadu_si128(p + 0), d32_0));
- _mm_storeu_si128(p + 1, _mm_add_epi32(_mm_loadu_si128(p + 1), d32_1));
+ _mm_storeu_si128(
+ p + 0,
+ _mm_srai_epi32(_mm_add_epi32(_mm_loadu_si128(p + 0), d32_0), 1));
+ _mm_storeu_si128(
+ p + 1,
+ _mm_srai_epi32(_mm_add_epi32(_mm_loadu_si128(p + 1), d32_1), 1));
} else {
_mm_storeu_si128(p + 0, d32_0);
_mm_storeu_si128(p + 1, d32_1);
@@ -491,7 +507,8 @@
d32_0 = _mm_sll_epi32(d32_0, left_shift);
__m128i *const p = (__m128i *)&dst[j];
if (do_average) {
- _mm_storeu_si128(p, _mm_add_epi32(_mm_loadu_si128(p), d32_0));
+ _mm_storeu_si128(
+ p, _mm_srai_epi32(_mm_add_epi32(_mm_loadu_si128(p), d32_0), 1));
} else {
_mm_storeu_si128(p, d32_0);
}
@@ -509,7 +526,8 @@
d32_0 = _mm_sll_epi32(d32_0, left_shift);
__m128i *const p = (__m128i *)&dst[j];
if (do_average) {
- _mm_storel_epi64(p, _mm_add_epi32(_mm_loadl_epi64(p), d32_0));
+ _mm_storel_epi64(
+ p, _mm_srai_epi32(_mm_add_epi32(_mm_loadl_epi64(p), d32_0), 1));
} else {
_mm_storel_epi64(p, d32_0);
}
@@ -707,39 +725,52 @@
if (do_average) {
__m128i mul = _mm_mullo_epi16(d32_0, wt1);
__m128i weighted_res = _mm_sll_epi32(mul, left_shift);
- __m128i sum = _mm_add_epi32(_mm_loadu_si128(p + 0), weighted_res);
- d32_0 = sum;
+ __m128i tmp = _mm_loadu_si128(p + 0);
+ __m128i sum =
+ _mm_add_epi32(_mm_mullo_epi16(tmp, wt0), weighted_res);
+ d32_0 = _mm_srai_epi32(sum, DIST_PRECISION_BITS);
mul = _mm_mullo_epi16(d32_1, wt1);
weighted_res = _mm_sll_epi32(mul, left_shift);
- sum = _mm_add_epi32(_mm_loadu_si128(p + 1), weighted_res);
- d32_1 = sum;
+ tmp = _mm_loadu_si128(p + 1);
+ sum = _mm_add_epi32(_mm_mullo_epi16(tmp, wt0), weighted_res);
+ d32_1 = _mm_srai_epi32(sum, DIST_PRECISION_BITS);
mul = _mm_mullo_epi16(d32_2, wt1);
weighted_res = _mm_sll_epi32(mul, left_shift);
- sum = _mm_add_epi32(_mm_loadu_si128(p + 2), weighted_res);
- d32_2 = sum;
+ tmp = _mm_loadu_si128(p + 2);
+ sum = _mm_add_epi32(_mm_mullo_epi16(tmp, wt0), weighted_res);
+ d32_2 = _mm_srai_epi32(sum, DIST_PRECISION_BITS);
mul = _mm_mullo_epi16(d32_3, wt1);
weighted_res = _mm_sll_epi32(mul, left_shift);
- sum = _mm_add_epi32(_mm_loadu_si128(p + 3), weighted_res);
- d32_3 = sum;
+ tmp = _mm_loadu_si128(p + 3);
+ sum = _mm_add_epi32(_mm_mullo_epi16(tmp, wt0), weighted_res);
+ d32_3 = _mm_srai_epi32(sum, DIST_PRECISION_BITS);
} else {
- d32_0 = _mm_sll_epi32(_mm_mullo_epi16(d32_0, wt0), left_shift);
- d32_1 = _mm_sll_epi32(_mm_mullo_epi16(d32_1, wt0), left_shift);
- d32_2 = _mm_sll_epi32(_mm_mullo_epi16(d32_2, wt0), left_shift);
- d32_3 = _mm_sll_epi32(_mm_mullo_epi16(d32_3, wt0), left_shift);
+ d32_0 = _mm_sll_epi32(d32_0, left_shift);
+ d32_1 = _mm_sll_epi32(d32_1, left_shift);
+ d32_2 = _mm_sll_epi32(d32_2, left_shift);
+ d32_3 = _mm_sll_epi32(d32_3, left_shift);
}
} else {
if (do_average) {
- d32_0 = _mm_add_epi32(_mm_loadu_si128(p + 0),
- _mm_sll_epi32(d32_0, left_shift));
- d32_1 = _mm_add_epi32(_mm_loadu_si128(p + 1),
- _mm_sll_epi32(d32_1, left_shift));
- d32_2 = _mm_add_epi32(_mm_loadu_si128(p + 2),
- _mm_sll_epi32(d32_2, left_shift));
- d32_3 = _mm_add_epi32(_mm_loadu_si128(p + 3),
- _mm_sll_epi32(d32_3, left_shift));
+ d32_0 =
+ _mm_srai_epi32(_mm_add_epi32(_mm_loadu_si128(p + 0),
+ _mm_sll_epi32(d32_0, left_shift)),
+ 1);
+ d32_1 =
+ _mm_srai_epi32(_mm_add_epi32(_mm_loadu_si128(p + 1),
+ _mm_sll_epi32(d32_1, left_shift)),
+ 1);
+ d32_2 =
+ _mm_srai_epi32(_mm_add_epi32(_mm_loadu_si128(p + 2),
+ _mm_sll_epi32(d32_2, left_shift)),
+ 1);
+ d32_3 =
+ _mm_srai_epi32(_mm_add_epi32(_mm_loadu_si128(p + 3),
+ _mm_sll_epi32(d32_3, left_shift)),
+ 1);
} else {
d32_0 = _mm_sll_epi32(d32_0, left_shift);
d32_1 = _mm_sll_epi32(d32_1, left_shift);
@@ -769,23 +800,30 @@
if (do_average) {
__m128i mul = _mm_mullo_epi16(d32_0, wt1);
__m128i weighted_res = _mm_sll_epi32(mul, left_shift);
- __m128i sum = _mm_add_epi32(_mm_loadu_si128(p + 0), weighted_res);
- d32_0 = sum;
+ __m128i tmp = _mm_loadu_si128(p + 0);
+ __m128i sum =
+ _mm_add_epi32(_mm_mullo_epi16(tmp, wt0), weighted_res);
+ d32_0 = _mm_srai_epi32(sum, DIST_PRECISION_BITS);
mul = _mm_mullo_epi16(d32_1, wt1);
weighted_res = _mm_sll_epi32(mul, left_shift);
- sum = _mm_add_epi32(_mm_loadu_si128(p + 1), weighted_res);
- d32_1 = sum;
+ tmp = _mm_loadu_si128(p + 1);
+ sum = _mm_add_epi32(_mm_mullo_epi16(tmp, wt0), weighted_res);
+ d32_1 = _mm_srai_epi32(sum, DIST_PRECISION_BITS);
} else {
- d32_0 = _mm_sll_epi32(_mm_mullo_epi16(d32_0, wt0), left_shift);
- d32_1 = _mm_sll_epi32(_mm_mullo_epi16(d32_1, wt0), left_shift);
+ d32_0 = _mm_sll_epi32(d32_0, left_shift);
+ d32_1 = _mm_sll_epi32(d32_1, left_shift);
}
} else {
if (do_average) {
- d32_0 = _mm_add_epi32(_mm_loadu_si128(p + 0),
- _mm_sll_epi32(d32_0, left_shift));
- d32_1 = _mm_add_epi32(_mm_loadu_si128(p + 1),
- _mm_sll_epi32(d32_1, left_shift));
+ d32_0 =
+ _mm_srai_epi32(_mm_add_epi32(_mm_loadu_si128(p + 0),
+ _mm_sll_epi32(d32_0, left_shift)),
+ 1);
+ d32_1 =
+ _mm_srai_epi32(_mm_add_epi32(_mm_loadu_si128(p + 1),
+ _mm_sll_epi32(d32_1, left_shift)),
+ 1);
} else {
d32_0 = _mm_sll_epi32(d32_0, left_shift);
d32_1 = _mm_sll_epi32(d32_1, left_shift);
@@ -810,15 +848,19 @@
if (do_average) {
__m128i mul = _mm_mullo_epi16(d32_0, wt1);
__m128i weighted_res = _mm_sll_epi32(mul, left_shift);
- __m128i sum = _mm_add_epi32(_mm_loadu_si128(p + 0), weighted_res);
- d32_0 = sum;
+ __m128i tmp = _mm_loadu_si128(p + 0);
+ __m128i sum =
+ _mm_add_epi32(_mm_mullo_epi16(tmp, wt0), weighted_res);
+ d32_0 = _mm_srai_epi32(sum, DIST_PRECISION_BITS);
} else {
- d32_0 = _mm_sll_epi32(_mm_mullo_epi16(d32_0, wt0), left_shift);
+ d32_0 = _mm_sll_epi32(d32_0, left_shift);
}
} else {
if (do_average) {
- d32_0 = _mm_add_epi32(_mm_loadu_si128(p + 0),
- _mm_sll_epi32(d32_0, left_shift));
+ d32_0 =
+ _mm_srai_epi32(_mm_add_epi32(_mm_loadu_si128(p + 0),
+ _mm_sll_epi32(d32_0, left_shift)),
+ 1);
} else {
d32_0 = _mm_sll_epi32(d32_0, left_shift);
}
@@ -841,15 +883,19 @@
if (do_average) {
__m128i mul = _mm_mullo_epi16(d32_0, wt1);
__m128i weighted_res = _mm_sll_epi32(mul, left_shift);
- __m128i sum = _mm_add_epi32(_mm_loadl_epi64(p), weighted_res);
- d32_0 = sum;
+ __m128i tmp = _mm_loadl_epi64(p);
+ __m128i sum =
+ _mm_add_epi32(_mm_mullo_epi16(tmp, wt0), weighted_res);
+ d32_0 = _mm_srai_epi32(sum, DIST_PRECISION_BITS);
} else {
- d32_0 = _mm_sll_epi32(_mm_mullo_epi16(d32_0, wt0), left_shift);
+ d32_0 = _mm_sll_epi32(d32_0, left_shift);
}
} else {
if (do_average) {
- d32_0 = _mm_add_epi32(_mm_loadl_epi64(p),
- _mm_sll_epi32(d32_0, left_shift));
+ d32_0 =
+ _mm_srai_epi32(_mm_add_epi32(_mm_loadl_epi64(p),
+ _mm_sll_epi32(d32_0, left_shift)),
+ 1);
} else {
d32_0 = _mm_sll_epi32(d32_0, left_shift);
}
diff --git a/av1/common/x86/convolve_avx2.c b/av1/common/x86/convolve_avx2.c
index 63f6138..ffc870b 100644
--- a/av1/common/x86/convolve_avx2.c
+++ b/av1/common/x86/convolve_avx2.c
@@ -452,7 +452,8 @@
_mm256_add_epi32(res_lo_0_shift, round_const), round_shift);
// Accumulate values into the destination buffer
- add_store_aligned(&dst[i * dst_stride + j], &res_lo_0_round, &avg_mask);
+ add_store_aligned(&dst[i * dst_stride + j], &res_lo_0_round, &avg_mask,
+ conv_params->do_average);
const __m256i res_lo_1_32b =
_mm256_cvtepi16_epi32(_mm256_extracti128_si256(res_lo, 1));
@@ -462,7 +463,7 @@
_mm256_add_epi32(res_lo_1_shift, round_const), round_shift);
add_store_aligned(&dst[i * dst_stride + j + dst_stride], &res_lo_1_round,
- &avg_mask);
+ &avg_mask, conv_params->do_average);
if (w - j > 8) {
const __m256i res_hi = convolve_lowbd(s + 4, coeffs);
@@ -475,7 +476,7 @@
_mm256_add_epi32(res_hi_0_shift, round_const), round_shift);
add_store_aligned(&dst[i * dst_stride + j + 8], &res_hi_0_round,
- &avg_mask);
+ &avg_mask, conv_params->do_average);
const __m256i res_hi_1_32b =
_mm256_cvtepi16_epi32(_mm256_extracti128_si256(res_hi, 1));
@@ -485,7 +486,7 @@
_mm256_add_epi32(res_hi_1_shift, round_const), round_shift);
add_store_aligned(&dst[i * dst_stride + j + 8 + dst_stride],
- &res_hi_1_round, &avg_mask);
+ &res_hi_1_round, &avg_mask, conv_params->do_average);
}
s[0] = s[1];
s[1] = s[2];
@@ -711,10 +712,11 @@
const __m256i res_hi_shift = _mm256_slli_epi32(res_hi_round, bits);
// Accumulate values into the destination buffer
- add_store_aligned(&dst[i * dst_stride + j], &res_lo_shift, &avg_mask);
+ add_store_aligned(&dst[i * dst_stride + j], &res_lo_shift, &avg_mask,
+ conv_params->do_average);
if (w - j > 8) {
add_store_aligned(&dst[i * dst_stride + j + 8], &res_hi_shift,
- &avg_mask);
+ &avg_mask, conv_params->do_average);
}
}
}
diff --git a/av1/common/x86/convolve_sse2.c b/av1/common/x86/convolve_sse2.c
index ab35226..a03f0ef 100644
--- a/av1/common/x86/convolve_sse2.c
+++ b/av1/common/x86/convolve_sse2.c
@@ -75,11 +75,12 @@
}
static INLINE void add_store(CONV_BUF_TYPE *const dst, const __m128i *const res,
- const __m128i *const avg_mask) {
+ const __m128i *const avg_mask, int shift) {
__m128i d;
d = _mm_load_si128((__m128i *)dst);
d = _mm_and_si128(d, *avg_mask);
d = _mm_add_epi32(d, *res);
+ if (shift) d = _mm_srai_epi32(d, 1);
_mm_store_si128((__m128i *)dst, d);
}
@@ -141,7 +142,7 @@
res_shift = _mm_sll_epi32(res, left_shift);
res_shift =
_mm_sra_epi32(_mm_add_epi32(res_shift, round_const), round_shift);
- add_store(dst, &res_shift, &avg_mask);
+ add_store(dst, &res_shift, &avg_mask, conv_params->do_average);
src_ptr += src_stride;
dst += dst_stride;
@@ -149,7 +150,7 @@
res_shift = _mm_sll_epi32(res, left_shift);
res_shift =
_mm_sra_epi32(_mm_add_epi32(res_shift, round_const), round_shift);
- add_store(dst, &res_shift, &avg_mask);
+ add_store(dst, &res_shift, &avg_mask, conv_params->do_average);
src_ptr += src_stride;
dst += dst_stride;
@@ -204,8 +205,10 @@
round_shift);
res_hi_shift = _mm_sra_epi32(_mm_add_epi32(res_hi_shift, round_const),
round_shift);
- add_store(dst + i * dst_stride + j + 0, &res_lo_shift, &avg_mask);
- add_store(dst + i * dst_stride + j + 4, &res_hi_shift, &avg_mask);
+ add_store(dst + i * dst_stride + j + 0, &res_lo_shift, &avg_mask,
+ conv_params->do_average);
+ add_store(dst + i * dst_stride + j + 4, &res_hi_shift, &avg_mask,
+ conv_params->do_average);
i++;
res_lo = convolve_lo_y(s + 1, coeffs); // Filter low index pixels
@@ -216,8 +219,10 @@
round_shift);
res_hi_shift = _mm_sra_epi32(_mm_add_epi32(res_hi_shift, round_const),
round_shift);
- add_store(dst + i * dst_stride + j + 0, &res_lo_shift, &avg_mask);
- add_store(dst + i * dst_stride + j + 4, &res_hi_shift, &avg_mask);
+ add_store(dst + i * dst_stride + j + 0, &res_lo_shift, &avg_mask,
+ conv_params->do_average);
+ add_store(dst + i * dst_stride + j + 4, &res_hi_shift, &avg_mask,
+ conv_params->do_average);
i++;
s[0] = s[2];
@@ -276,7 +281,7 @@
const __m128i res_lo_shift = _mm_sll_epi32(res_lo_round, left_shift);
// Accumulate values into the destination buffer
- add_store(dst, &res_lo_shift, &avg_mask);
+ add_store(dst, &res_lo_shift, &avg_mask, conv_params->do_average);
src_ptr += src_stride;
dst += dst_stride;
} while (--h);
@@ -315,8 +320,10 @@
const __m128i res_hi_shift = _mm_sll_epi32(res_hi_round, left_shift);
// Accumulate values into the destination buffer
- add_store(dst + i * dst_stride + j + 0, &res_lo_shift, &avg_mask);
- add_store(dst + i * dst_stride + j + 4, &res_hi_shift, &avg_mask);
+ add_store(dst + i * dst_stride + j + 0, &res_lo_shift, &avg_mask,
+ conv_params->do_average);
+ add_store(dst + i * dst_stride + j + 4, &res_hi_shift, &avg_mask,
+ conv_params->do_average);
j += 8;
} while (j < w);
} while (++i < h);
diff --git a/av1/common/x86/highbd_convolve_2d_avx2.c b/av1/common/x86/highbd_convolve_2d_avx2.c
index cb13e9b..73c2bea 100644
--- a/av1/common/x86/highbd_convolve_2d_avx2.c
+++ b/av1/common/x86/highbd_convolve_2d_avx2.c
@@ -374,18 +374,26 @@
__m128i *const p = (__m128i *)&dst[i * dst_stride + j];
if (do_average) {
_mm_storeu_si128(
- p + 0, _mm_add_epi32(_mm_loadu_si128(p + 0),
- _mm256_extractf128_si256(res_lo_round, 0)));
+ p + 0, _mm_srai_epi32(_mm_add_epi32(_mm_loadu_si128(p + 0),
+ _mm256_extractf128_si256(
+ res_lo_round, 0)),
+ 1));
_mm_storeu_si128(
- p + 1, _mm_add_epi32(_mm_loadu_si128(p + 1),
- _mm256_extractf128_si256(res_hi_round, 0)));
+ p + 1, _mm_srai_epi32(_mm_add_epi32(_mm_loadu_si128(p + 1),
+ _mm256_extractf128_si256(
+ res_hi_round, 0)),
+ 1));
if (w - j > 8) {
- _mm_storeu_si128(p + 2, _mm_add_epi32(_mm_loadu_si128(p + 2),
- _mm256_extractf128_si256(
- res_lo_round, 1)));
- _mm_storeu_si128(p + 3, _mm_add_epi32(_mm_loadu_si128(p + 3),
- _mm256_extractf128_si256(
- res_hi_round, 1)));
+ _mm_storeu_si128(
+ p + 2, _mm_srai_epi32(_mm_add_epi32(_mm_loadu_si128(p + 2),
+ _mm256_extractf128_si256(
+ res_lo_round, 1)),
+ 1));
+ _mm_storeu_si128(
+ p + 3, _mm_srai_epi32(_mm_add_epi32(_mm_loadu_si128(p + 3),
+ _mm256_extractf128_si256(
+ res_hi_round, 1)),
+ 1));
}
} else {
_mm_storeu_si128(p + 0, _mm256_extractf128_si256(res_lo_round, 0));
diff --git a/av1/common/x86/highbd_convolve_2d_sse4.c b/av1/common/x86/highbd_convolve_2d_sse4.c
index 979d1dd..6980960 100644
--- a/av1/common/x86/highbd_convolve_2d_sse4.c
+++ b/av1/common/x86/highbd_convolve_2d_sse4.c
@@ -201,23 +201,35 @@
__m128i *const p = (__m128i *)&dst[i * dst_stride + j];
if (conv_params->use_jnt_comp_avg) {
if (do_average) {
- const __m128i jnt_sum_lo = _mm_add_epi32(
- _mm_loadu_si128(p + 0), _mm_mullo_epi32(res_lo_round, wt1));
- const __m128i jnt_sum_hi = _mm_add_epi32(
- _mm_loadu_si128(p + 1), _mm_mullo_epi32(res_hi_round, wt1));
+ const __m128i tmp_lo = _mm_loadu_si128(p + 0);
+ const __m128i tmp_hi = _mm_loadu_si128(p + 1);
+ const __m128i jnt_sum_lo =
+ _mm_add_epi32(_mm_mullo_epi32(tmp_lo, wt0),
+ _mm_mullo_epi32(res_lo_round, wt1));
+ const __m128i jnt_sum_hi =
+ _mm_add_epi32(_mm_mullo_epi32(tmp_hi, wt0),
+ _mm_mullo_epi32(res_hi_round, wt1));
+ const __m128i final_lo =
+ _mm_srai_epi32(jnt_sum_lo, DIST_PRECISION_BITS);
+ const __m128i final_hi =
+ _mm_srai_epi32(jnt_sum_hi, DIST_PRECISION_BITS);
- _mm_storeu_si128(p + 0, jnt_sum_lo);
- _mm_storeu_si128(p + 1, jnt_sum_hi);
+ _mm_storeu_si128(p + 0, final_lo);
+ _mm_storeu_si128(p + 1, final_hi);
} else {
- _mm_storeu_si128(p + 0, _mm_mullo_epi32(res_lo_round, wt0));
- _mm_storeu_si128(p + 1, _mm_mullo_epi32(res_hi_round, wt0));
+ _mm_storeu_si128(p + 0, res_lo_round);
+ _mm_storeu_si128(p + 1, res_hi_round);
}
} else {
if (do_average) {
_mm_storeu_si128(
- p + 0, _mm_add_epi32(_mm_loadu_si128(p + 0), res_lo_round));
+ p + 0,
+ _mm_srai_epi32(
+ _mm_add_epi32(_mm_loadu_si128(p + 0), res_lo_round), 1));
_mm_storeu_si128(
- p + 1, _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi_round));
+ p + 1,
+ _mm_srai_epi32(
+ _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi_round), 1));
} else {
_mm_storeu_si128(p + 0, res_lo_round);
_mm_storeu_si128(p + 1, res_hi_round);
diff --git a/av1/common/x86/highbd_convolve_2d_ssse3.c b/av1/common/x86/highbd_convolve_2d_ssse3.c
index ee948f8..ce348ac 100644
--- a/av1/common/x86/highbd_convolve_2d_ssse3.c
+++ b/av1/common/x86/highbd_convolve_2d_ssse3.c
@@ -192,10 +192,14 @@
// Accumulate values into the destination buffer
__m128i *const p = (__m128i *)&dst[i * dst_stride + j];
if (do_average) {
- _mm_storeu_si128(p + 0,
- _mm_add_epi32(_mm_loadu_si128(p + 0), res_lo_round));
- _mm_storeu_si128(p + 1,
- _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi_round));
+ _mm_storeu_si128(
+ p + 0,
+ _mm_srai_epi32(
+ _mm_add_epi32(_mm_loadu_si128(p + 0), res_lo_round), 1));
+ _mm_storeu_si128(
+ p + 1,
+ _mm_srai_epi32(
+ _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi_round), 1));
} else {
_mm_storeu_si128(p + 0, res_lo_round);
_mm_storeu_si128(p + 1, res_hi_round);
diff --git a/av1/common/x86/highbd_warp_plane_sse4.c b/av1/common/x86/highbd_warp_plane_sse4.c
index 5647eb3..4ebd8a6 100644
--- a/av1/common/x86/highbd_warp_plane_sse4.c
+++ b/av1/common/x86/highbd_warp_plane_sse4.c
@@ -309,19 +309,22 @@
#if CONFIG_JNT_COMP
if (conv_params->use_jnt_comp_avg) {
if (comp_avg) {
- const __m128i sum = _mm_add_epi32(_mm_loadu_si128(p),
- _mm_mullo_epi32(res_lo, wt1));
- res_lo = sum;
- } else {
- res_lo = _mm_mullo_epi32(res_lo, wt0);
+ const __m128i sum =
+ _mm_add_epi32(_mm_mullo_epi32(_mm_loadu_si128(p), wt0),
+ _mm_mullo_epi32(res_lo, wt1));
+ res_lo = _mm_srai_epi32(sum, DIST_PRECISION_BITS);
}
} else {
- if (comp_avg) res_lo = _mm_add_epi32(_mm_loadu_si128(p), res_lo);
+ if (comp_avg)
+ res_lo =
+ _mm_srai_epi32(_mm_add_epi32(_mm_loadu_si128(p), res_lo), 1);
}
_mm_storeu_si128(p, res_lo);
#else
- if (comp_avg) res_lo = _mm_add_epi32(_mm_loadu_si128(p), res_lo);
+ if (comp_avg)
+ res_lo =
+ _mm_srai_epi32(_mm_add_epi32(_mm_loadu_si128(p), res_lo), 1);
_mm_storeu_si128(p, res_lo);
#endif
@@ -332,21 +335,22 @@
#if CONFIG_JNT_COMP
if (conv_params->use_jnt_comp_avg) {
if (comp_avg) {
- const __m128i sum = _mm_add_epi32(_mm_loadu_si128(p + 1),
- _mm_mullo_epi32(res_hi, wt1));
- res_hi = sum;
- } else {
- res_hi = _mm_mullo_epi32(res_hi, wt0);
+ const __m128i sum =
+ _mm_add_epi32(_mm_mullo_epi32(_mm_loadu_si128(p + 1), wt0),
+ _mm_mullo_epi32(res_hi, wt1));
+ res_hi = _mm_srai_epi32(sum, DIST_PRECISION_BITS);
}
} else {
if (comp_avg)
- res_hi = _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi);
+ res_hi = _mm_srai_epi32(
+ _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi), 1);
}
_mm_storeu_si128(p + 1, res_hi);
#else
if (comp_avg)
- res_hi = _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi);
+ res_hi = _mm_srai_epi32(
+ _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi), 1);
_mm_storeu_si128(p + 1, res_hi);
#endif
}
diff --git a/av1/common/x86/jnt_convolve_sse4.c b/av1/common/x86/jnt_convolve_sse4.c
index bc23365..54bef5a 100644
--- a/av1/common/x86/jnt_convolve_sse4.c
+++ b/av1/common/x86/jnt_convolve_sse4.c
@@ -76,23 +76,28 @@
}
static INLINE void add_store(CONV_BUF_TYPE *const dst, const __m128i *const res,
- const __m128i *const avg_mask) {
+ const __m128i *const avg_mask, int shift) {
__m128i d;
d = _mm_load_si128((__m128i *)dst);
d = _mm_and_si128(d, *avg_mask);
d = _mm_add_epi32(d, *res);
+ if (shift) d = _mm_srai_epi32(d, 1);
_mm_store_si128((__m128i *)dst, d);
}
#if CONFIG_JNT_COMP
static INLINE void mult_add_store(CONV_BUF_TYPE *const dst,
const __m128i *const res,
- const __m128i *const avg_mask,
- const __m128i *const wt) {
+ const __m128i *const wt0,
+ const __m128i *const wt1, int do_average) {
__m128i d;
- d = _mm_load_si128((__m128i *)dst);
- d = _mm_and_si128(d, *avg_mask);
- d = _mm_add_epi32(d, _mm_mullo_epi32(*res, *wt));
+ if (do_average) {
+ d = _mm_load_si128((__m128i *)dst);
+ d = _mm_add_epi32(_mm_mullo_epi32(d, *wt0), _mm_mullo_epi32(*res, *wt1));
+ d = _mm_srai_epi32(d, DIST_PRECISION_BITS);
+ } else {
+ d = *res;
+ }
_mm_store_si128((__m128i *)dst, d);
}
@@ -111,7 +116,6 @@
const __m128i avg_mask = _mm_set1_epi32(conv_params->do_average ? -1 : 0);
const __m128i wt0 = _mm_set1_epi32(conv_params->fwd_offset);
const __m128i wt1 = _mm_set1_epi32(conv_params->bck_offset);
- const __m128i wt = conv_params->do_average ? wt1 : wt0;
const __m128i round_const = _mm_set1_epi32((1 << conv_params->round_1) >> 1);
const __m128i round_shift = _mm_cvtsi32_si128(conv_params->round_1);
__m128i coeffs[4];
@@ -156,9 +160,9 @@
res_shift =
_mm_sra_epi32(_mm_add_epi32(res_shift, round_const), round_shift);
if (conv_params->use_jnt_comp_avg)
- mult_add_store(dst, &res_shift, &avg_mask, &wt);
+ mult_add_store(dst, &res_shift, &wt0, &wt1, conv_params->do_average);
else
- add_store(dst, &res_shift, &avg_mask);
+ add_store(dst, &res_shift, &avg_mask, conv_params->do_average);
src_ptr += src_stride;
dst += dst_stride;
@@ -167,9 +171,9 @@
res_shift =
_mm_sra_epi32(_mm_add_epi32(res_shift, round_const), round_shift);
if (conv_params->use_jnt_comp_avg)
- mult_add_store(dst, &res_shift, &avg_mask, &wt);
+ mult_add_store(dst, &res_shift, &wt0, &wt1, conv_params->do_average);
else
- add_store(dst, &res_shift, &avg_mask);
+ add_store(dst, &res_shift, &avg_mask, conv_params->do_average);
src_ptr += src_stride;
dst += dst_stride;
@@ -225,13 +229,15 @@
res_hi_shift = _mm_sra_epi32(_mm_add_epi32(res_hi_shift, round_const),
round_shift);
if (conv_params->use_jnt_comp_avg) {
- mult_add_store(dst + i * dst_stride + j + 0, &res_lo_shift, &avg_mask,
- &wt);
- mult_add_store(dst + i * dst_stride + j + 4, &res_hi_shift, &avg_mask,
- &wt);
+ mult_add_store(dst + i * dst_stride + j + 0, &res_lo_shift, &wt0,
+ &wt1, conv_params->do_average);
+ mult_add_store(dst + i * dst_stride + j + 4, &res_hi_shift, &wt0,
+ &wt1, conv_params->do_average);
} else {
- add_store(dst + i * dst_stride + j + 0, &res_lo_shift, &avg_mask);
- add_store(dst + i * dst_stride + j + 4, &res_hi_shift, &avg_mask);
+ add_store(dst + i * dst_stride + j + 0, &res_lo_shift, &avg_mask,
+ conv_params->do_average);
+ add_store(dst + i * dst_stride + j + 4, &res_hi_shift, &avg_mask,
+ conv_params->do_average);
}
i++;
@@ -244,13 +250,15 @@
res_hi_shift = _mm_sra_epi32(_mm_add_epi32(res_hi_shift, round_const),
round_shift);
if (conv_params->use_jnt_comp_avg) {
- mult_add_store(dst + i * dst_stride + j + 0, &res_lo_shift, &avg_mask,
- &wt);
- mult_add_store(dst + i * dst_stride + j + 4, &res_hi_shift, &avg_mask,
- &wt);
+ mult_add_store(dst + i * dst_stride + j + 0, &res_lo_shift, &wt0,
+ &wt1, conv_params->do_average);
+ mult_add_store(dst + i * dst_stride + j + 4, &res_hi_shift, &wt0,
+ &wt1, conv_params->do_average);
} else {
- add_store(dst + i * dst_stride + j + 0, &res_lo_shift, &avg_mask);
- add_store(dst + i * dst_stride + j + 4, &res_hi_shift, &avg_mask);
+ add_store(dst + i * dst_stride + j + 0, &res_lo_shift, &avg_mask,
+ conv_params->do_average);
+ add_store(dst + i * dst_stride + j + 4, &res_hi_shift, &avg_mask,
+ conv_params->do_average);
}
i++;
@@ -285,7 +293,6 @@
const int w1 = conv_params->bck_offset;
const __m128i wt0 = _mm_set1_epi32(w0);
const __m128i wt1 = _mm_set1_epi32(w1);
- const __m128i wt = conv_params->do_average ? wt1 : wt0;
__m128i coeffs[4];
(void)filter_params_y;
@@ -314,9 +321,9 @@
// Accumulate values into the destination buffer
if (conv_params->use_jnt_comp_avg)
- mult_add_store(dst, &res_lo_shift, &avg_mask, &wt);
+ mult_add_store(dst, &res_lo_shift, &wt0, &wt1, conv_params->do_average);
else
- add_store(dst, &res_lo_shift, &avg_mask);
+ add_store(dst, &res_lo_shift, &avg_mask, conv_params->do_average);
src_ptr += src_stride;
dst += dst_stride;
} while (--h);
@@ -356,13 +363,15 @@
// Accumulate values into the destination buffer
if (conv_params->use_jnt_comp_avg) {
- mult_add_store(dst + i * dst_stride + j + 0, &res_lo_shift, &avg_mask,
- &wt);
- mult_add_store(dst + i * dst_stride + j + 4, &res_hi_shift, &avg_mask,
- &wt);
+ mult_add_store(dst + i * dst_stride + j + 0, &res_lo_shift, &wt0,
+ &wt1, conv_params->do_average);
+ mult_add_store(dst + i * dst_stride + j + 4, &res_hi_shift, &wt0,
+ &wt1, conv_params->do_average);
} else {
- add_store(dst + i * dst_stride + j + 0, &res_lo_shift, &avg_mask);
- add_store(dst + i * dst_stride + j + 4, &res_hi_shift, &avg_mask);
+ add_store(dst + i * dst_stride + j + 0, &res_lo_shift, &avg_mask,
+ conv_params->do_average);
+ add_store(dst + i * dst_stride + j + 4, &res_hi_shift, &avg_mask,
+ conv_params->do_average);
}
j += 8;
} while (j < w);
@@ -553,24 +562,34 @@
// original c function at: av1/common/convolve.c: av1_convolve_2d_c
__m128i *const p = (__m128i *)&dst[i * dst_stride + j];
if (do_average) {
- _mm_storeu_si128(p + 0,
- _mm_add_epi32(_mm_loadu_si128(p + 0),
- _mm_mullo_epi32(res_lo_round, wt1)));
- _mm_storeu_si128(p + 1,
- _mm_add_epi32(_mm_loadu_si128(p + 1),
- _mm_mullo_epi32(res_hi_round, wt1)));
+ _mm_storeu_si128(
+ p + 0,
+ _mm_srai_epi32(
+ _mm_add_epi32(_mm_mullo_epi32(_mm_loadu_si128(p + 0), wt0),
+ _mm_mullo_epi32(res_lo_round, wt1)),
+ DIST_PRECISION_BITS));
+ _mm_storeu_si128(
+ p + 1,
+ _mm_srai_epi32(
+ _mm_add_epi32(_mm_mullo_epi32(_mm_loadu_si128(p + 1), wt0),
+ _mm_mullo_epi32(res_hi_round, wt1)),
+ DIST_PRECISION_BITS));
} else {
- _mm_storeu_si128(p + 0, _mm_mullo_epi32(res_lo_round, wt0));
- _mm_storeu_si128(p + 1, _mm_mullo_epi32(res_hi_round, wt0));
+ _mm_storeu_si128(p + 0, res_lo_round);
+ _mm_storeu_si128(p + 1, res_hi_round);
}
} else {
// Accumulate values into the destination buffer
__m128i *const p = (__m128i *)&dst[i * dst_stride + j];
if (do_average) {
_mm_storeu_si128(
- p + 0, _mm_add_epi32(_mm_loadu_si128(p + 0), res_lo_round));
+ p + 0,
+ _mm_srai_epi32(
+ _mm_add_epi32(_mm_loadu_si128(p + 0), res_lo_round), 1));
_mm_storeu_si128(
- p + 1, _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi_round));
+ p + 1,
+ _mm_srai_epi32(
+ _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi_round), 1));
} else {
_mm_storeu_si128(p + 0, res_lo_round);
_mm_storeu_si128(p + 1, res_hi_round);
diff --git a/av1/common/x86/warp_plane_sse4.c b/av1/common/x86/warp_plane_sse4.c
index 1e8ad47..b05b3b8 100644
--- a/av1/common/x86/warp_plane_sse4.c
+++ b/av1/common/x86/warp_plane_sse4.c
@@ -484,18 +484,21 @@
#if CONFIG_JNT_COMP
if (conv_params->use_jnt_comp_avg) {
if (comp_avg) {
- res_lo = _mm_add_epi32(_mm_loadu_si128(p),
+ res_lo = _mm_add_epi32(_mm_mullo_epi32(_mm_loadu_si128(p), wt0),
_mm_mullo_epi32(res_lo, wt1));
- } else {
- res_lo = _mm_mullo_epi32(res_lo, wt0);
+ res_lo = _mm_srai_epi32(res_lo, DIST_PRECISION_BITS);
}
} else {
- if (comp_avg) res_lo = _mm_add_epi32(_mm_loadu_si128(p), res_lo);
+ if (comp_avg)
+ res_lo =
+ _mm_srai_epi32(_mm_add_epi32(_mm_loadu_si128(p), res_lo), 1);
}
_mm_storeu_si128(p, res_lo);
#else
- if (comp_avg) res_lo = _mm_add_epi32(_mm_loadu_si128(p), res_lo);
+ if (comp_avg)
+ res_lo =
+ _mm_srai_epi32(_mm_add_epi32(_mm_loadu_si128(p), res_lo), 1);
_mm_storeu_si128(p, res_lo);
#endif
if (p_width > 4) {
@@ -504,20 +507,22 @@
#if CONFIG_JNT_COMP
if (conv_params->use_jnt_comp_avg) {
if (comp_avg) {
- res_hi = _mm_add_epi32(_mm_loadu_si128(p + 1),
- _mm_mullo_epi32(res_hi, wt1));
- } else {
- res_hi = _mm_mullo_epi32(res_hi, wt0);
+ res_hi =
+ _mm_add_epi32(_mm_mullo_epi32(_mm_loadu_si128(p + 1), wt0),
+ _mm_mullo_epi32(res_hi, wt1));
+ res_hi = _mm_srai_epi32(res_hi, DIST_PRECISION_BITS);
}
} else {
if (comp_avg)
- res_hi = _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi);
+ res_hi = _mm_srai_epi32(
+ _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi), 1);
}
_mm_storeu_si128(p + 1, res_hi);
#else
if (comp_avg)
- res_hi = _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi);
+ res_hi = _mm_srai_epi32(
+ _mm_add_epi32(_mm_loadu_si128(p + 1), res_hi), 1);
_mm_storeu_si128(p + 1, res_hi);
#endif
}
