Disable AVX2 optimizations for aom_highbd_${bd}_sub_pixel_variance256x256_avx2
The unit tests for these function were disabled by mistake. If enabled, they fail when compared against the C versions for some extreme values. So, we disable these AVX2 functions.
Also, sse2 versions of these are added instead to minimize runtime impact.
diff --git a/aom_dsp/aom_dsp_rtcd_defs.pl b/aom_dsp/aom_dsp_rtcd_defs.pl
index 5674bb4..3f3038c 100644
--- a/aom_dsp/aom_dsp_rtcd_defs.pl
+++ b/aom_dsp/aom_dsp_rtcd_defs.pl
@@ -1002,9 +1002,9 @@
# Subpixel Variance
#
if (aom_config("CONFIG_EXT_RECUR_PARTITIONS") eq "yes"){
- # specialize qw/aom_highbd_12_sub_pixel_variance256x256 avx2/;
- # specialize qw/aom_highbd_12_sub_pixel_variance256x128 avx2/;
- # specialize qw/aom_highbd_12_sub_pixel_variance128x256 avx2/;
+ specialize qw/aom_highbd_12_sub_pixel_variance256x256 sse2/;
+ specialize qw/aom_highbd_12_sub_pixel_variance256x128 sse2/;
+ specialize qw/aom_highbd_12_sub_pixel_variance128x256 sse2/;
}
add_proto qw/uint32_t aom_highbd_12_sub_pixel_variance128x128/, "const uint16_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint16_t *ref_ptr, int ref_stride, uint32_t *sse";
@@ -1099,9 +1099,9 @@
}
if (aom_config("CONFIG_EXT_RECUR_PARTITIONS") eq "yes"){
- specialize qw/aom_highbd_10_sub_pixel_variance256x256 avx2/;
- specialize qw/aom_highbd_10_sub_pixel_variance256x128 avx2/;
- specialize qw/aom_highbd_10_sub_pixel_variance128x256 avx2/;
+ specialize qw/aom_highbd_10_sub_pixel_variance256x256 sse2/;
+ specialize qw/aom_highbd_10_sub_pixel_variance256x128 sse2 avx2/;
+ specialize qw/aom_highbd_10_sub_pixel_variance128x256 sse2 avx2/;
}
add_proto qw/uint32_t aom_highbd_10_sub_pixel_variance128x128/, "const uint16_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint16_t *ref_ptr, int ref_stride, uint32_t *sse";
@@ -1184,9 +1184,9 @@
if (aom_config("CONFIG_EXT_RECUR_PARTITIONS") eq "yes"){
- specialize qw/aom_highbd_8_sub_pixel_variance256x256 avx2/;
- specialize qw/aom_highbd_8_sub_pixel_variance256x128 avx2/;
- specialize qw/aom_highbd_8_sub_pixel_variance128x256 avx2/;
+ specialize qw/aom_highbd_8_sub_pixel_variance256x256 sse2/;
+ specialize qw/aom_highbd_8_sub_pixel_variance256x128 sse2 avx2/;
+ specialize qw/aom_highbd_8_sub_pixel_variance128x256 sse2 avx2/;
}
add_proto qw/uint32_t aom_highbd_8_sub_pixel_variance128x128/, "const uint16_t *src_ptr, int source_stride, int xoffset, int yoffset, const uint16_t *ref_ptr, int ref_stride, uint32_t *sse";
diff --git a/aom_dsp/x86/highbd_variance_avx2.c b/aom_dsp/x86/highbd_variance_avx2.c
index 13ce9c8..ddafa74 100644
--- a/aom_dsp/x86/highbd_variance_avx2.c
+++ b/aom_dsp/x86/highbd_variance_avx2.c
@@ -871,7 +871,7 @@
}
#if CONFIG_EXT_RECUR_PARTITIONS
-HIGHBD_SUBPIX_VAR(256, 256, 16);
+// HIGHBD_SUBPIX_VAR(256, 256, 16);
HIGHBD_SUBPIX_VAR(256, 128, 15);
HIGHBD_SUBPIX_VAR(128, 256, 15);
#endif // CONFIG_EXT_RECUR_PARTITIONS
diff --git a/aom_dsp/x86/highbd_variance_sse2.c b/aom_dsp/x86/highbd_variance_sse2.c
index 096e994..c187a77 100644
--- a/aom_dsp/x86/highbd_variance_sse2.c
+++ b/aom_dsp/x86/highbd_variance_sse2.c
@@ -272,11 +272,9 @@
int se = 0; \
unsigned int sse = 0; \
unsigned int sse2; \
- int row_rep = (w > 64) ? 2 : 1; \
+ const int row_rep = (w > 128) ? 4 : (w > 64) ? 2 : 1; \
const int wr = AOMMIN(w, 64); \
for (int wd_64 = 0; wd_64 < row_rep; wd_64++) { \
- src += wd_64 * 64; \
- dst += wd_64 * 64; \
int se2 = aom_highbd_sub_pixel_variance##wf##xh_##opt( \
src, src_stride, x_offset, y_offset, dst, dst_stride, h, &sse2, \
NULL, NULL); \
@@ -323,6 +321,8 @@
} \
} \
} \
+ src += 64; \
+ dst += 64; \
} \
*sse_ptr = sse; \
return sse - (uint32_t)((cast se * se) >> (wlog2 + hlog2)); \
@@ -335,11 +335,9 @@
uint32_t sse; \
uint64_t long_sse = 0; \
int se = 0; \
- int row_rep = (w > 64) ? 2 : 1; \
+ const int row_rep = (w > 128) ? 4 : (w > 64) ? 2 : 1; \
const int wr = AOMMIN(w, 64); \
for (int wd_64 = 0; wd_64 < row_rep; wd_64++) { \
- src += wd_64 * 64; \
- dst += wd_64 * 64; \
int se2 = aom_highbd_sub_pixel_variance##wf##xh_##opt( \
src, src_stride, x_offset, y_offset, dst, dst_stride, h, &sse, NULL, \
NULL); \
@@ -387,6 +385,8 @@
} \
} \
} \
+ src += 64; \
+ dst += 64; \
} \
se = ROUND_POWER_OF_TWO(se, 2); \
sse = (uint32_t)ROUND_POWER_OF_TWO(long_sse, 4); \
@@ -403,7 +403,7 @@
int se = 0; \
int64_t var; \
uint64_t long_sse = 0; \
- int row_rep = (w > 64) ? 2 : 1; \
+ const int row_rep = (w > 128) ? 4 : (w > 64) ? 2 : 1; \
const int wr = AOMMIN(w, 64); \
for (start_row = 0; start_row < h; start_row += 16) { \
uint32_t sse2; \
@@ -411,8 +411,6 @@
uint16_t *src_tmp = (uint16_t *)src + (start_row * src_stride); \
uint16_t *dst_tmp = (uint16_t *)dst + (start_row * dst_stride); \
for (int wd_64 = 0; wd_64 < row_rep; wd_64++) { \
- src_tmp += wd_64 * 64; \
- dst_tmp += wd_64 * 64; \
int se2 = aom_highbd_sub_pixel_variance##wf##xh_##opt( \
src_tmp, src_stride, x_offset, y_offset, dst_tmp, dst_stride, \
height, &sse2, NULL, NULL); \
@@ -459,6 +457,8 @@
} \
} \
} \
+ src_tmp += 64; \
+ dst_tmp += 64; \
} \
} \
se = ROUND_POWER_OF_TWO(se, 4); \
@@ -472,6 +472,9 @@
// fixing alignment issues.
#if CONFIG_EXT_RECUR_PARTITIONS
#define FNS(opt) \
+ FN(256, 256, 16, 8, 8, opt, (int64_t)); \
+ FN(256, 128, 16, 8, 7, opt, (int64_t)); \
+ FN(128, 256, 16, 7, 8, opt, (int64_t)); \
FN(128, 128, 16, 7, 7, opt, (int64_t)); \
FN(128, 64, 16, 7, 6, opt, (int64_t)); \
FN(64, 128, 16, 6, 7, opt, (int64_t)); \
diff --git a/test/variance_test.cc b/test/variance_test.cc
index 223f781..1bbbc48 100644
--- a/test/variance_test.cc
+++ b/test/variance_test.cc
@@ -1567,12 +1567,12 @@
const SubpelVarianceParams kArrayHBDSubpelVariance_avx2[] = {
#if CONFIG_EXT_RECUR_PARTITIONS
-// SubpelVarianceParams(8, 8, &aom_highbd_12_sub_pixel_variance256x256_avx2,
-// 12),
-// SubpelVarianceParams(8, 7, &aom_highbd_12_sub_pixel_variance256x128_avx2,
-// 12),
-// SubpelVarianceParams(7, 8, &aom_highbd_12_sub_pixel_variance128x256_avx2,
-// 12),
+// SubpelVarianceParams(8, 8, &aom_highbd_12_sub_pixel_variance256x256_avx2,
+// 12),
+// SubpelVarianceParams(8, 7, &aom_highbd_12_sub_pixel_variance256x128_avx2,
+// 12),
+// SubpelVarianceParams(7, 8, &aom_highbd_12_sub_pixel_variance128x256_avx2,
+// 12),
#endif // CONFIG_EXT_RECUR_PARTITIONS
// SubpelVarianceParams(7, 7, &aom_highbd_12_sub_pixel_variance128x128_avx2,
// 12),
@@ -1655,6 +1655,11 @@
#endif // HAVE_AVX2
const SubpelVarianceParams kArrayHBDSubpelVariance_sse2[] = {
+#if CONFIG_EXT_RECUR_PARTITIONS
+ SubpelVarianceParams(8, 8, &aom_highbd_12_sub_pixel_variance256x256_sse2, 12),
+ SubpelVarianceParams(8, 7, &aom_highbd_12_sub_pixel_variance256x128_sse2, 12),
+ SubpelVarianceParams(7, 8, &aom_highbd_12_sub_pixel_variance128x256_sse2, 12),
+#endif // CONFIG_EXT_RECUR_PARTITIONS
SubpelVarianceParams(7, 7, &aom_highbd_12_sub_pixel_variance128x128_sse2, 12),
SubpelVarianceParams(7, 6, &aom_highbd_12_sub_pixel_variance128x64_sse2, 12),
SubpelVarianceParams(6, 7, &aom_highbd_12_sub_pixel_variance64x128_sse2, 12),
@@ -1671,6 +1676,12 @@
SubpelVarianceParams(3, 4, &aom_highbd_12_sub_pixel_variance8x16_sse2, 12),
SubpelVarianceParams(3, 3, &aom_highbd_12_sub_pixel_variance8x8_sse2, 12),
SubpelVarianceParams(3, 2, &aom_highbd_12_sub_pixel_variance8x4_sse2, 12),
+
+#if CONFIG_EXT_RECUR_PARTITIONS
+ SubpelVarianceParams(8, 8, &aom_highbd_10_sub_pixel_variance256x256_sse2, 10),
+ SubpelVarianceParams(8, 7, &aom_highbd_10_sub_pixel_variance256x128_sse2, 10),
+ SubpelVarianceParams(7, 8, &aom_highbd_10_sub_pixel_variance128x256_sse2, 10),
+#endif // CONFIG_EXT_RECUR_PARTITIONS
SubpelVarianceParams(7, 7, &aom_highbd_10_sub_pixel_variance128x128_sse2, 10),
SubpelVarianceParams(7, 6, &aom_highbd_10_sub_pixel_variance128x64_sse2, 10),
SubpelVarianceParams(6, 7, &aom_highbd_10_sub_pixel_variance64x128_sse2, 10),
@@ -1687,6 +1698,12 @@
SubpelVarianceParams(3, 4, &aom_highbd_10_sub_pixel_variance8x16_sse2, 10),
SubpelVarianceParams(3, 3, &aom_highbd_10_sub_pixel_variance8x8_sse2, 10),
SubpelVarianceParams(3, 2, &aom_highbd_10_sub_pixel_variance8x4_sse2, 10),
+
+#if CONFIG_EXT_RECUR_PARTITIONS
+ SubpelVarianceParams(8, 8, &aom_highbd_8_sub_pixel_variance256x256_sse2, 8),
+ SubpelVarianceParams(8, 7, &aom_highbd_8_sub_pixel_variance256x128_sse2, 8),
+ SubpelVarianceParams(7, 8, &aom_highbd_8_sub_pixel_variance128x256_sse2, 8),
+#endif // CONFIG_EXT_RECUR_PARTITIONS
SubpelVarianceParams(7, 7, &aom_highbd_8_sub_pixel_variance128x128_sse2, 8),
SubpelVarianceParams(7, 6, &aom_highbd_8_sub_pixel_variance128x64_sse2, 8),
SubpelVarianceParams(6, 7, &aom_highbd_8_sub_pixel_variance64x128_sse2, 8),
