Fix a bug in avx2/sse2 temporal filtering.
avx2/sse2 function only supports square block size. However, under
`YUV 4:2:2` format, subsampling_x does not equal to subsampling_y. In
this case, avx2/sse2 function will fail. C version is used instead.
Bug=aomedia:2643
Change-Id: I8bc4f83242129eb1f87606dd3273ddf290c061d5
diff --git a/av1/encoder/temporal_filter.c b/av1/encoder/temporal_filter.c
index a637df5..e8c0b70 100644
--- a/av1/encoder/temporal_filter.c
+++ b/av1/encoder/temporal_filter.c
@@ -807,10 +807,21 @@
const uint8_t *pred, uint32_t *accum, uint16_t *count) {
assert(num_planes >= 1 && num_planes <= MAX_MB_PLANE);
+ // Determines whether the video is with `YUV 4:2:2` format, since avx2/sse2
+ // function only supports square block size.
+ int is_yuv422_format = 0;
+ for (int plane = 1; plane < num_planes; ++plane) {
+ if (mbd->plane[plane].subsampling_x != mbd->plane[plane].subsampling_y) {
+ is_yuv422_format = 1;
+ break;
+ }
+ }
+
if (TF_ENABLE_PLANEWISE_STRATEGY) {
// TODO(any): avx2 and sse2 version should be changed to align with C
// function before using.
- if (is_frame_high_bitdepth(frame_to_filter) || block_size != BLOCK_32X32) {
+ if (is_frame_high_bitdepth(frame_to_filter) || block_size != BLOCK_32X32 ||
+ is_yuv422_format) {
av1_apply_temporal_filter_planewise_c(
frame_to_filter, mbd, block_size, mb_row, mb_col, num_planes,
noise_levels, use_subblock, block_mse, subblock_mses, q_factor, pred,
