Make the CDEF RDO handle 4:2:2 properly This fixes an assert: av1/common/cdef_block.c:561: cdef_filter_fb: Assertion `bsize == BLOCK_8X8 || bsize == BLOCK_4X4' failed. The RDO simply assigned a strength of 0 in the 4:2:2 case and called cdef_filter_fb(), but cdef_filter_fb() will complain about 4:2:2 even if the strength is 0. The fix assigns a chroma mse of 0 when the the subsampling is different for x and y rather than to call the filter. This is faster also. The mse isn't really 0, but calculating the actual chroma mse doesn't change result. BUG=aomedia:881 Change-Id: I6154e21ddcca30e51baf805684dace10459c3350
diff --git a/av1/encoder/pickcdef.c b/av1/encoder/pickcdef.c index accc97e..cf6ccca 100644 --- a/av1/encoder/pickcdef.c +++ b/av1/encoder/pickcdef.c
@@ -423,35 +423,40 @@ int xsize = (nhb << mi_wide_l2[pli]) + CDEF_HBORDER * (fbc != nhfb - 1) + xoff; sec_strength = gi % CDEF_SEC_STRENGTHS; + if (pli && !chroma_cdef) { + curr_mse = 0; + } else { #if CONFIG_CDEF_SINGLEPASS - copy_sb16_16(&in[(-yoff * CDEF_BSTRIDE - xoff)], CDEF_BSTRIDE, - src[pli], - (fbr * MI_SIZE_64X64 << mi_high_l2[pli]) - yoff, - (fbc * MI_SIZE_64X64 << mi_wide_l2[pli]) - xoff, - stride[pli], ysize, xsize); - cdef_filter_fb(NULL, tmp_dst, CDEF_BSTRIDE, in, xdec[pli], ydec[pli], - dir, &dirinit, var, pli, dlist, cdef_count, threshold, - sec_strength + (sec_strength == 3), pri_damping, - sec_damping, coeff_shift); -#else - if (sec_strength == 0) copy_sb16_16(&in[(-yoff * CDEF_BSTRIDE - xoff)], CDEF_BSTRIDE, src[pli], (fbr * MI_SIZE_64X64 << mi_high_l2[pli]) - yoff, (fbc * MI_SIZE_64X64 << mi_wide_l2[pli]) - xoff, stride[pli], ysize, xsize); - cdef_filter_fb(sec_strength ? NULL : (uint8_t *)in, CDEF_BSTRIDE, - tmp_dst, in, xdec[pli], ydec[pli], dir, &dirinit, var, - pli, dlist, cdef_count, threshold, - sec_strength + (sec_strength == 3), sec_damping, - pri_damping, coeff_shift, sec_strength != 0, 1); + cdef_filter_fb(NULL, tmp_dst, CDEF_BSTRIDE, in, xdec[pli], + ydec[pli], dir, &dirinit, var, pli, dlist, + cdef_count, threshold, + sec_strength + (sec_strength == 3), pri_damping, + sec_damping, coeff_shift); +#else + if (sec_strength == 0) + copy_sb16_16(&in[(-yoff * CDEF_BSTRIDE - xoff)], CDEF_BSTRIDE, + src[pli], + (fbr * MI_SIZE_64X64 << mi_high_l2[pli]) - yoff, + (fbc * MI_SIZE_64X64 << mi_wide_l2[pli]) - xoff, + stride[pli], ysize, xsize); + cdef_filter_fb(sec_strength ? NULL : (uint8_t *)in, CDEF_BSTRIDE, + tmp_dst, in, xdec[pli], ydec[pli], dir, &dirinit, + var, pli, dlist, cdef_count, threshold, + sec_strength + (sec_strength == 3), sec_damping, + pri_damping, coeff_shift, sec_strength != 0, 1); #endif - curr_mse = compute_cdef_dist( - ref_coeff[pli] + - (fbr * MI_SIZE_64X64 << mi_high_l2[pli]) * stride[pli] + - (fbc * MI_SIZE_64X64 << mi_wide_l2[pli]), - stride[pli], tmp_dst, dlist, cdef_count, bsize[pli], coeff_shift, - pli); + curr_mse = compute_cdef_dist( + ref_coeff[pli] + + (fbr * MI_SIZE_64X64 << mi_high_l2[pli]) * stride[pli] + + (fbc * MI_SIZE_64X64 << mi_wide_l2[pli]), + stride[pli], tmp_dst, dlist, cdef_count, bsize[pli], + coeff_shift, pli); + } if (pli < 2) mse[pli][sb_count][gi] = curr_mse; else