[ext-intra] fix bug in sample interpolation when dy < 0 When the "dy" variable is between -1 and 0 (base2 == -1) we should interpolate between the upper-left(-1,-1) and left(-1,0) samples, instead of just taking the left sample. Coding gain improves a bit with this change. BUG=aomedia:357 Change-Id: If520f71b22115e79120499c21f8d6925c1fa24be
diff --git a/av1/common/reconintra.c b/av1/common/reconintra.c index 7cf30eb..d23a81a 100644 --- a/av1/common/reconintra.c +++ b/av1/common/reconintra.c
@@ -758,18 +758,13 @@ #endif // CONFIG_INTRA_INTERP } else { base2 = y >> 8; - if (base2 >= 0) { - shift2 = y & 0xFF; + shift2 = y & 0xFF; #if CONFIG_INTRA_INTERP - val = - intra_subpel_interp(base2, shift2, left, 0, bs - 1, filter_type); + val = intra_subpel_interp(base2, shift2, left, -1, bs - 1, filter_type); #else - val = left[base2] * (256 - shift2) + left[base2 + 1] * shift2; - val = ROUND_POWER_OF_TWO(val, 8); + val = left[base2] * (256 - shift2) + left[base2 + 1] * shift2; + val = ROUND_POWER_OF_TWO(val, 8); #endif // CONFIG_INTRA_INTERP - } else { - val = left[0]; - } } dst[c] = clip_pixel(val); } @@ -1053,18 +1048,14 @@ x = c + 1; y = (r << 8) - x * dy; base = y >> 8; - if (base >= 0) { - shift = y & 0xFF; + shift = y & 0xFF; #if CONFIG_INTRA_INTERP - val = highbd_intra_subpel_interp(base, shift, left, 0, bs - 1, - filter_type); + val = highbd_intra_subpel_interp(base, shift, left, -1, bs - 1, + filter_type); #else - val = left[base] * (256 - shift) + left[base + 1] * shift; - val = ROUND_POWER_OF_TWO(val, 8); + val = left[base] * (256 - shift) + left[base + 1] * shift; + val = ROUND_POWER_OF_TWO(val, 8); #endif // CONFIG_INTRA_INTERP - } else { - val = left[0]; - } } dst[c] = clip_pixel_highbd(val, bd); } @@ -1559,9 +1550,10 @@ int i; uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); - DECLARE_ALIGNED(16, uint16_t, left_col[MAX_TX_SIZE * 2]); + DECLARE_ALIGNED(16, uint16_t, left_data[MAX_TX_SIZE * 2 + 16]); DECLARE_ALIGNED(16, uint16_t, above_data[MAX_TX_SIZE * 2 + 16]); uint16_t *above_row = above_data + 16; + uint16_t *left_col = left_data + 16; const uint16_t *const_above_row = above_row; const int bs = tx_size_wide[tx_size]; int need_left = extend_modes[mode] & NEED_LEFT; @@ -1683,6 +1675,7 @@ if (need_above_left) { above_row[-1] = n_top_px > 0 ? (n_left_px > 0 ? above_ref[-1] : base + 1) : base - 1; + left_col[-1] = above_row[-1]; } #if CONFIG_FILTER_INTRA @@ -1727,10 +1720,11 @@ int n_left_px, int n_bottomleft_px, int plane) { int i; - DECLARE_ALIGNED(16, uint8_t, left_col[MAX_TX_SIZE * 2]); const uint8_t *above_ref = ref - ref_stride; + DECLARE_ALIGNED(16, uint8_t, left_data[MAX_TX_SIZE * 2 + 16]); DECLARE_ALIGNED(16, uint8_t, above_data[MAX_TX_SIZE * 2 + 16]); uint8_t *above_row = above_data + 16; + uint8_t *left_col = left_data + 16; const uint8_t *const_above_row = above_row; const int bs = tx_size_wide[tx_size]; int need_left = extend_modes[mode] & NEED_LEFT; @@ -1850,6 +1844,7 @@ if (need_above_left) { above_row[-1] = n_top_px > 0 ? (n_left_px > 0 ? above_ref[-1] : 129) : 127; + left_col[-1] = above_row[-1]; } #if CONFIG_FILTER_INTRA