commit 8de99a6e3d68d8a77a1b8316c19ce3a19242d173
author Hui Su <huisu@google.com> Fri Dec 01 12:04:56 2017 -0800
committer Yaowu Xu <yaowu@google.com> Sat Dec 02 17:13:05 2017 +0000
tree 7ac48f18ec6b6628673eb0ccf078f62cb6bdda90
parent 3e632744972ef42352fa17da99d26a64f417b3d7

intrabc: fix SB index calculation in RDO

It was wrong when ext-partition is on and sb_size=64, potentially causing
big compression loss.

Change-Id: I39cba439811bc0ab7c5532842887cf82bb3b5657

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 098f11a..84aa8fd 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -8606,8 +8606,8 @@
   const int mi_col = -xd->mb_to_left_edge / (8 * MI_SIZE);
   const int w = block_size_wide[bsize];
   const int h = block_size_high[bsize];
-  const int sb_row = mi_row / MAX_MIB_SIZE;
-  const int sb_col = mi_col / MAX_MIB_SIZE;
+  const int sb_row = mi_row >> cm->mib_size_log2;
+  const int sb_col = mi_col >> cm->mib_size_log2;
 
   MB_MODE_INFO_EXT *const mbmi_ext = x->mbmi_ext;
   MV_REFERENCE_FRAME ref_frame = INTRA_FRAME;
@@ -8656,16 +8656,16 @@
         x->mv_limits.col_min = (tile->mi_col_start - mi_col) * MI_SIZE;
         x->mv_limits.col_max = (tile->mi_col_end - mi_col) * MI_SIZE - w;
         x->mv_limits.row_min = (tile->mi_row_start - mi_row) * MI_SIZE;
-        x->mv_limits.row_max = (sb_row * MAX_MIB_SIZE - mi_row) * MI_SIZE - h;
+        x->mv_limits.row_max = (sb_row * cm->mib_size - mi_row) * MI_SIZE - h;
         break;
       case IBC_MOTION_LEFT:
         x->mv_limits.col_min = (tile->mi_col_start - mi_col) * MI_SIZE;
-        x->mv_limits.col_max = (sb_col * MAX_MIB_SIZE - mi_col) * MI_SIZE - w;
+        x->mv_limits.col_max = (sb_col * cm->mib_size - mi_col) * MI_SIZE - w;
         // TODO(aconverse@google.com): Minimize the overlap between above and
         // left areas.
         x->mv_limits.row_min = (tile->mi_row_start - mi_row) * MI_SIZE;
         int bottom_coded_mi_edge =
-            AOMMIN((sb_row + 1) * MAX_MIB_SIZE, tile->mi_row_end);
+            AOMMIN((sb_row + 1) * cm->mib_size, tile->mi_row_end);
         x->mv_limits.row_max = (bottom_coded_mi_edge - mi_row) * MI_SIZE - h;
         break;
       default: assert(0);