commit b9e4f2514f6a10934ea2e3d5b9c8a411738d0fc9
author Hui Su <huisu@google.com> Wed Mar 14 12:22:11 2018 -0700
committer Hui Su <huisu@google.com> Wed Mar 14 22:43:49 2018 +0000
tree b7f1db9140a2241fe1d6c3df8c17a20ef44003a1
parent 33c10f2f16d21283bc615ff7cbc893d643714d54

Skip intra txb hashing for blocks on tile boundary

BUG=aomedia:1577

Change-Id: I0fe6bc64c9817420c58526396eb1e5b5e9e2e3c0

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 64b54db..65ceee7 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -1872,8 +1872,16 @@
 
   TXB_RD_INFO *intra_txb_rd_info = NULL;
   uint16_t cur_joint_ctx = 0;
-  if (cpi->sf.use_intra_txb_hash && frame_is_intra_only(cm) && !is_inter &&
-      plane == 0 && tx_size_wide[tx_size] == tx_size_high[tx_size]) {
+  const int mi_row = -xd->mb_to_top_edge >> (3 + MI_SIZE_LOG2);
+  const int mi_col = -xd->mb_to_left_edge >> (3 + MI_SIZE_LOG2);
+  const int within_border =
+      mi_row >= xd->tile.mi_row_start &&
+      (mi_row + mi_size_high[plane_bsize] < xd->tile.mi_row_end) &&
+      mi_col >= xd->tile.mi_col_start &&
+      (mi_col + mi_size_wide[plane_bsize] < xd->tile.mi_col_end);
+  if (within_border && cpi->sf.use_intra_txb_hash && frame_is_intra_only(cm) &&
+      !is_inter && plane == 0 &&
+      tx_size_wide[tx_size] == tx_size_high[tx_size]) {
     const uint32_t intra_hash =
         get_intra_txb_hash(x, plane, blk_row, blk_col, plane_bsize, tx_size);
     const int intra_hash_idx =