commit ff5e2098d9a6fc7fe34e045ce95b4a4230c40dd2
author hui su <huisu@google.com> Mon Mar 27 10:07:59 2017 -0700
committer Hui Su <huisu@google.com> Mon Mar 27 21:53:53 2017 +0000
tree 1ade17dea0e8f05ce0bfdfdb7c190c553e2d1eaf
parent 4d8b8f492a15dacb9d875d3c24953aabd80f6f81

rect-tx/ext-tx: fix an assertion fail

Fix an assertion fail in rd_pick_intra_sub_8x8_y_subblock_mode().

BUG=aomedia:410

Change-Id: I22d4aaea0566336605230e09d9590bc235719685

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 570ebb4..11f591f 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2690,7 +2690,10 @@
   assert(bsize < BLOCK_8X8);
   assert(tx_width < 8 || tx_height < 8);
 #if CONFIG_EXT_TX && CONFIG_RECT_TX
-  assert(tx_width == pred_block_width && tx_height == pred_block_height);
+  if (is_lossless)
+    assert(tx_width == 4 && tx_height == 4);
+  else
+    assert(tx_width == pred_block_width && tx_height == pred_block_height);
 #else
   assert(tx_width == 4 && tx_height == 4);
 #endif  // CONFIG_EXT_TX && CONFIG_RECT_TX
@@ -3227,7 +3230,6 @@
 
         bmode_costs = cpi->y_mode_costs[A][L];
       }
-
       this_rd = rd_pick_intra_sub_8x8_y_subblock_mode(
           cpi, mb, idy, idx, &best_mode, bmode_costs,
           xd->plane[0].above_context + idx, xd->plane[0].left_context + idy, &r,