commit 0ac2bca267112460a547fc4bc7112ca233aa69d5
author Hui Su <huisu@google.com> Wed Mar 07 11:12:45 2018 -0800
committer Hui Su <huisu@google.com> Thu Mar 08 05:09:26 2018 +0000
tree 77b576a5050dd8db92caac1078dc4db6c9109c35
parent 07afef66e8938491f38ead37fe38fd347f5de25f

Properly set is_cost_valid flag in select_tx_block()

BUG=aomedia:1518

Change-Id: I8ce304aa34411e0512e38cba1a549a7e9590c256

av1/encoder/encodemb.c

diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index a7ec79f..e13644c 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -295,6 +295,10 @@
       plane ? av1_get_uv_tx_size(mbmi, pd->subsampling_x, pd->subsampling_y)
             : mbmi->inter_tx_size[av1_get_txb_size_index(plane_bsize, blk_row,
                                                          blk_col)];
+  if (!plane) {
+    assert(tx_size_wide[tx_size] >= tx_size_wide[plane_tx_size] &&
+           tx_size_high[tx_size] >= tx_size_high[plane_tx_size]);
+  }
 
   if (tx_size == plane_tx_size || plane) {
     encode_block(plane, block, blk_row, blk_col, plane_bsize, tx_size, arg,

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 6aca957..985aed9 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -3511,8 +3511,10 @@
     best_tx_type = mbmi->txk_type[txk_type_idx];
   }
 
-  if (cpi->sf.adaptive_txb_search)
-    if (this_rd > ref_best_rd) return;
+  if (cpi->sf.adaptive_txb_search && this_rd > ref_best_rd) {
+    *is_cost_valid = 0;
+    return;
+  }
 
   int tx_split_prune_flag = 0;
   if (cpi->sf.tx_type_search.prune_mode >= PRUNE_2D_ACCURATE)