commit 106b3cbc7a50156fe0e4404d18619a19d50bedae
author Sarah Parker <sarahparker@google.com> Fri Apr 21 12:13:37 2017 -0700
committer Sarah Parker <sarahparker@google.com> Mon Apr 24 19:08:09 2017 +0000
tree 8ec1b101c19c411c7a9d04d655e771eed3c08a61
parent 8e1734297fa7d19e36cf690abbe3341092317ddb

Use correct bsize in vartx sub8x8 max tx_size lookup

This fixes crashes due to infinite recursion when var_tx,
ext_tx and rect_tx are enabled without cb4x4. This
is the first part of an ongoing fix for this experiment
interaction.

Change-Id: I674f28294666102aff2265f6b6112816cac17378

av1/encoder/encodemb.c

diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index 4199380..bd1d648 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -1320,7 +1320,7 @@
     const BLOCK_SIZE plane_bsize = get_plane_block_size(bsize, pd);
     const int mi_width = block_size_wide[plane_bsize] >> tx_size_wide_log2[0];
     const int mi_height = block_size_high[plane_bsize] >> tx_size_wide_log2[0];
-    const TX_SIZE max_tx_size = max_txsize_rect_lookup[plane_bsize];
+    const TX_SIZE max_tx_size = get_vartx_max_txsize(mbmi, plane_bsize);
     const BLOCK_SIZE txb_size = txsize_to_bsize[max_tx_size];
     const int bw = block_size_wide[txb_size] >> tx_size_wide_log2[0];
     const int bh = block_size_high[txb_size] >> tx_size_wide_log2[0];