commit 28d924b7b82c5dc7fbc2e24596a6b80c565a9fc3
author Debargha Mukherjee <debargha@google.com> Wed Oct 05 00:48:28 2016 -0700
committer Debargha Mukherjee <debargha@google.com> Thu Oct 06 11:14:27 2016 -0700
tree 8089abdcdb324c3976c49c7035212b1495aacf84
parent cb603790b080120f28efd538ca8a5d61927434d2

A bug fix for var-tx

Fixes a crash with supertx, ext-tx and rect-tx

Change-Id: I6b5f4cfd6e209558541a791be685b55156aa0138

av1/encoder/encodeframe.c

diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 6cf25fb..87e7d51 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -1304,10 +1304,13 @@
 #if CONFIG_VAR_TX
   {
     const TX_SIZE mtx = mbmi->tx_size;
+    const int num_4x4_blocks_wide = num_4x4_blocks_wide_txsize_lookup[mtx] >> 1;
+    const int num_4x4_blocks_high = num_4x4_blocks_high_txsize_lookup[mtx] >> 1;
     int idy, idx;
-    for (idy = 0; idy < (1 << mtx) / 2; ++idy)
-      for (idx = 0; idx < (1 << mtx) / 2; ++idx)
-        mbmi->inter_tx_size[idy][idx] = mbmi->tx_size;
+    mbmi->inter_tx_size[0][0] = mtx;
+    for (idy = 0; idy < num_4x4_blocks_high; ++idy)
+      for (idx = 0; idx < num_4x4_blocks_wide; ++idx)
+        mbmi->inter_tx_size[idy][idx] = mtx;
   }
 #endif  // CONFIG_VAR_TX
   // Turn motion variation off for supertx

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 2966e77..5e3f285 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -3321,7 +3321,7 @@
   TX_TYPE tx_type, best_tx_type = DCT_DCT;
   const int is_inter = is_inter_block(mbmi);
   TX_SIZE best_tx_size[MAX_MIB_SIZE][MAX_MIB_SIZE];
-  TX_SIZE best_tx = TX_SIZES;
+  TX_SIZE best_tx = max_txsize_lookup[bsize];
   uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE * 4];
   const int n4 = 1 << (num_pels_log2_lookup[bsize] - 4);
   int idx, idy;
@@ -7707,6 +7707,7 @@
     } else {
       x->skip = 1;
       *disable_skip = 1;
+      mbmi->tx_size = tx_size_from_tx_mode(bsize, cm->tx_mode, 1);
 
 // The cost of skip bit needs to be added.
 #if CONFIG_OBMC || CONFIG_WARPED_MOTION