commit 9fe188e4a534d84b7b4c069ad30121a4d9bb281a
author Julia Robson <juliamrobson@gmail.com> Thu Jan 21 17:14:29 2016 +0000
committer Debargha Mukherjee <debargha@google.com> Thu Jan 21 17:46:03 2016 -0800
tree 0b1892481a80414ecbb6580d6c72c50a70321f2b
parent b1d49f24b475b94ac48c30e1c016b1e124e10401

Fixes for var_tx when ext_tx is not enabled

This patch fixes a couple of issues caused by change-id:
I15d20ce5292b70f0c2b4ba55c1f1318181481596
Changes to the code for when the ext_tx experiment is not enabled
were merged from master but as var_tx does not exist on master
the changes had not been applied to the case when var_tx experiment
is enabled

Change-Id: Iaf2b717e6b8626b2b6a03226127221b776b49884

vp10/encoder/rdopt.c

diff --git a/vp10/encoder/rdopt.c b/vp10/encoder/rdopt.c
index b55f53a..a22769c 100644
--- a/vp10/encoder/rdopt.c
+++ b/vp10/encoder/rdopt.c
@@ -1136,6 +1136,8 @@
 
   mbmi->tx_size = best_tx;
   mbmi->tx_type = best_tx_type;
+  if (mbmi->tx_size >= TX_32X32)
+    assert(mbmi->tx_type == DCT_DCT);
   txfm_rd_in_plane(x,
 #if CONFIG_VAR_TX
                    cpi,
@@ -2516,7 +2518,6 @@
   }
 }
 
-#if CONFIG_EXT_TX
 static void select_tx_type_yrd(const VP10_COMP *cpi, MACROBLOCK *x,
                                int *rate, int64_t *distortion, int *skippable,
                                int64_t *sse, BLOCK_SIZE bsize,
@@ -2528,7 +2529,6 @@
   int64_t rd = INT64_MAX;
   int64_t best_rd = INT64_MAX;
   TX_TYPE tx_type, best_tx_type = DCT_DCT;
-  int ext_tx_set;
   const int is_inter = is_inter_block(mbmi);
   vpx_prob skip_prob = vp10_get_skip_prob(cm, xd);
   int s0 = vp10_cost_bit(skip_prob, 0);
@@ -2538,20 +2538,21 @@
   uint8_t best_blk_skip[256];
   const int n4 = 1 << (num_pels_log2_lookup[bsize] - 4);
   int idx, idy;
+#if CONFIG_EXT_TX
+  int ext_tx_set = get_ext_tx_set(max_tx_size, bsize, is_inter);
+#endif
 
   *distortion = INT64_MAX;
   *rate       = INT_MAX;
   *skippable  = 0;
   *sse        = INT64_MAX;
 
-  ext_tx_set = get_ext_tx_set(max_tx_size, bsize, is_inter);
-
   for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) {
     int this_rate = 0;
     int this_skip = 1;
     int64_t this_dist = 0;
     int64_t this_sse  = 0;
-
+#if CONFIG_EXT_TX
     if (is_inter) {
       if (!ext_tx_used_inter[ext_tx_set][tx_type])
         continue;
@@ -2589,6 +2590,26 @@
                                        [mbmi->mode][mbmi->tx_type];
       }
     }
+#else  // CONFIG_EXT_TX
+      if (max_tx_size >= TX_32X32 && tx_type != DCT_DCT) {
+        continue;
+      }
+      mbmi->tx_type = tx_type;
+
+      inter_block_yrd(cpi, x, &this_rate, &this_dist, &this_skip, &this_sse,
+                      bsize, ref_best_rd);
+
+      if (max_tx_size < TX_32X32 &&
+          !xd->lossless[xd->mi[0]->mbmi.segment_id] &&
+          this_rate != INT_MAX) {
+        if (is_inter)
+          this_rate += cpi->inter_tx_type_costs[max_tx_size][mbmi->tx_type];
+        else
+          this_rate += cpi->intra_tx_type_costs[max_tx_size]
+              [intra_mode_to_tx_type_context[mbmi->mode]]
+              [mbmi->tx_type];
+      }
+#endif  // CONFIG_EXT_TX
 
     if (this_rate == INT_MAX)
       continue;
@@ -2625,7 +2646,6 @@
   mbmi->tx_size = best_tx;
   memcpy(x->blk_skip[0], best_blk_skip, sizeof(best_blk_skip[0]) * n4);
 }
-#endif
 
 static void tx_block_rd(const VP10_COMP *cpi, MACROBLOCK *x,
                         int blk_row, int blk_col, int plane, int block,
@@ -5002,13 +5022,8 @@
     vp10_subtract_plane(x, bsize, 0);
 #if CONFIG_VAR_TX
     if (cm->tx_mode == TX_MODE_SELECT || xd->lossless[mbmi->segment_id]) {
-#if CONFIG_EXT_TX
       select_tx_type_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse,
                          bsize, ref_best_rd);
-#else
-      inter_block_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse,
-                      bsize, ref_best_rd);
-#endif
     } else {
       int idx, idy;
       super_block_yrd(cpi, x, rate_y, &distortion_y, &skippable_y, psse,