commit 18ad894a23e7ee53019c1d90c48a8a906dc1d04e
author Angie Chiang <angiebird@google.com> Tue Apr 11 12:37:07 2017 -0700
committer Angie Chiang <angiebird@google.com> Tue Apr 11 23:56:30 2017 +0000
tree 6654335602d0486c677726f21966e087405dcf70
parent f1cb075db149f627cbe3b945839f566cf1cc6b7d

Fix invalid tx_type returned by get_tx_type

1) Check if tx_type is valid in get_tx_type

2) Remove scan_order from rdcost_block_args
When lv_map is on, scan_order depends on tx_type but tx_type is
not decided before entering block_rd_txfm yet. Therefore
assigning a scan_order into rdcost_block_args and then passing it
into block_rd_txfm will cause error.

3) Pass correct index into intra_mode_to_tx_type_context in
get_tx_type

This CL doesn't affect baseline/supertx's stats.

Change-Id: I59eb12aaf1edd9110ce7a92ce61f81bf89cd5920

av1/common/blockd.h

diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 997b6c7..bc88a4a 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -901,11 +901,10 @@
 
 static INLINE TX_TYPE get_tx_type(PLANE_TYPE plane_type, const MACROBLOCKD *xd,
                                   int block, TX_SIZE tx_size) {
-#if !CONFIG_LV_MAP
   const MODE_INFO *const mi = xd->mi[0];
   const MB_MODE_INFO *const mbmi = &mi->mbmi;
-
   const int block_raster_idx = av1_block_index_to_raster_order(tx_size, block);
+#if !CONFIG_LV_MAP
 
   if (FIXED_TX_TYPE)
     return get_default_tx_type(plane_type, xd, block_raster_idx, tx_size);
@@ -956,21 +955,23 @@
 #endif  // CONFIG_EXT_TX
 #else   // !CONFIG_LV_MAP
   (void)tx_size;
-  const MODE_INFO *const mi = xd->mi[0];
-  const MB_MODE_INFO *const mbmi = &mi->mbmi;
-
+  TX_TYPE tx_type;
   if (plane_type != PLANE_TYPE_Y || xd->lossless[mbmi->segment_id] ||
-      mbmi->tx_size >= TX_32X32)
-    return DCT_DCT;
-
-  if (mbmi->sb_type < BLOCK_8X8) {
-    if (mbmi->ref_frame[0] < LAST_FRAME)
-      return intra_mode_to_tx_type_context[mi->bmi[block].as_mode];
-    else
-      return DCT_DCT;
+      mbmi->tx_size >= TX_32X32) {
+    tx_type = DCT_DCT;
+  } else if (mbmi->sb_type < BLOCK_8X8) {
+    if (is_inter_block(mbmi))  // Sub8x8-Inter
+      tx_type = DCT_DCT;
+    else  // Sub8x8 Intra OR UV-Intra
+      tx_type =
+          intra_mode_to_tx_type_context[plane_type == PLANE_TYPE_Y
+                                            ? get_y_mode(mi, block_raster_idx)
+                                            : mbmi->uv_mode];
+  } else {
+    tx_type = mbmi->txk_type[block];
   }
-
-  return mbmi->txk_type[block];
+  assert(tx_type >= DCT_DCT && tx_type < TX_TYPES);
+  return tx_type;
 #endif  // !CONFIG_LV_MAP
 }