commit eeacc4c07a525707768610c988909925ba5a782a
author Yue Chen <yuec@google.com> Tue Jan 17 17:29:17 2017 -0800
committer Yue Chen <yuec@google.com> Thu Jan 19 01:16:54 2017 +0000
tree 1bc0d83e0e5790673df129204383f80de4487ad6
parent f7159223848a75f2429e645261d2f3f02d7f02a1

Bug fix: determine tx_mode based on lossless mode of all segments

When segment feature is on, frame level cm->tx_mode can be set to
ONLY_4X4 only if all segments are lossless. Otherwise will cause
bugs when xd->lossless[i] is 0 and xd->lossless[0] is 1.
Also fix the condition of coding tx_type, which should be on when
the qindex of current segment is > 0.

BUG=aomedia:106
BUG=aomedia:104

Change-Id: Ic076083bb78b3b99a6f7d17ec82ee402c64bcc52

av1/decoder/decodemv.c

diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c
index 84de6aa..069e87b 100644
--- a/av1/decoder/decodemv.c
+++ b/av1/decoder/decodemv.c
@@ -700,7 +700,9 @@
 #if CONFIG_EXT_TX
     const TX_SIZE square_tx_size = txsize_sqr_map[tx_size];
     if (get_ext_tx_types(tx_size, mbmi->sb_type, inter_block) > 1 &&
-        cm->base_qindex > 0 && !mbmi->skip &&
+        ((!cm->seg.enabled && cm->base_qindex > 0) ||
+         (cm->seg.enabled && xd->qindex[mbmi->segment_id] > 0)) &&
+        !mbmi->skip &&
 #if CONFIG_SUPERTX
         !supertx_enabled &&
 #endif  // CONFIG_SUPERTX
@@ -731,12 +733,17 @@
       mbmi->tx_type = DCT_DCT;
     }
 #else
-    if (tx_size < TX_32X32 && cm->base_qindex > 0 && !mbmi->skip &&
+
+    if (tx_size < TX_32X32 &&
+        ((!cm->seg.enabled && cm->base_qindex > 0) ||
+         (cm->seg.enabled && xd->qindex[mbmi->segment_id] > 0)) &&
+        !mbmi->skip &&
 #if CONFIG_SUPERTX
         !supertx_enabled &&
 #endif  // CONFIG_SUPERTX
         !segfeature_active(&cm->seg, mbmi->segment_id, SEG_LVL_SKIP)) {
       FRAME_COUNTS *counts = xd->counts;
+
       if (inter_block) {
 #if CONFIG_EC_MULTISYMBOL
         mbmi->tx_type = av1_ext_tx_inv[aom_read_symbol(