commit eeacc4c07a525707768610c988909925ba5a782a
author Yue Chen <yuec@google.com> Tue Jan 17 17:29:17 2017 -0800
committer Yue Chen <yuec@google.com> Thu Jan 19 01:16:54 2017 +0000
tree 1bc0d83e0e5790673df129204383f80de4487ad6
parent f7159223848a75f2429e645261d2f3f02d7f02a1

Bug fix: determine tx_mode based on lossless mode of all segments

When segment feature is on, frame level cm->tx_mode can be set to
ONLY_4X4 only if all segments are lossless. Otherwise will cause
bugs when xd->lossless[i] is 0 and xd->lossless[0] is 1.
Also fix the condition of coding tx_type, which should be on when
the qindex of current segment is > 0.

BUG=aomedia:106
BUG=aomedia:104

Change-Id: Ic076083bb78b3b99a6f7d17ec82ee402c64bcc52

av1/encoder/encodeframe.c

diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 34c1c98..afa3bd3 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -4727,7 +4727,19 @@
 }
 
 static TX_MODE select_tx_mode(const AV1_COMP *cpi, MACROBLOCKD *const xd) {
-  if (xd->lossless[0]) return ONLY_4X4;
+  int i, all_lossless = 1;
+
+  if (cpi->common.seg.enabled) {
+    for (i = 0; i < MAX_SEGMENTS; ++i) {
+      if (!xd->lossless[i]) {
+        all_lossless = 0;
+        break;
+      }
+    }
+  } else {
+    all_lossless = xd->lossless[0];
+  }
+  if (all_lossless) return ONLY_4X4;
   if (cpi->sf.tx_size_search_method == USE_LARGESTALL)
     return ALLOW_32X32 + CONFIG_TX64X64;
   else if (cpi->sf.tx_size_search_method == USE_FULL_RD ||
@@ -5002,6 +5014,7 @@
                            : cm->base_qindex;
     xd->lossless[i] = qindex == 0 && cm->y_dc_delta_q == 0 &&
                       cm->uv_dc_delta_q == 0 && cm->uv_ac_delta_q == 0;
+    xd->qindex[i] = qindex;
   }
 
   if (!cm->seg.enabled && xd->lossless[0]) x->optimize = 0;