commit 4c98733ec8c50a0a41e3832eba4bd1d3e6aa967d
author Debargha Mukherjee <debargha@google.com> Tue Dec 05 19:25:36 2017 -0800
committer Debargha Mukherjee <debargha@google.com> Thu Dec 07 01:09:44 2017 +0000
tree 42fc4d5081d6a1058d3ea8d0d5b2538da0fafd93
parent b997b6a52a9f8f972e6884133c90b95426d554e2

Fix aq_complexity tx_size reset

BUG=aomedia:1065

Change-Id: I0951a276865a5d810eb04bbb5251ed5c1b417ca4

av1/encoder/aq_complexity.c

diff --git a/av1/encoder/aq_complexity.c b/av1/encoder/aq_complexity.c
index 935638e..219050f 100644
--- a/av1/encoder/aq_complexity.c
+++ b/av1/encoder/aq_complexity.c
@@ -123,8 +123,9 @@
     segment = DEFAULT_AQ2_SEG;
   } else {
     // Rate depends on fraction of a SB64 in frame (xmis * ymis / bw * bh).
-    // It is converted to bits * 256 units.
-    const int64_t num = (int64_t)cpi->rc.sb64_target_rate * xmis * ymis * 256;
+    // It is converted to bits << AV1_PROB_COST_SHIFT units.
+    const int64_t num = (int64_t)(cpi->rc.sb64_target_rate * xmis * ymis)
+                        << AV1_PROB_COST_SHIFT;
     const int denom = cm->mib_size * cm->mib_size;
     const int target_rate = (int)(num / denom);
     double logvar;

av1/encoder/encodeframe.c

diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 17715b3..c061cee 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -427,6 +427,14 @@
   } else if (tx_mode != TX_MODE_SELECT) {
     mbmi->tx_size =
         tx_size_from_tx_mode(mbmi->sb_type, tx_mode, is_inter_block(mbmi));
+  } else {
+    BLOCK_SIZE bsize = mbmi->sb_type;
+    TX_SIZE max_rect_txsize = get_max_rect_tx_size(bsize, is_inter_block(mbmi));
+    TX_SIZE min_tx_size =
+        (TX_SIZE)AOMMAX((int)TX_4X4,
+                        txsize_sqr_map[max_rect_txsize] - MAX_TX_DEPTH +
+                            is_rect_tx(max_rect_txsize));
+    mbmi->tx_size = (TX_SIZE)AOMMAX(mbmi->tx_size, min_tx_size);
   }
 }