commit 4c98733ec8c50a0a41e3832eba4bd1d3e6aa967d
author Debargha Mukherjee <debargha@google.com> Tue Dec 05 19:25:36 2017 -0800
committer Debargha Mukherjee <debargha@google.com> Thu Dec 07 01:09:44 2017 +0000
tree 42fc4d5081d6a1058d3ea8d0d5b2538da0fafd93
parent b997b6a52a9f8f972e6884133c90b95426d554e2

Fix aq_complexity tx_size reset

BUG=aomedia:1065

Change-Id: I0951a276865a5d810eb04bbb5251ed5c1b417ca4

av1/encoder/aq_complexity.c

diff --git a/av1/encoder/aq_complexity.c b/av1/encoder/aq_complexity.c
index 935638e..219050f 100644
--- a/av1/encoder/aq_complexity.c
+++ b/av1/encoder/aq_complexity.c
@@ -123,8 +123,9 @@
     segment = DEFAULT_AQ2_SEG;
   } else {
     // Rate depends on fraction of a SB64 in frame (xmis * ymis / bw * bh).
-    // It is converted to bits * 256 units.
-    const int64_t num = (int64_t)cpi->rc.sb64_target_rate * xmis * ymis * 256;
+    // It is converted to bits << AV1_PROB_COST_SHIFT units.
+    const int64_t num = (int64_t)(cpi->rc.sb64_target_rate * xmis * ymis)
+                        << AV1_PROB_COST_SHIFT;
     const int denom = cm->mib_size * cm->mib_size;
     const int target_rate = (int)(num / denom);
     double logvar;