commit 89d648be89f566f2ec708a80b56c47d69327ab24
author Jingning Han <jingning@google.com> Tue Nov 22 11:22:08 2016 -0800
committer Jingning Han <jingning@google.com> Tue Nov 22 22:26:09 2016 +0000
tree f3fa9504e5a4c76aece7bd6e4b3edd7661175267
parent fe45b216aca251869272735b64f04651073dfd26

Fix coding performance drop in var-tx, supertx, and ext-partition

This commit fixes the use of uninitialized value in the RD search
loop. It brings back the coding performance loss when the above
three experiments are all turned on.

BUG=aomedia:92

Change-Id: I19d7904c354e7a94e5f672bf416267971791c371

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index f34195c..54b40b6 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -3515,7 +3515,13 @@
   const int n4 = 1 << (num_pels_log2_lookup[bsize] - 4);
   int idx, idy;
   int prune = 0;
+  const int count32 = 1 << (2 * (cpi->common.mib_size_log2 -
+                                 mi_width_log2_lookup[BLOCK_32X32]));
+#if CONFIG_EXT_PARTITION
+  RD_STATS rd_stats_stack[16];
+#else
   RD_STATS rd_stats_stack[4];
+#endif
 #if CONFIG_EXT_TX
   int ext_tx_set = get_ext_tx_set(max_tx_size, bsize, is_inter);
 #endif  // CONFIG_EXT_TX
@@ -3529,7 +3535,8 @@
 
   av1_invalid_rd_stats(rd_stats);
 
-  for (idx = 0; idx < 4; ++idx) av1_invalid_rd_stats(&rd_stats_stack[idx]);
+  for (idx = 0; idx < count32; ++idx)
+    av1_invalid_rd_stats(&rd_stats_stack[idx]);
 
   for (tx_type = DCT_DCT; tx_type < TX_TYPES; ++tx_type) {
     RD_STATS this_rd_stats;