commit e3b81bcf6aeff86bc8faf21156c56d33b901a5fa
author Jingning Han <jingning@google.com> Fri Jun 23 11:43:52 2017 -0700
committer Jingning Han <jingning@google.com> Tue Jun 27 02:35:37 2017 +0000
tree 51c5af33ca25dc73f6ae8b5452ef21040fa10b89
parent a65f3051d62a90bde092ba97e253b38437f1385a

Rework recursive transform block partition search

Support transform block level kernel selection in the recursive
transform block partitioning search.

Change-Id: I511c39705ee636b0c9fabbe4720fe5a9764b964a

av1/encoder/encodetxb.c

diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c
index 2e29db5..0101b16 100644
--- a/av1/encoder/encodetxb.c
+++ b/av1/encoder/encodetxb.c
@@ -1870,7 +1870,11 @@
   TX_TYPE best_tx_type = txk_start;
   int64_t best_rd = INT64_MAX;
   const int coeff_ctx = combine_entropy_contexts(*a, *l);
+  RD_STATS best_rd_stats;
   TX_TYPE tx_type;
+
+  av1_invalid_rd_stats(&best_rd_stats);
+
   for (tx_type = txk_start; tx_type <= txk_end; ++tx_type) {
     if (plane == 0) mbmi->txk_type[block] = tx_type;
     TX_TYPE ref_tx_type =
@@ -1894,10 +1898,13 @@
     int rd = RDCOST(x->rdmult, this_rd_stats.rate, this_rd_stats.dist);
     if (rd < best_rd) {
       best_rd = rd;
-      *rd_stats = this_rd_stats;
+      best_rd_stats = this_rd_stats;
       best_tx_type = tx_type;
     }
   }
+
+  av1_merge_rd_stats(rd_stats, &best_rd_stats);
+
   if (plane == 0) mbmi->txk_type[block] = best_tx_type;
   // TODO(angiebird): Instead of re-call av1_xform_quant and av1_optimize_b,
   // copy the best result in the above tx_type search for loop