commit d25ef8c67840e5dc7ecb442a8dc3fbe29d1c841c
author Sarah Parker <sarahparker@google.com> Fri Oct 06 12:17:30 2017 -0700
committer Sarah Parker <sarahparker@google.com> Mon Oct 09 21:41:32 2017 +0000
tree 3afff3a639f3179d352ec3d3977d8b3396ba3a5f
parent 415c8f1f8ce3fe7dea2feddb6e644e6d3a39548c

Change rectangular vartx recursion depth to 2

0.15% improvement on lowres set

Change-Id: If16a8e07797c64508f9e2d9b26ae874ac53c57a4

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index e02933b..63d34dc 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -4991,6 +4991,8 @@
     const int bw = tx_size_wide_unit[max_tx_size];
     int idx, idy;
     int block = 0;
+    int init_depth =
+        (mi_height != mi_width) ? RECT_VARTX_DEPTH_INIT : SQR_VARTX_DEPTH_INIT;
     int step = tx_size_wide_unit[max_tx_size] * tx_size_high_unit[max_tx_size];
     ENTROPY_CONTEXT ctxa[2 * MAX_MIB_SIZE];
     ENTROPY_CONTEXT ctxl[2 * MAX_MIB_SIZE];
@@ -5006,10 +5008,9 @@
 
     for (idy = 0; idy < mi_height; idy += bh) {
       for (idx = 0; idx < mi_width; idx += bw) {
-        select_tx_block(cpi, x, idy, idx, 0, block, max_tx_size,
-                        mi_height != mi_width, plane_bsize, ctxa, ctxl,
-                        tx_above, tx_left, &pn_rd_stats, ref_best_rd - this_rd,
-                        &is_cost_valid);
+        select_tx_block(cpi, x, idy, idx, 0, block, max_tx_size, init_depth,
+                        plane_bsize, ctxa, ctxl, tx_above, tx_left,
+                        &pn_rd_stats, ref_best_rd - this_rd, &is_cost_valid);
         if (pn_rd_stats.rate == INT_MAX) {
           av1_invalid_rd_stats(rd_stats);
           return;