commit 439ba63880ded2da1a1842c57127b4a5428daa3f
author Luc Trudeau <luc@trud.ca> Thu Dec 21 13:17:01 2017 -0500
committer Luc Trudeau <luc@trud.ca> Fri Dec 22 19:43:39 2017 +0000
tree 555c6e56f932db5617ec99f5134af4cab53109a4
parent 3d75209a20ce036528aca31fa93d37a64ed552a5

[CFL] Code cleaning and Fix for TXK_SEL

Small code cleaning and partially resolves issue 1171. I can't confirm
that it's a complete fix because the code asserts further a long

aomenc: ../av1/common/blockd.h:1064: av1_get_tx_type:
  Assertion `tx_type < TX_TYPES' failed.

BUG=aomedia:1171

Change-Id: I3f1c91d751e937dd8e07f7e2f786a75eba4b6891

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 95c0062..e21707d 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2087,7 +2087,7 @@
   int bw = block_size_wide[plane_bsize];
   int bh = block_size_high[plane_bsize];
   int disable_early_skip =
-      x->using_dist_8x8 && plane == 0 && bw >= 8 && bh >= 8 &&
+      x->using_dist_8x8 && plane == AOM_PLANE_Y && bw >= 8 && bh >= 8 &&
       (tx_size == TX_4X4 || tx_size == TX_4X8 || tx_size == TX_8X4) &&
       x->tune_metric != AOM_TUNE_PSNR;
 #endif  // CONFIG_DIST_8X8
@@ -2166,12 +2166,6 @@
     args->exit_early = 1;
     return;
   }
-#if CONFIG_CFL
-  if (plane == AOM_PLANE_Y && xd->cfl.store_y && is_cfl_allowed(mbmi)) {
-    assert(!is_inter_block(mbmi) || plane_bsize < BLOCK_8X8);
-    cfl_store_tx(xd, blk_row, blk_col, tx_size, plane_bsize);
-  }
-#endif  // CONFIG_CFL
   const PLANE_TYPE plane_type = get_plane_type(plane);
   const TX_TYPE tx_type =
       av1_get_tx_type(plane_type, xd, blk_row, blk_col, tx_size);
@@ -2186,6 +2180,13 @@
                       &this_rd_stats);
 #endif  // !CONFIG_TXK_SEL
 
+#if CONFIG_CFL
+  if (plane == AOM_PLANE_Y && xd->cfl.store_y && is_cfl_allowed(mbmi)) {
+    assert(!is_inter_block(mbmi) || plane_bsize < BLOCK_8X8);
+    cfl_store_tx(xd, blk_row, blk_col, tx_size, plane_bsize);
+  }
+#endif  // CONFIG_CFL
+
 #if CONFIG_RD_DEBUG
   av1_update_txb_coeff_cost(&this_rd_stats, plane, tx_size, blk_row, blk_col,
                             this_rd_stats.rate);
@@ -2446,12 +2447,12 @@
     return INT64_MAX;
   }
 #endif
-  txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, 0, bs, tx_size,
+  txfm_rd_in_plane(x, cpi, rd_stats, ref_best_rd, AOM_PLANE_Y, bs, tx_size,
                    cpi->sf.use_fast_coef_costing);
   if (rd_stats->rate == INT_MAX) return INT64_MAX;
 #if !CONFIG_TXK_SEL
-  int plane = 0;
-  rd_stats->rate += av1_tx_type_cost(cm, x, xd, bs, plane, tx_size, tx_type);
+  rd_stats->rate +=
+      av1_tx_type_cost(cm, x, xd, bs, AOM_PLANE_Y, tx_size, tx_type);
 #endif
 
   if (rd_stats->skip) {