new_multisymbol: use cdf-based cost of 1:4 tx flag

Change-Id: I16f1ddfd59760a49c2184b88bf285fa86960d3be

av1/encoder/block.h

diff --git a/av1/encoder/block.h b/av1/encoder/block.h
index b6c91e0..0b32634 100644
--- a/av1/encoder/block.h
+++ b/av1/encoder/block.h
@@ -293,6 +293,9 @@
   int cfl_cost[CFL_JOINT_SIGNS][CFL_PRED_PLANES][CFL_ALPHABET_SIZE];
 #endif  // CONFIG_CFL
   int tx_size_cost[TX_SIZES - 1][TX_SIZE_CONTEXTS][TX_SIZES];
+#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
+  int quarter_tx_size_cost[2];
+#endif
 #if CONFIG_EXT_TX
 #if CONFIG_LGT_FROM_PRED
   int intra_lgt_cost[LGT_SIZES][INTRA_MODES][2];

av1/encoder/encodeframe.c

diff --git a/av1/encoder/encodeframe.c b/av1/encoder/encodeframe.c
index 2e7d05e..a66823d 100644
--- a/av1/encoder/encodeframe.c
+++ b/av1/encoder/encodeframe.c
@@ -5298,8 +5298,11 @@
           quarter_txsize_lookup[bsize] != max_txsize_rect_lookup[bsize] &&
           (mbmi->tx_size == quarter_txsize_lookup[bsize] ||
            mbmi->tx_size == max_txsize_rect_lookup[bsize])) {
-        ++td->counts
-              ->quarter_tx_size[mbmi->tx_size == quarter_txsize_lookup[bsize]];
+        const int use_qttx = mbmi->tx_size == quarter_txsize_lookup[bsize];
+        ++td->counts->quarter_tx_size[use_qttx];
+#if CONFIG_NEW_MULTISYMBOL
+        update_cdf(xd->tile_ctx->quarter_tx_size_cdf, use_qttx, 2);
+#endif
       }
 #endif
 #if CONFIG_EXT_TX && CONFIG_RECT_TX

av1/encoder/rd.c

diff --git a/av1/encoder/rd.c b/av1/encoder/rd.c
index 3f5e623..4ebe8db 100644
--- a/av1/encoder/rd.c
+++ b/av1/encoder/rd.c
@@ -231,6 +231,16 @@
       av1_cost_tokens_from_cdf(x->tx_size_cost[i][j], fc->tx_size_cdf[i][j],
                                NULL);
 
+#if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
+#if CONFIG_NEW_MULTISYMBOL
+  av1_cost_tokens_from_cdf(x->quarter_tx_size_cost, fc->quarter_tx_size_cdf,
+                           NULL);
+#else
+  x->quarter_tx_size_cost[0] = av1_cost_bit(fc->quarter_tx_size_prob, 0);
+  x->quarter_tx_size_cost[1] = av1_cost_bit(fc->quarter_tx_size_prob, 1);
+#endif
+#endif
+
 #if CONFIG_EXT_TX
 #if CONFIG_LGT_FROM_PRED
   if (LGT_FROM_PRED_INTRA) {

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 2e3d90c..1952dcc 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -2399,9 +2399,8 @@
   }
 }
 
-static int tx_size_cost(const AV1_COMP *const cpi, const MACROBLOCK *const x,
+static int tx_size_cost(const AV1_COMMON *const cm, const MACROBLOCK *const x,
                         BLOCK_SIZE bsize, TX_SIZE tx_size) {
-  const AV1_COMMON *const cm = &cpi->common;
   const MACROBLOCKD *const xd = &x->e_mbd;
   const MB_MODE_INFO *const mbmi = &xd->mi[0]->mbmi;
 
@@ -2415,8 +2414,8 @@
     int r_tx_size = x->tx_size_cost[tx_size_cat][tx_size_ctx][depth];
 #if CONFIG_RECT_TX_EXT && (CONFIG_EXT_TX || CONFIG_VAR_TX)
     if (is_quarter_tx_allowed(xd, mbmi, is_inter) && tx_size != coded_tx_size)
-      r_tx_size += av1_cost_bit(cm->fc->quarter_tx_size_prob,
-                                tx_size == quarter_txsize_lookup[bsize]);
+      r_tx_size +=
+          x->quarter_tx_size_cost[tx_size == quarter_txsize_lookup[bsize]];
 #endif
     return r_tx_size;
   } else {
@@ -2506,7 +2505,7 @@
   const int tx_select =
       cm->tx_mode == TX_MODE_SELECT && mbmi->sb_type >= BLOCK_8X8;
 
-  const int r_tx_size = tx_size_cost(cpi, x, bs, tx_size);
+  const int r_tx_size = tx_size_cost(cm, x, bs, tx_size);
 
 #if CONFIG_PVQ
   assert(tx_size >= TX_4X4);
@@ -3384,7 +3383,8 @@
       this_rd = RDCOST(x->rdmult, this_rate, tokenonly_rd_stats.dist);
       if (!xd->lossless[mbmi->segment_id] &&
           block_signals_txsize(mbmi->sb_type)) {
-        tokenonly_rd_stats.rate -= tx_size_cost(cpi, x, bsize, mbmi->tx_size);
+        tokenonly_rd_stats.rate -=
+            tx_size_cost(&cpi->common, x, bsize, mbmi->tx_size);
       }
       if (this_rd < *best_rd) {
         *best_rd = this_rd;
@@ -4496,7 +4496,8 @@
       // tokenonly rate, but for intra blocks, tx_size is always coded
       // (prediction granularity), so we account for it in the full rate,
       // not the tokenonly rate.
-      this_rate_tokenonly -= tx_size_cost(cpi, x, bsize, mbmi->tx_size);
+      this_rate_tokenonly -=
+          tx_size_cost(&cpi->common, x, bsize, mbmi->tx_size);
     }
     if (try_palette && mbmi->mode == DC_PRED) {
       this_rate +=
@@ -4928,7 +4929,7 @@
 #if CONFIG_RECT_TX_EXT
     if (check_qttx) {
       assert(blk_row == 0 && blk_col == 0);
-      rd_stats->rate += av1_cost_bit(cpi->common.fc->quarter_tx_size_prob, 0);
+      rd_stats->rate += x->quarter_tx_size_cost[0];
     }
 #endif
     this_rd = RDCOST(x->rdmult, rd_stats->rate, rd_stats->dist);
@@ -5020,8 +5021,7 @@
         rd_stats_qttx.rate +=
             av1_cost_bit(cpi->common.fc->txfm_partition_prob[ctx], 0);
       }
-      rd_stats_qttx.rate +=
-          av1_cost_bit(cpi->common.fc->quarter_tx_size_prob, 1);
+      rd_stats_qttx.rate += x->quarter_tx_size_cost[1];
       rd_qttx = RDCOST(x->rdmult, rd_stats_qttx.rate, rd_stats_qttx.dist);
 #if CONFIG_LV_MAP
       eobs_qttx[0] = p->txb_entropy_ctx[0];
@@ -10367,7 +10367,7 @@
     // tokenonly rate, but for intra blocks, tx_size is always coded
     // (prediction granularity), so we account for it in the full rate,
     // not the tokenonly rate.
-    rate_y -= tx_size_cost(cpi, x, bsize, mbmi->tx_size);
+    rate_y -= tx_size_cost(cm, x, bsize, mbmi->tx_size);
   }
 
   rate2 += av1_cost_bit(cm->fc->filter_intra_probs[0],
@@ -11151,7 +11151,7 @@
         // tokenonly rate, but for intra blocks, tx_size is always coded
         // (prediction granularity), so we account for it in the full rate,
         // not the tokenonly rate.
-        rate_y -= tx_size_cost(cpi, x, bsize, mbmi->tx_size);
+        rate_y -= tx_size_cost(cm, x, bsize, mbmi->tx_size);
       }
 #if CONFIG_EXT_INTRA
       if (is_directional_mode) {