Clean up av1_cost_coeffs

Pass reduced_tx_set_used instead of cm ptr.
Also removed some unneeded checks.

Change-Id: Ie517c5c4c63a2954fd850e222092fe329a3ef726
diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c
index 0723d82..bcd0c55 100644
--- a/av1/encoder/encodetxb.c
+++ b/av1/encoder/encodetxb.c
@@ -726,12 +726,12 @@
 }
 
 static AOM_FORCE_INLINE int warehouse_efficients_txb(
-    const AV1_COMMON *const cm, const MACROBLOCK *x, const int plane,
-    const int block, const TX_SIZE tx_size, const TXB_CTX *const txb_ctx,
+    const MACROBLOCK *x, const int plane, const int block,
+    const TX_SIZE tx_size, const TXB_CTX *const txb_ctx,
     const struct macroblock_plane *p, const int eob,
     const PLANE_TYPE plane_type, const LV_MAP_COEFF_COST *const coeff_costs,
-    const MACROBLOCKD *const xd, const TX_TYPE tx_type,
-    const TX_CLASS tx_class) {
+    const MACROBLOCKD *const xd, const TX_TYPE tx_type, const TX_CLASS tx_class,
+    int reduced_tx_set_used) {
   const tran_low_t *const qcoeff = p->qcoeff + BLOCK_OFFSET(block);
   const int txb_skip_ctx = txb_ctx->txb_skip_ctx;
   const int bwl = get_txb_bwl(tx_size);
@@ -749,8 +749,7 @@
 
   av1_txb_init_levels(qcoeff, width, height, levels);
 
-  cost +=
-      get_tx_type_cost(x, xd, plane, tx_size, tx_type, cm->reduced_tx_set_used);
+  cost += get_tx_type_cost(x, xd, plane, tx_size, tx_type, reduced_tx_set_used);
 
   cost += get_eob_cost(eob, eob_costs, coeff_costs, tx_class);
 
@@ -789,7 +788,7 @@
     const int coeff_ctx = coeff_contexts[pos];
     const tran_low_t v = qcoeff[pos];
     const int level = abs(v);
-    const int cost0 = base_cost[coeff_ctx][AOMMIN(level, 3)];
+    cost += base_cost[coeff_ctx][AOMMIN(level, 3)];
     if (v) {
       // sign bit cost
       cost += av1_cost_literal(1);
@@ -798,9 +797,9 @@
         cost += get_br_cost(level, lps_cost[ctx]);
       }
     }
-    cost += cost0;
   }
-  if (c == 0) {
+  // c == 0 after previous loop
+  {
     const int pos = scan[c];
     const tran_low_t v = qcoeff[pos];
     const int coeff_ctx = coeff_contexts[pos];
@@ -822,9 +821,9 @@
   return cost;
 }
 
-int av1_cost_coeffs_txb(const AV1_COMMON *const cm, const MACROBLOCK *x,
-                        const int plane, const int block, const TX_SIZE tx_size,
-                        const TX_TYPE tx_type, const TXB_CTX *const txb_ctx) {
+int av1_cost_coeffs_txb(const MACROBLOCK *x, const int plane, const int block,
+                        const TX_SIZE tx_size, const TX_TYPE tx_type,
+                        const TXB_CTX *const txb_ctx, int reduced_tx_set_used) {
   const struct macroblock_plane *p = &x->plane[plane];
   const int eob = p->eobs[block];
   const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size);
@@ -838,9 +837,9 @@
   const MACROBLOCKD *const xd = &x->e_mbd;
   const TX_CLASS tx_class = tx_type_to_class[tx_type];
 
-  return warehouse_efficients_txb(cm, x, plane, block, tx_size, txb_ctx, p, eob,
+  return warehouse_efficients_txb(x, plane, block, tx_size, txb_ctx, p, eob,
                                   plane_type, coeff_costs, xd, tx_type,
-                                  tx_class);
+                                  tx_class, reduced_tx_set_used);
 }
 
 static int optimize_txb(TxbInfo *txb_info, const LV_MAP_COEFF_COST *txb_costs,
diff --git a/av1/encoder/encodetxb.h b/av1/encoder/encodetxb.h
index 3ad7e82..677b4f9 100644
--- a/av1/encoder/encodetxb.h
+++ b/av1/encoder/encodetxb.h
@@ -52,9 +52,9 @@
 
 void av1_alloc_txb_buf(AV1_COMP *cpi);
 void av1_free_txb_buf(AV1_COMP *cpi);
-int av1_cost_coeffs_txb(const AV1_COMMON *const cm, const MACROBLOCK *x,
-                        const int plane, const int block, const TX_SIZE tx_size,
-                        const TX_TYPE tx_type, const TXB_CTX *const txb_ctx);
+int av1_cost_coeffs_txb(const MACROBLOCK *x, const int plane, const int block,
+                        const TX_SIZE tx_size, const TX_TYPE tx_type,
+                        const TXB_CTX *const txb_ctx, int reduced_tx_set_used);
 void av1_write_coeffs_txb(const AV1_COMMON *const cm, MACROBLOCK *const x,
                           aom_writer *w, int blk_row, int blk_col, int plane,
                           int block, TX_SIZE tx_size);
diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 0ec6040..faea6de 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -3439,8 +3439,9 @@
       av1_xform_quant(
           cm, x, plane, block, blk_row, blk_col, plane_bsize, tx_size, tx_type,
           USE_B_QUANT_NO_TRELLIS ? AV1_XFORM_QUANT_B : AV1_XFORM_QUANT_FP);
-      rate_cost = av1_cost_coeffs(cm, x, plane, block, tx_size, tx_type,
-                                  txb_ctx, use_fast_coef_costing);
+      rate_cost =
+          av1_cost_coeffs(x, plane, block, tx_size, tx_type, txb_ctx,
+                          use_fast_coef_costing, cm->reduced_tx_set_used);
     } else {
       av1_xform_quant(cm, x, plane, block, blk_row, blk_col, plane_bsize,
                       tx_size, tx_type, AV1_XFORM_QUANT_FP);
diff --git a/av1/encoder/rdopt.h b/av1/encoder/rdopt.h
index 090481e..d24401b 100644
--- a/av1/encoder/rdopt.h
+++ b/av1/encoder/rdopt.h
@@ -83,18 +83,18 @@
   return coeff_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][1];
 }
 
-static INLINE int av1_cost_coeffs(const AV1_COMMON *const cm, MACROBLOCK *x,
-                                  int plane, int block, TX_SIZE tx_size,
-                                  const TX_TYPE tx_type,
+static INLINE int av1_cost_coeffs(MACROBLOCK *x, int plane, int block,
+                                  TX_SIZE tx_size, const TX_TYPE tx_type,
                                   const TXB_CTX *const txb_ctx,
-                                  int use_fast_coef_costing) {
+                                  int use_fast_coef_costing,
+                                  int reduced_tx_set_used) {
 #if TXCOEFF_COST_TIMER
   struct aom_usec_timer timer;
   aom_usec_timer_start(&timer);
 #endif
   (void)use_fast_coef_costing;
-  const int cost =
-      av1_cost_coeffs_txb(cm, x, plane, block, tx_size, tx_type, txb_ctx);
+  const int cost = av1_cost_coeffs_txb(x, plane, block, tx_size, tx_type,
+                                       txb_ctx, reduced_tx_set_used);
 #if TXCOEFF_COST_TIMER
   AV1_COMMON *tmp_cm = (AV1_COMMON *)&cpi->common;
   aom_usec_timer_mark(&timer);