commit 6eeb4ceea7db6ece03edb6bc158d2b944366ddc8
author Jingning Han <jingning@google.com> Sun Oct 26 12:49:43 2025 -0700
committer Jingning Han <jingning@google.com> Tue Oct 28 10:37:09 2025 -0700
tree 833509c80bd239e3347cb63110fa531319c16d46
parent 43f3869edc8f0002c677247150833aa115699010

Use a consistent trellis optimization process

STATS_CHANGED

Change-Id: Ia8c4c75cbeaf59eac70eaca82fd9cafe41b44227

av1/encoder/encodemb.c

diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index 4707ccf..3226cda 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -248,18 +248,6 @@
   }
 }
 
-// Settings for optimization type. NOTE: To set optimization type for all intra
-// frames, both `KEY_BLOCK_OPT_TYPE` and `INTRA_BLOCK_OPT_TYPE` should be set.
-// TODO(yjshen): These settings are hard-coded and look okay for now. They
-// should be made configurable later.
-// Blocks of key frames ONLY.
-static const OPT_TYPE KEY_BLOCK_OPT_TYPE = TRELLIS_DROPOUT_OPT;
-// Blocks of intra frames (key frames EXCLUSIVE).
-static const OPT_TYPE INTRA_BLOCK_OPT_TYPE = TRELLIS_DROPOUT_OPT;
-// Blocks of inter frames. (NOTE: Dropout optimization is DISABLED by default
-// if trellis optimization is on for inter frames.)
-static const OPT_TYPE INTER_BLOCK_OPT_TYPE = TRELLIS_DROPOUT_OPT;
-
 enum {
   QUANT_FUNC_LOWBD = 0,
   QUANT_FUNC_HIGHBD = 1,
@@ -432,23 +420,12 @@
                       &quant_param);
     av1_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, &txfm_param,
                     &quant_param);
-
-    // Whether trellis or dropout optimization is required for inter frames.
-    const bool do_trellis = INTER_BLOCK_OPT_TYPE == TRELLIS_OPT ||
-                            INTER_BLOCK_OPT_TYPE == TRELLIS_DROPOUT_OPT;
-    const bool do_dropout = INTER_BLOCK_OPT_TYPE == DROPOUT_OPT ||
-                            INTER_BLOCK_OPT_TYPE == TRELLIS_DROPOUT_OPT;
-
-    if (quant_param.use_optimize_b && do_trellis) {
+    if (use_trellis) {
       TXB_CTX txb_ctx;
       get_txb_ctx(plane_bsize, tx_size, plane, a, l, &txb_ctx);
       av1_optimize_b(args->cpi, x, plane, block, tx_size, tx_type, &txb_ctx,
                      &dummy_rate_cost);
     }
-    if (!quant_param.use_optimize_b && do_dropout) {
-      av1_dropout_qcoeff(x, plane, block, tx_size, tx_type,
-                         cm->quant_params.base_qindex);
-    }
   } else {
     p->eobs[block] = 0;
     p->txb_entropy_ctx[block] = 0;
@@ -780,32 +757,12 @@
 
     av1_xform_quant(x, plane, block, blk_row, blk_col, plane_bsize, &txfm_param,
                     &quant_param);
-
-    // Whether trellis or dropout optimization is required for key frames and
-    // intra frames.
-    const bool do_trellis = (frame_is_intra_only(cm) &&
-                             (KEY_BLOCK_OPT_TYPE == TRELLIS_OPT ||
-                              KEY_BLOCK_OPT_TYPE == TRELLIS_DROPOUT_OPT)) ||
-                            (!frame_is_intra_only(cm) &&
-                             (INTRA_BLOCK_OPT_TYPE == TRELLIS_OPT ||
-                              INTRA_BLOCK_OPT_TYPE == TRELLIS_DROPOUT_OPT));
-    const bool do_dropout = (frame_is_intra_only(cm) &&
-                             (KEY_BLOCK_OPT_TYPE == DROPOUT_OPT ||
-                              KEY_BLOCK_OPT_TYPE == TRELLIS_DROPOUT_OPT)) ||
-                            (!frame_is_intra_only(cm) &&
-                             (INTRA_BLOCK_OPT_TYPE == DROPOUT_OPT ||
-                              INTRA_BLOCK_OPT_TYPE == TRELLIS_DROPOUT_OPT));
-
-    if (quant_param.use_optimize_b && do_trellis) {
+    if (use_trellis) {
       TXB_CTX txb_ctx;
       get_txb_ctx(plane_bsize, tx_size, plane, a, l, &txb_ctx);
       av1_optimize_b(args->cpi, x, plane, block, tx_size, tx_type, &txb_ctx,
                      &dummy_rate_cost);
     }
-    if (do_dropout) {
-      av1_dropout_qcoeff(x, plane, block, tx_size, tx_type,
-                         cm->quant_params.base_qindex);
-    }
   }
 
   if (*eob) {