commit 66205b7b5c0f40d735fd1b8a58a0a8d3f1a2972f
author Peng Bin <binpengsmail@gmail.com> Tue Sep 04 15:11:08 2018 +0800
committer Yunqing Wang <yunqingwang@google.com> Tue Sep 04 21:31:39 2018 +0000
tree 4468b98c47a9406010c574e37bdaebc3e65b9d05
parent 57dacd0c601451a5fca2993d0c691ee1d9ef2401

Remove overwrite of tx_type when best_eob==0

The overwrite of tx_type when best_eob==0 is not
necessary, and may cause the encode result not match
with the best result of RDO. Such as only tx_type
other than DCT_DCT can produce zero eob, after
overwrite, the encode process will use DCT_DCT which
can't produce zero eob.

STATS_CHANGED expected

Change-Id: Iccb9c51785a940d8e820c956908cf0413fbfb5b4

av1/encoder/encodetxb.c

diff --git a/av1/encoder/encodetxb.c b/av1/encoder/encodetxb.c
index 9b8535f..5a31d93 100644
--- a/av1/encoder/encodetxb.c
+++ b/av1/encoder/encodetxb.c
@@ -496,8 +496,12 @@
                           aom_writer *w, int blk_row, int blk_col, int plane,
                           TX_SIZE tx_size, const tran_low_t *tcoeff,
                           uint16_t eob, TXB_CTX *txb_ctx) {
-  const PLANE_TYPE plane_type = get_plane_type(plane);
   const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size);
+  FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
+  aom_write_symbol(w, eob == 0,
+                   ec_ctx->txb_skip_cdf[txs_ctx][txb_ctx->txb_skip_ctx], 2);
+  if (eob == 0) return;
+  const PLANE_TYPE plane_type = get_plane_type(plane);
   const TX_TYPE tx_type = av1_get_tx_type(plane_type, xd, blk_row, blk_col,
                                           tx_size, cm->reduced_tx_set_used);
   const TX_CLASS tx_class = tx_type_to_class[tx_type];
@@ -507,18 +511,10 @@
   const int bwl = get_txb_bwl(tx_size);
   const int width = get_txb_wide(tx_size);
   const int height = get_txb_high(tx_size);
-  FRAME_CONTEXT *ec_ctx = xd->tile_ctx;
+
   uint8_t levels_buf[TX_PAD_2D];
   uint8_t *const levels = set_levels(levels_buf, width);
   DECLARE_ALIGNED(16, int8_t, coeff_contexts[MAX_TX_SQUARE]);
-
-  aom_write_symbol(w, eob == 0,
-                   ec_ctx->txb_skip_cdf[txs_ctx][txb_ctx->txb_skip_ctx], 2);
-  if (plane == 0 && eob == 0) {
-    assert(tx_type == DCT_DCT);
-  }
-  if (eob == 0) return;
-
   av1_txb_init_levels(tcoeff, width, height, levels);
 
   av1_write_tx_type(cm, xd, blk_row, blk_col, plane, tx_size, w);

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 4c419be..6299551 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -3177,7 +3177,6 @@
   assert(best_rd != INT64_MAX);
 
   best_rd_stats->skip = best_eob == 0;
-  if (best_eob == 0) best_tx_type = DCT_DCT;
   if (plane == 0) {
     update_txk_array(mbmi->txk_type, plane_bsize, blk_row, blk_col, tx_size,
                      best_tx_type);