commit 8aec7f30e79647afa5a37f163303105422bcd0cc
author Debargha Mukherjee <debargha@google.com> Wed Dec 20 15:48:49 2017 -0800
committer Debargha Mukherjee <debargha@google.com> Fri Dec 22 17:33:25 2017 +0000
tree c7d51ea6657f825d45e8e7bdc5ed8258570abcd5
parent 80592c72f214ca46502442cb60c1b37e69bd971d

Make chroma loopfiltering tx_sizes consistent

Removes existing inconsistencies between chroma tx_sizes
used for chroma loopfiltering.
Includes various refactoring to remove the uv_txsize_lookup
array eventually.

BUG=aomedia:1090

Change-Id: Ib74299b41280ca3ebeaf9a9293242d531d68ad28

av1/encoder/rdopt.c

diff --git a/av1/encoder/rdopt.c b/av1/encoder/rdopt.c
index 42235ba..b32329e 100644
--- a/av1/encoder/rdopt.c
+++ b/av1/encoder/rdopt.c
@@ -4314,7 +4314,6 @@
   BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
   const int tx_row = blk_row >> (1 - pd->subsampling_y);
   const int tx_col = blk_col >> (1 - pd->subsampling_x);
-  TX_SIZE plane_tx_size;
   const int max_blocks_high = max_block_high(xd, plane_bsize, plane);
   const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane);
 
@@ -4322,8 +4321,8 @@
 
   if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
 
-  plane_tx_size =
-      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
+  const TX_SIZE plane_tx_size =
+      plane ? av1_get_uv_tx_size_vartx(mbmi, pd, bsize, tx_row, tx_col)
             : mbmi->inter_tx_size[tx_row][tx_col];
 
   int ctx = txfm_partition_context(tx_above + blk_col, tx_left + blk_row,
@@ -4332,7 +4331,7 @@
   av1_init_rd_stats(rd_stats);
   if (tx_size == plane_tx_size
 #if DISABLE_VARTX_FOR_CHROMA
-      || pd->subsampling_x || pd->subsampling_y
+      || plane
 #endif  // DISABLE_VARTX_FOR_CHROMA
       ) {
     ENTROPY_CONTEXT *ta = above_ctx + blk_col;
@@ -5020,7 +5019,6 @@
   BLOCK_SIZE bsize = txsize_to_bsize[tx_size];
   const int tx_row = blk_row >> (1 - pd->subsampling_y);
   const int tx_col = blk_col >> (1 - pd->subsampling_x);
-  TX_SIZE plane_tx_size;
   const int max_blocks_high = max_block_high(xd, plane_bsize, plane);
   const int max_blocks_wide = max_block_wide(xd, plane_bsize, plane);
 
@@ -5028,13 +5026,13 @@
 
   if (blk_row >= max_blocks_high || blk_col >= max_blocks_wide) return;
 
-  plane_tx_size =
-      plane ? uv_txsize_lookup[bsize][mbmi->inter_tx_size[tx_row][tx_col]][0][0]
+  const TX_SIZE plane_tx_size =
+      plane ? av1_get_uv_tx_size_vartx(mbmi, pd, bsize, tx_row, tx_col)
             : mbmi->inter_tx_size[tx_row][tx_col];
 
   if (tx_size == plane_tx_size
 #if DISABLE_VARTX_FOR_CHROMA
-      || pd->subsampling_x || pd->subsampling_y
+      || plane
 #endif  // DISABLE_VARTX_FOR_CHROMA
       ) {
     ENTROPY_CONTEXT *ta = above_ctx + blk_col;