commit 4be1a4d49ba70339132242c601b6690494d4e63d
author Jingning Han <jingning@google.com> Fri Jan 06 10:59:20 2017 -0800
committer Jingning Han <jingning@google.com> Fri Jan 06 23:57:05 2017 +0000
tree 6d57760b699fdae689173e07bcef8a903a367f2d
parent 8324fa85ce1d44f713e2bd23ac7be8405b227d7d

Fix frame header tx_size syntax setting

Fix an intricacy due to interactions between cb4x4 and var-tx that
sets frame header away from tx_mode_select. This resolves a rare
enc/dec mismatch issue.

Change-Id: I6981f21f7e6f04f2a47ef32f744f83a8fd34355b

av1/common/blockd.h

diff --git a/av1/common/blockd.h b/av1/common/blockd.h
index 19a4fcd..4deb059 100644
--- a/av1/common/blockd.h
+++ b/av1/common/blockd.h
@@ -680,8 +680,13 @@
 #if CONFIG_VAR_TX
   const TX_SIZE max_tx_size = max_txsize_rect_lookup[bsize];
 
+#if CONFIG_CB4X4
+  if (!is_inter || bsize == BLOCK_4X4)
+    return AOMMIN(max_txsize_lookup[bsize], largest_tx_size);
+#else
   if (!is_inter || bsize < BLOCK_8X8)
     return AOMMIN(max_txsize_lookup[bsize], largest_tx_size);
+#endif
 
   if (txsize_sqr_map[max_tx_size] <= largest_tx_size)
     return max_tx_size;

av1/decoder/decodemv.c

diff --git a/av1/decoder/decodemv.c b/av1/decoder/decodemv.c
index 3a5f25d..31f42b6 100644
--- a/av1/decoder/decodemv.c
+++ b/av1/decoder/decodemv.c
@@ -415,7 +415,11 @@
   TX_MODE tx_mode = cm->tx_mode;
   BLOCK_SIZE bsize = xd->mi[0]->mbmi.sb_type;
   if (xd->lossless[xd->mi[0]->mbmi.segment_id]) return TX_4X4;
+#if CONFIG_CB4X4 && CONFIG_VAR_TX
+  if (bsize > BLOCK_4X4) {
+#else
   if (bsize >= BLOCK_8X8) {
+#endif
     if (allow_select && tx_mode == TX_MODE_SELECT) {
       const TX_SIZE coded_tx_size =
           read_selected_tx_size(cm, xd, inter_tx_size_cat_lookup[bsize], r);