commit 008ea46a191b954434833088a50bc789e723622e
author Jingning Han <jingning@google.com> Thu Mar 23 11:53:51 2017 -0700
committer Jingning Han <jingning@google.com> Thu Mar 23 22:34:37 2017 +0000
tree bc93336da1a71083f58a69c28d91b5b16606805f
parent e9f7742437eb4bbcd6b28d726febc1646ca8d889

Account for 2x2 tx_size in extrabits_size

Offset the transform block size with respect to TX_4X4 when
counting the cat6 extra bit size. This accounts for the potential
use of 2x2 transform block size in cb4x4.

BUG=aomedia:405

Change-Id: Ia15ac123e4f31dc66455c3aa6925ae5e754de493

av1/common/entropy.h

diff --git a/av1/common/entropy.h b/av1/common/entropy.h
index 7d0851a..33f287a 100644
--- a/av1/common/entropy.h
+++ b/av1/common/entropy.h
@@ -119,7 +119,12 @@
   // TODO(debargha): Does TX_64X64 require an additional extrabit?
   if (tx_size > TX_32X32) tx_size = TX_32X32;
 #endif
-  int bits = (int)bit_depth + 3 + (int)tx_size;
+#if CONFIG_CB4X4
+  int tx_offset = (tx_size < TX_4X4) ? 0 : (int)(tx_size - TX_4X4);
+#else
+  int tx_offset = (int)(tx_size - TX_4X4);
+#endif
+  int bits = (int)bit_depth + 3 + tx_offset;
   assert(bits <= (int)sizeof(av1_cat6_prob));
   return bits;
 }