commit da3d94fe2c0a993b42c2146ddf6e669fa6be7e0a
author Alex Converse <aconverse@google.com> Wed Mar 15 14:54:29 2017 -0700
committer Alex Converse <aconverse@google.com> Thu Mar 16 16:36:19 2017 +0000
tree 665c0a61f3c602c49bbf4499529b4e3989e3c5c3
parent e063e2dd3dadeb4812189526ab76f4f718ae7e4c

Account for elided extrabits during rate cost calculation.

Fixes some rd-debug mismatches coding cat6 tokens with tx size < 32x32.
For these tokens the high extrabits are elided during tokenization and
detokenization, but the rd cost was computed with the old tables from
VP9 where these high extrabits are always coded.

Change-Id: I4a9a6ea822ff821e1932c351d43a57bdb4d6d466

av1/encoder/bitstream.c

diff --git a/av1/encoder/bitstream.c b/av1/encoder/bitstream.c
index eaec9b8..3a45f47 100644
--- a/av1/encoder/bitstream.c
+++ b/av1/encoder/bitstream.c
@@ -846,10 +846,10 @@
       const int bit_string_length = extra_bits->len;  // Length of extra bits to
       // be written excluding
       // the sign bit.
-      int skip_bits =
-          (extra_bits->base_val == CAT6_MIN_VAL)
-              ? TX_SIZES - 1 - txsize_sqr_up_map[tx_size] + (12 - bit_depth)
-              : 0;
+      int skip_bits = (extra_bits->base_val == CAT6_MIN_VAL)
+                          ? (int)sizeof(av1_cat6_prob) -
+                                av1_get_cat6_extrabits_size(tx_size, bit_depth)
+                          : 0;
 
       if (bit_string_length > 0) {
         const unsigned char *pb = extra_bits->prob;
@@ -943,10 +943,10 @@
       const int bit_string_length = extra_bits->len;  // Length of extra bits to
                                                       // be written excluding
                                                       // the sign bit.
-      int skip_bits =
-          (extra_bits->base_val == CAT6_MIN_VAL)
-              ? TX_SIZES - 1 - txsize_sqr_up_map[tx_size] + (12 - bit_depth)
-              : 0;
+      int skip_bits = (extra_bits->base_val == CAT6_MIN_VAL)
+                          ? (int)sizeof(av1_cat6_prob) -
+                                av1_get_cat6_extrabits_size(tx_size, bit_depth)
+                          : 0;
       if (bit_string_length > 0) {
         const unsigned char *pb = extra_bits->prob;
         const int value = bit_string >> 1;