commit da3d94fe2c0a993b42c2146ddf6e669fa6be7e0a
author Alex Converse <aconverse@google.com> Wed Mar 15 14:54:29 2017 -0700
committer Alex Converse <aconverse@google.com> Thu Mar 16 16:36:19 2017 +0000
tree 665c0a61f3c602c49bbf4499529b4e3989e3c5c3
parent e063e2dd3dadeb4812189526ab76f4f718ae7e4c

Account for elided extrabits during rate cost calculation.

Fixes some rd-debug mismatches coding cat6 tokens with tx size < 32x32.
For these tokens the high extrabits are elided during tokenization and
detokenization, but the rd cost was computed with the old tables from
VP9 where these high extrabits are always coded.

Change-Id: I4a9a6ea822ff821e1932c351d43a57bdb4d6d466

av1/encoder/encodemb.c

diff --git a/av1/encoder/encodemb.c b/av1/encoder/encodemb.c
index 38051cc..c2c68a5 100644
--- a/av1/encoder/encodemb.c
+++ b/av1/encoder/encodemb.c
@@ -153,9 +153,9 @@
                                        : band_translate[eob - 1];
   int pt, i, final_eob;
 #if CONFIG_AOM_HIGHBITDEPTH
-  const int *cat6_high_cost = av1_get_high_cost_table(xd->bd);
+  const int cat6_bits = av1_get_cat6_extrabits_size(tx_size, xd->bd);
 #else
-  const int *cat6_high_cost = av1_get_high_cost_table(8);
+  const int cat6_bits = av1_get_cat6_extrabits_size(tx_size, 8);
 #endif
   unsigned int(*token_costs)[2][COEFF_CONTEXTS][ENTROPY_TOKENS] =
       mb->token_costs[txsize_sqr_map[tx_size]][plane_type][ref];
@@ -182,7 +182,7 @@
 
   for (i = 0; i < eob; i++) {
     const int rc = scan[i];
-    tokens[i][0].rate = av1_get_token_cost(qcoeff[rc], &t0, cat6_high_cost);
+    tokens[i][0].rate = av1_get_token_cost(qcoeff[rc], &t0, cat6_bits);
     tokens[i][0].token = t0;
     token_cache[rc] = av1_pt_energy_class[t0];
   }
@@ -302,7 +302,7 @@
         t1 = tokens[next][1].token == EOB_TOKEN ? EOB_TOKEN : ZERO_TOKEN;
         base_bits = 0;
       } else {
-        base_bits = av1_get_token_cost(x, &t0, cat6_high_cost);
+        base_bits = av1_get_token_cost(x, &t0, cat6_bits);
         t1 = t0;
       }